From def0f4619e65902107638bfb05f1747722145121 Mon Sep 17 00:00:00 2001 From: Roberto Hidalgo Date: Fri, 16 Dec 2022 23:40:43 -0600 Subject: [PATCH] delete removed fields --- cmd/get.go | 5 + cmd/set.go | 47 ++++++--- internal/op-client/cli.go | 73 +++++++------ pkg/config/config.go | 213 +++++++++----------------------------- pkg/config/entry.go | 12 ++- pkg/config/lookup.go | 50 +++++++++ pkg/config/readers.go | 75 ++++++++++++++ pkg/config/util.go | 11 ++ 8 files changed, 273 insertions(+), 213 deletions(-) create mode 100644 pkg/config/readers.go diff --git a/cmd/get.go b/cmd/get.go index 1a92f45..e3b39ab 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -16,6 +16,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "sort" "strings" "git.rob.mx/nidito/joao/internal/command" @@ -42,6 +43,8 @@ func keyFinder(cmd *command.Command, currentValue string) ([]string, cobra.Shell return nil, flag, err } + sort.Strings(keys) + return keys, cobra.ShellCompDirectiveDefault, nil } @@ -92,6 +95,8 @@ looks at the filesystem or remotely, using 1password (over the CLI if available, for k := range opts { options = append(options, k) } + sort.Strings(options) + return options, flag, err }, }, diff --git a/cmd/set.go b/cmd/set.go index 4e70e18..aa3c60d 100644 --- a/cmd/set.go +++ b/cmd/set.go @@ -33,10 +33,9 @@ var setCommand = (&command.Command{ Path: []string{"set"}, Summary: "updates configuration values", Description: ` -looks at the filesystem or remotely, using 1password (over the CLI if available, or 1password-connect, if configured). +Updates the value at ﹅PATH﹅ in a local ﹅CONFIG﹅ file. Specify ﹅--secret﹅ to keep the value secret, or ﹅--delete﹅ to delete the key at PATH. -Will read from stdin (or ﹅--from﹅ a file) and store it at the ﹅PATH -﹅ of ﹅CONFIG﹅, optionally ﹅--flush﹅ing to 1Password.`, +Will read values from stdin (or ﹅--from﹅ a file) and store it at the ﹅PATH﹅ of ﹅CONFIG﹅, optionally ﹅--flush﹅ing to 1Password.`, Arguments: command.Arguments{ { Name: "config", @@ -70,12 +69,16 @@ Will read from stdin (or ﹅--from﹅ a file) and store it at the ﹅PATH Description: "Store value as a secret string", Type: "bool", }, + "delete": { + Description: "Delete the value at the given PATH", + Type: "bool", + }, "json": { Description: "Treat input as JSON-encoded", Type: "bool", }, "flush": { - Description: "Save to 1Password after saving to file", + Description: "Save to 1Password after saving to PATH", Type: "bool", }, }, @@ -86,10 +89,23 @@ Will read from stdin (or ﹅--from﹅ a file) and store it at the ﹅PATH var cfg *config.Config var err error secret := cmd.Options["secret"].ToValue().(bool) + delete := cmd.Options["delete"].ToValue().(bool) input := cmd.Options["input"].ToValue().(string) parseJSON := cmd.Options["json"].ToValue().(bool) flush := cmd.Options["flush"].ToValue().(bool) + if secret && delete { + return fmt.Errorf("cannot --delete and set a --secret at the same time") + } + + if secret && parseJSON { + return fmt.Errorf("cannot set a --secret that is JSON encoded, encode individual values instead") + } + + if delete && input != "" { + logrus.Warn("Ignoring --file while deleting") + } + cfg, err = config.Load(path, false) if err != nil { return err @@ -97,23 +113,26 @@ Will read from stdin (or ﹅--from﹅ a file) and store it at the ﹅PATH parts := strings.Split(query, ".") - valueBytes, err := os.ReadFile(input) - if err != nil { - return err + if delete { + if err := cfg.Delete(parts); err != nil { + return err + } + } else { + valueBytes, err := os.ReadFile(input) + if err != nil { + return err + } + if err := cfg.Set(parts, valueBytes, secret, parseJSON); err != nil { + return err + } } - if err := cfg.Set(parts, valueBytes, secret, parseJSON); err != nil { - return err - } - - // b, err := cfg.AsJSON(false, true) b, err := cfg.AsYAML(false) if err != nil { return err } - var mode fs.FileMode = 644 - // var mode uint32 = + var mode fs.FileMode = 0644 if info, err := os.Stat(path); err == nil { mode = info.Mode().Perm() } diff --git a/internal/op-client/cli.go b/internal/op-client/cli.go index 63798b7..3d5c471 100644 --- a/internal/op-client/cli.go +++ b/internal/op-client/cli.go @@ -31,7 +31,18 @@ func invoke(vault string, args ...string) (bytes.Buffer, error) { if vault != "" { args = append([]string{"--vault", shellescape.Quote(vault)}, args...) } - logrus.Debugf("invoking op with args: %s", args) + + argString := "" + for _, arg := range args { + parts := strings.Split(arg, "]=") + if strings.HasSuffix(parts[0], "[password") { + parts[1] = "*****" + argString += fmt.Sprintf("%s]=%v", parts[0], parts[1]) + } else { + argString += " " + arg + } + } + logrus.Debugf("invoking op with args: %s", argString) cmd := exec.Command("op", args...) cmd.Env = os.Environ() @@ -98,53 +109,53 @@ const ( HashMismatch ) -func hashesMatch(item *op.Item) (hashResult, error) { - stdout, err := invoke(item.Vault.ID, "item", "get", "--fields", "label=password", item.Title) - if err != nil { - if strings.Contains(stdout.String(), fmt.Sprintf("\"%s\" isn't an item in the \"%s\" vault", item.Vault.ID, item.Title)) { - return HashItemMissing, nil - } - - return HashItemError, err +func keyForField(field *op.ItemField) string { + name := strings.ReplaceAll(field.Label, ".", "\\.") + if field.Section != nil { + name = field.Section.ID + "." + name } - - res := HashMismatch - if strings.TrimSpace(stdout.String()) == item.GetValue("password") { - res = HashMatch - } - return res, nil + return name } func (b *CLI) Update(vault, name string, item *op.Item) error { - status, err := hashesMatch(item) + remote, err := b.Get(vault, name) if err != nil { - return err + if strings.Contains(err.Error(), fmt.Sprintf("\"%s\" isn't an item in the ", name)) { + return b.create(item) + } + + return fmt.Errorf("could not fetch remote 1password item to compare against: %w", err) } - switch status { - case HashItemMissing: - return b.create(item) - case HashMatch: + if remote.GetValue("password") == item.GetValue("password") { logrus.Warn("item is already up to date") return nil - case HashMismatch: - logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title) } + logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title) + args := []string{"item", "edit", name, "--"} + localKeys := map[string]int{} for _, field := range item.Fields { - kind := strings.ToLower(field.Purpose) - if kind != "password" { + kind := "" + if field.Type == "CONCEALED" { + kind = "password" + } else { kind = "text" } - name := strings.ReplaceAll(field.Label, ".", "\\.") - if field.Section != nil { - name = field.Section.ID + "." + name - } - - key := fmt.Sprintf("%s[%s]", name, kind) + keyName := keyForField(field) + key := fmt.Sprintf("%s[%s]", keyName, kind) args = append(args, fmt.Sprintf("%s=%s", key, field.Value)) + localKeys[keyName] = 1 + } + + for _, field := range remote.Fields { + key := keyForField(field) + if _, exists := localKeys[key]; !exists { + logrus.Debugf("Deleting remote key %s", key) + args = append(args, key+"[delete]=") + } } stdout, err := invoke(vault, args...) diff --git a/pkg/config/config.go b/pkg/config/config.go index b24904b..83ecc63 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,8 +17,6 @@ import ( "crypto/md5" "encoding/json" "fmt" - "io/ioutil" - "sort" "strings" op "github.com/1Password/connect-sdk-go/onepassword" @@ -29,12 +27,6 @@ import ( const YAMLTypeSecret string = "!!secret" const YAMLTypeMetaConfig string = "!!joao" -type Config struct { - Vault string - Name string - Tree *Entry -} - var redactOutput = false var annotationsSection = &op.ItemSection{ ID: "~annotations", @@ -56,6 +48,13 @@ var defaultItemFields = []*op.ItemField{ }, } +type Config struct { + Vault string + Name string + Tree *Entry +} + +// ToMap turns a config into a dictionary of strings to values. func (cfg *Config) ToMap() map[string]any { ret := map[string]any{} for _, child := range cfg.Tree.Content { @@ -67,6 +66,7 @@ func (cfg *Config) ToMap() map[string]any { return ret } +// ToOp turns a config into an 1Password Item. func (cfg *Config) ToOP() *op.Item { sections := []*op.ItemSection{annotationsSection} fields := append([]*op.ItemField{}, defaultItemFields...) @@ -102,166 +102,12 @@ func (cfg *Config) ToOP() *op.Item { } } -type opDetails struct { - Vault string `yaml:"vault"` - Name string `yaml:"name"` - NameTemplate string `yaml:"nameTemplate"` - Repo string -} - -// type opConfig interface { -// Name() string -// Vault() string -// } - -// type inFileConfig struct { -// *opDetails -// *yaml.Node -// } - -// type virtualConfig struct { -// *opDetails -// } - -// func (ifc *inFileConfig) MarshalYAML() (any, error) { -// return ifc.Node, nil -// } - -// func (vc *virtualConfig) MarshalYAML() (any, error) { -// return nil, nil -// } - -// func (ifc *inFileConfig) UnmarshalYAML(node *yaml.Node) error { -// ifc.Node = node -// d := &opDetails{} - -// if err := node.Decode(&d); err != nil { -// return err -// } -// ifc.opDetails = d - -// return nil -// } - -// func (ifc *inFileConfig) Name() string { -// return ifc.opDetails.Name -// } - -// func (ifc *inFileConfig) Vault() string { -// return ifc.opDetails.Name -// } - -type singleModeConfig struct { - Config *opDetails `yaml:"_config,omitempty"` -} - -// FromFile reads a path and returns a config. -func FromFile(path string) (*Config, error) { - buf, err := ioutil.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("could not read file %s", path) - } - - if len(buf) == 0 { - buf = []byte("{}") - } - - name, vault, err := vaultAndNameFrom(path, buf) - if err != nil { - return nil, err - } - logrus.Debugf("Found name: %s and vault: %s", name, vault) - - cfg, err := FromYAML(buf) - if err != nil { - return nil, err - } - cfg.Name = name - cfg.Vault = vault - - return cfg, nil -} - -// FromYAML reads yaml bytes and returns a config. -func FromYAML(data []byte) (*Config, error) { - cfg := &Config{ - Tree: NewEntry("root", yaml.MappingNode), - } - - err := yaml.Unmarshal(data, &cfg.Tree) - if err != nil { - return nil, err - } - - return cfg, nil -} - -func scalarsIn(data map[string]yaml.Node, parents []string) ([]string, error) { - keys := []string{} - for key, leaf := range data { - if key == "_config" && len(parents) == 0 { - continue - } - switch leaf.Kind { - case yaml.ScalarNode: - newKey := strings.Join(append(parents, key), ".") - keys = append(keys, newKey) - case yaml.MappingNode, yaml.DocumentNode, yaml.SequenceNode: - sub := map[string]yaml.Node{} - if leaf.Kind == yaml.SequenceNode { - list := []yaml.Node{} - if err := leaf.Decode(&list); err != nil { - return keys, err - } - - for idx, child := range list { - sub[fmt.Sprintf("%d", idx)] = child - } - } else { - if err := leaf.Decode(&sub); err != nil { - return keys, err - } - } - ret, err := scalarsIn(sub, append(parents, key)) - if err != nil { - return keys, err - } - keys = append(keys, ret...) - default: - logrus.Fatalf("found unknown %v at %s", leaf.Kind, key) - } - } - - sort.Strings(keys) - return keys, nil -} - -func KeysFromYAML(data []byte) ([]string, error) { - cfg := map[string]yaml.Node{} - err := yaml.Unmarshal(data, &cfg) - if err != nil { - return nil, err - } - - return scalarsIn(cfg, []string{}) -} - -// FromOP reads a config from an op item and returns a config. -func FromOP(item *op.Item) (*Config, error) { - cfg := &Config{ - Vault: item.Vault.ID, - Name: item.Title, - Tree: NewEntry("root", yaml.MappingNode), - } - - err := cfg.Tree.FromOP(item.Fields) - return cfg, err -} - +// MarshalYAML implements `yaml.Marshal``. func (cfg *Config) MarshalYAML() (any, error) { return cfg.Tree.MarshalYAML() } +// AsYAML returns the config encoded as YAML func (cfg *Config) AsYAML(redacted bool) ([]byte, error) { redactOutput = redacted var out bytes.Buffer @@ -273,6 +119,7 @@ func (cfg *Config) AsYAML(redacted bool) ([]byte, error) { return out.Bytes(), nil } +// AsJSON returns the config enconded as JSON, optionally encoding as a 1Password item. func (cfg *Config) AsJSON(redacted bool, item bool) ([]byte, error) { var repr any if item { @@ -289,6 +136,44 @@ func (cfg *Config) AsJSON(redacted bool, item bool) ([]byte, error) { return bytes, nil } +// Delete a value at path. +func (cfg *Config) Delete(path []string) error { + parent := cfg.Tree + + for idx, key := range path { + if len(path)-1 == idx { + newContents := []*Entry{} + found := false + for idx, child := range parent.Content { + if child.Name() == key { + found = true + logrus.Debugf("Deleting %s", strings.Join(path, ".")) + if parent.Kind == yaml.DocumentNode || parent.Kind == yaml.MappingNode { + newContents = newContents[0 : idx-1] + } + continue + } + newContents = append(newContents, child) + } + + if !found { + return fmt.Errorf("no value found at %s", key) + } + + parent.Content = newContents + break + } + + parent = parent.ChildNamed(key) + if parent == nil { + return fmt.Errorf("no value found at %s", key) + } + } + + return nil +} + +// Set a new value, optionally parsing the supplied bytes as a secret or a JSON-encoded value. func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) error { newEntry := NewEntry(path[len(path)-1], yaml.ScalarNode) newEntry.Path = path diff --git a/pkg/config/entry.go b/pkg/config/entry.go index 4f24d63..f565edc 100644 --- a/pkg/config/entry.go +++ b/pkg/config/entry.go @@ -32,6 +32,9 @@ func isNumeric(s string) bool { } type secretValue string + +// Entry is a configuration entry. +// Basically a copy of a yaml.Node with extra methods type Entry struct { Value string Kind yaml.Kind @@ -44,7 +47,8 @@ type Entry struct { HeadComment string Line int Column int - Type string + // The ShortTag + Type string } func NewEntry(name string, kind yaml.Kind) *Entry { @@ -55,7 +59,7 @@ func NewEntry(name string, kind yaml.Kind) *Entry { } } -func CopyFromNode(e *Entry, n *yaml.Node) *Entry { +func copyFromNode(e *Entry, n *yaml.Node) *Entry { if e.Content == nil { e.Content = []*Entry{} } @@ -103,13 +107,13 @@ func (e *Entry) SetPath(parent []string, current string) { } func (e *Entry) UnmarshalYAML(node *yaml.Node) error { - CopyFromNode(e, node) + copyFromNode(e, node) switch node.Kind { case yaml.SequenceNode, yaml.ScalarNode: for _, n := range node.Content { sub := &Entry{} - CopyFromNode(sub, n) + copyFromNode(sub, n) if err := n.Decode(&sub); err != nil { return err } diff --git a/pkg/config/lookup.go b/pkg/config/lookup.go index 587184a..68121ae 100644 --- a/pkg/config/lookup.go +++ b/pkg/config/lookup.go @@ -17,6 +17,7 @@ import ( "os" "strings" + "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) @@ -52,3 +53,52 @@ func findRepoConfig(from string) (*opDetails, error) { return nil, nil } + +func scalarsIn(data map[string]yaml.Node, parents []string) ([]string, error) { + keys := []string{} + for key, leaf := range data { + if key == "_config" && len(parents) == 0 { + continue + } + switch leaf.Kind { + case yaml.ScalarNode: + newKey := strings.Join(append(parents, key), ".") + keys = append(keys, newKey) + case yaml.MappingNode, yaml.DocumentNode, yaml.SequenceNode: + sub := map[string]yaml.Node{} + if leaf.Kind == yaml.SequenceNode { + list := []yaml.Node{} + if err := leaf.Decode(&list); err != nil { + return keys, err + } + + for idx, child := range list { + sub[fmt.Sprintf("%d", idx)] = child + } + } else { + if err := leaf.Decode(&sub); err != nil { + return keys, err + } + } + ret, err := scalarsIn(sub, append(parents, key)) + if err != nil { + return keys, err + } + keys = append(keys, ret...) + default: + logrus.Fatalf("found unknown %v at %s", leaf.Kind, key) + } + } + + return keys, nil +} + +func KeysFromYAML(data []byte) ([]string, error) { + cfg := map[string]yaml.Node{} + err := yaml.Unmarshal(data, &cfg) + if err != nil { + return nil, err + } + + return scalarsIn(cfg, []string{}) +} diff --git a/pkg/config/readers.go b/pkg/config/readers.go new file mode 100644 index 0000000..4a50fd0 --- /dev/null +++ b/pkg/config/readers.go @@ -0,0 +1,75 @@ +// Copyright © 2022 Roberto Hidalgo +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "fmt" + "io/ioutil" + + op "github.com/1Password/connect-sdk-go/onepassword" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" +) + +// FromFile reads a path and returns a config. +func FromFile(path string) (*Config, error) { + buf, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("could not read file %s", path) + } + + if len(buf) == 0 { + buf = []byte("{}") + } + + name, vault, err := vaultAndNameFrom(path, buf) + if err != nil { + return nil, err + } + logrus.Debugf("Found name: %s and vault: %s", name, vault) + + cfg, err := FromYAML(buf) + if err != nil { + return nil, err + } + cfg.Name = name + cfg.Vault = vault + + return cfg, nil +} + +// FromYAML reads yaml bytes and returns a config. +func FromYAML(data []byte) (*Config, error) { + cfg := &Config{ + Tree: NewEntry("root", yaml.MappingNode), + } + + err := yaml.Unmarshal(data, &cfg.Tree) + if err != nil { + return nil, err + } + + return cfg, nil +} + +// FromOP reads a config from an op item and returns a config. +func FromOP(item *op.Item) (*Config, error) { + cfg := &Config{ + Vault: item.Vault.ID, + Name: item.Title, + Tree: NewEntry("root", yaml.MappingNode), + } + + err := cfg.Tree.FromOP(item.Fields) + return cfg, err +} diff --git a/pkg/config/util.go b/pkg/config/util.go index 761cd80..a721ded 100644 --- a/pkg/config/util.go +++ b/pkg/config/util.go @@ -25,6 +25,17 @@ import ( "gopkg.in/yaml.v3" ) +type opDetails struct { + Vault string `yaml:"vault"` + Name string `yaml:"name"` + NameTemplate string `yaml:"nameTemplate"` + Repo string +} + +type singleModeConfig struct { + Config *opDetails `yaml:"_config,omitempty"` +} + func argIsYAMLFile(path string) bool { return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") }