implement fetch and flush

This commit is contained in:
Roberto Hidalgo 2022-12-18 12:16:46 -06:00
parent def0f4619e
commit ba63cc2419
8 changed files with 293 additions and 103 deletions

View File

@ -11,3 +11,82 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package cmd package cmd
import (
"fmt"
"io/fs"
"os"
"git.rob.mx/nidito/joao/internal/command"
"git.rob.mx/nidito/joao/internal/registry"
"git.rob.mx/nidito/joao/pkg/config"
"github.com/sirupsen/logrus"
)
func init() {
registry.Register(fetchCommand)
}
var fetchCommand = (&command.Command{
Path: []string{"fetch"},
Summary: "fetches configuration values from 1Password",
Description: `Fetches secrets for local ﹅CONFIG﹅ files from 1Password.`,
Arguments: command.Arguments{
{
Name: "config",
Description: "The configuration file(s) to fetch",
Required: false,
Variadic: true,
Values: &command.ValueSource{
Files: &[]string{"joao.yaml"},
},
},
},
Options: command.Options{
"dry-run": {
Description: "Don't persist to the filesystem",
Type: "bool",
},
},
Action: func(cmd *command.Command) error {
paths := cmd.Arguments[0].ToValue().([]string)
for _, path := range paths {
remote, err := config.Load(path, true)
if err != nil {
return err
}
local, err := config.Load(path, false)
if err != nil {
return err
}
if err = local.Merge(remote); err != nil {
return err
}
b, err := local.AsYAML(false)
if err != nil {
return err
}
if dryRun := cmd.Options["dry-run"].ToValue().(bool); dryRun {
logrus.Warnf("dry-run: would write to %s", path)
_, _ = cmd.Cobra.OutOrStdout().Write(b)
} else {
var mode fs.FileMode = 0644
if info, err := os.Stat(path); err == nil {
mode = info.Mode().Perm()
}
if err := os.WriteFile(path, b, mode); err != nil {
return fmt.Errorf("could not save changes to %s: %w", path, err)
}
}
logrus.Infof("Updated %s", path)
}
logrus.Info("Done")
return nil
},
}).SetBindings()

View File

@ -11,3 +11,61 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package cmd package cmd
import (
"fmt"
"git.rob.mx/nidito/joao/internal/command"
opclient "git.rob.mx/nidito/joao/internal/op-client"
"git.rob.mx/nidito/joao/internal/registry"
"git.rob.mx/nidito/joao/pkg/config"
"github.com/sirupsen/logrus"
)
func init() {
registry.Register(flushCommand)
}
var flushCommand = (&command.Command{
Path: []string{"flush"},
Summary: "flush configuration values to 1Password",
Description: `Creates or updates existing items for every ﹅CONFIG﹅ file provided. Does not delete 1Password items.`,
Arguments: command.Arguments{
{
Name: "config",
Description: "The configuration file(s) to flush",
Required: false,
Variadic: true,
Values: &command.ValueSource{
Files: &[]string{"yaml"},
},
},
},
Options: command.Options{
"dry-run": {
Description: "Don't persist to 1Password",
Type: "bool",
},
},
Action: func(cmd *command.Command) error {
paths := cmd.Arguments[0].ToValue().([]string)
if dryRun := cmd.Options["dry-run"].ToValue().(bool); dryRun {
opclient.Use(&opclient.CLI{DryRun: true})
}
for _, path := range paths {
cfg, err := config.Load(path, false)
if err != nil {
return err
}
if err := opclient.Update(cfg.Vault, cfg.Name, cfg.ToOP()); err != nil {
return fmt.Errorf("could not flush to 1password: %w", err)
}
}
logrus.Info("Done")
return nil
},
}).SetBindings()

View File

@ -25,7 +25,9 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
type CLI struct{} type CLI struct {
DryRun bool // Won't write to 1Password
}
func invoke(vault string, args ...string) (bytes.Buffer, error) { func invoke(vault string, args ...string) (bytes.Buffer, error) {
if vault != "" { if vault != "" {
@ -74,9 +76,9 @@ func (b *CLI) Get(vault, name string) (*op.Item, error) {
return item, nil return item, nil
} }
func (b *CLI) create(item *op.Item) error { func (b *CLI) Create(item *op.Item) error {
logrus.Infof("Creating new item: %s/%s", item.Vault.ID, item.Title) logrus.Infof("Creating new item: %s/%s", item.Vault.ID, item.Title)
cmd := exec.Command("op", "--vault", shellescape.Quote(item.Vault.ID), "item", "create") cmd := exec.Command("op", "--vault", shellescape.Quote(item.Vault.ID), "item", "create") // nolint: gosec
itemJSON, err := json.Marshal(item) itemJSON, err := json.Marshal(item)
if err != nil { if err != nil {
@ -89,6 +91,10 @@ func (b *CLI) create(item *op.Item) error {
cmd.Stdout = &stdout cmd.Stdout = &stdout
var stderr bytes.Buffer var stderr bytes.Buffer
cmd.Stderr = &stderr cmd.Stderr = &stderr
if b.DryRun {
logrus.Warnf("dry-run: Would have invoked %v", cmd.Args)
return nil
}
if err := cmd.Run(); err != nil { if err := cmd.Run(); err != nil {
return fmt.Errorf("could not create item: %w", err) return fmt.Errorf("could not create item: %w", err)
} }
@ -100,41 +106,8 @@ func (b *CLI) create(item *op.Item) error {
return nil return nil
} }
type hashResult int func (b *CLI) Update(item *op.Item, remote *op.Item) error {
args := []string{"item", "edit", item.Title, "--"}
const (
HashItemError hashResult = iota
HashItemMissing
HashMatch
HashMismatch
)
func keyForField(field *op.ItemField) string {
name := strings.ReplaceAll(field.Label, ".", "\\.")
if field.Section != nil {
name = field.Section.ID + "." + name
}
return name
}
func (b *CLI) Update(vault, name string, item *op.Item) error {
remote, err := b.Get(vault, name)
if err != nil {
if strings.Contains(err.Error(), fmt.Sprintf("\"%s\" isn't an item in the ", name)) {
return b.create(item)
}
return fmt.Errorf("could not fetch remote 1password item to compare against: %w", err)
}
if remote.GetValue("password") == item.GetValue("password") {
logrus.Warn("item is already up to date")
return nil
}
logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title)
args := []string{"item", "edit", name, "--"}
localKeys := map[string]int{} localKeys := map[string]int{}
for _, field := range item.Fields { for _, field := range item.Fields {
@ -158,7 +131,11 @@ func (b *CLI) Update(vault, name string, item *op.Item) error {
} }
} }
stdout, err := invoke(vault, args...) if b.DryRun {
logrus.Warnf("dry-run: Would have invoked op %v", args)
return nil
}
stdout, err := invoke(item.Vault.ID, args...)
if err != nil { if err != nil {
logrus.Errorf("op stderr: %s", stdout.String()) logrus.Errorf("op stderr: %s", stdout.String())
return err return err

View File

@ -13,14 +13,19 @@
package opclient package opclient
import ( import (
"fmt"
"strings"
op "github.com/1Password/connect-sdk-go/onepassword" op "github.com/1Password/connect-sdk-go/onepassword"
"github.com/sirupsen/logrus"
) )
var client opClient var client opClient
type opClient interface { type opClient interface {
Get(vault, name string) (*op.Item, error) Get(vault, name string) (*op.Item, error)
Update(vault, name string, item *op.Item) error Update(item *op.Item, remote *op.Item) error
Create(item *op.Item) error
List(vault, prefix string) ([]string, error) List(vault, prefix string) ([]string, error)
} }
@ -37,9 +42,32 @@ func Get(vault, name string) (*op.Item, error) {
} }
func Update(vault, name string, item *op.Item) error { func Update(vault, name string, item *op.Item) error {
return client.Update(vault, name, item) remote, err := client.Get(vault, name)
if err != nil {
if strings.Contains(err.Error(), fmt.Sprintf("\"%s\" isn't an item in the ", name)) {
return client.Create(item)
}
return fmt.Errorf("could not fetch remote 1password item to compare against: %w", err)
}
if remote.GetValue("password") == item.GetValue("password") {
logrus.Warn("item is already up to date")
return nil
}
logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title)
return client.Update(item, remote)
} }
func List(vault, prefix string) ([]string, error) { func List(vault, prefix string) ([]string, error) {
return client.List(vault, prefix) return client.List(vault, prefix)
} }
func keyForField(field *op.ItemField) string {
name := strings.ReplaceAll(field.Label, ".", "\\.")
if field.Section != nil {
name = field.Section.ID + "." + name
}
return name
}

View File

@ -14,13 +14,14 @@ package config
import ( import (
"bytes" "bytes"
"crypto/md5"
"encoding/json" "encoding/json"
"fmt" "fmt"
"sort"
"strings" "strings"
op "github.com/1Password/connect-sdk-go/onepassword" op "github.com/1Password/connect-sdk-go/onepassword"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"golang.org/x/crypto/blake2b"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@ -66,17 +67,39 @@ func (cfg *Config) ToMap() map[string]any {
return ret return ret
} }
func checksum(fields []*op.ItemField) string {
newHash, err := blake2b.New256(nil)
if err != nil {
panic(err)
}
// newHash := md5.New()
df := []string{}
for _, field := range fields {
if field.ID == "password" || field.ID == "notesPlain" || (field.Section != nil && field.Section.ID == "~annotations") {
continue
}
label := field.Label
if field.Section != nil && field.Section.ID != "" {
label = field.Section.ID + "." + label
}
df = append(df, label+field.Value)
}
sort.Strings(df)
newHash.Write([]byte(strings.Join(df, "")))
checksum := newHash.Sum(nil)
return fmt.Sprintf("%x", checksum)
}
// ToOp turns a config into an 1Password Item. // ToOp turns a config into an 1Password Item.
func (cfg *Config) ToOP() *op.Item { func (cfg *Config) ToOP() *op.Item {
sections := []*op.ItemSection{annotationsSection} sections := []*op.ItemSection{annotationsSection}
fields := append([]*op.ItemField{}, defaultItemFields...) fields := append([]*op.ItemField{}, defaultItemFields...)
newHash := md5.New()
datafields := cfg.Tree.ToOP() datafields := cfg.Tree.ToOP()
for _, field := range datafields { cs := checksum(datafields)
newHash.Write([]byte(field.ID + field.Value))
} fields[0].Value = cs
fields[0].Value = fmt.Sprintf("%x", newHash.Sum(nil))
fields = append(fields, datafields...) fields = append(fields, datafields...)
for i := 0; i < len(cfg.Tree.Content); i += 2 { for i := 0; i < len(cfg.Tree.Content); i += 2 {
@ -107,7 +130,7 @@ func (cfg *Config) MarshalYAML() (any, error) {
return cfg.Tree.MarshalYAML() return cfg.Tree.MarshalYAML()
} }
// AsYAML returns the config encoded as YAML // AsYAML returns the config encoded as YAML.
func (cfg *Config) AsYAML(redacted bool) ([]byte, error) { func (cfg *Config) AsYAML(redacted bool) ([]byte, error) {
redactOutput = redacted redactOutput = redacted
var out bytes.Buffer var out bytes.Buffer
@ -221,21 +244,23 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er
} }
if child := entry.ChildNamed(key); child != nil { if child := entry.ChildNamed(key); child != nil {
logrus.Infof("found child named %s, with len %v", key, len(child.Content))
entry = child entry = child
continue continue
} }
logrus.Infof("no child named %s found in %s", key, entry.Name())
kind := yaml.MappingNode kind := yaml.MappingNode
if isNumeric(key) { if isNumeric(key) {
kind = yaml.SequenceNode kind = yaml.SequenceNode
} }
sub := NewEntry(key, kind) sub := NewEntry(key, kind)
sub.Path = append(entry.Path, key) sub.Path = append(entry.Path, key) // nolint: gocritic
entry.Content = append(entry.Content, sub) entry.Content = append(entry.Content, sub)
entry = sub entry = sub
} }
return nil return nil
} }
func (cfg *Config) Merge(other *Config) error {
return cfg.Tree.Merge(other.Tree)
}

View File

@ -14,7 +14,6 @@ package config
import ( import (
"fmt" "fmt"
"strconv"
"strings" "strings"
op "github.com/1Password/connect-sdk-go/onepassword" op "github.com/1Password/connect-sdk-go/onepassword"
@ -31,10 +30,8 @@ func isNumeric(s string) bool {
return true return true
} }
type secretValue string
// Entry is a configuration entry. // Entry is a configuration entry.
// Basically a copy of a yaml.Node with extra methods // Basically a copy of a yaml.Node with extra methods.
type Entry struct { type Entry struct {
Value string Value string
Kind yaml.Kind Kind yaml.Kind
@ -59,7 +56,7 @@ func NewEntry(name string, kind yaml.Kind) *Entry {
} }
} }
func copyFromNode(e *Entry, n *yaml.Node) *Entry { func (e *Entry) copyFromNode(n *yaml.Node) {
if e.Content == nil { if e.Content == nil {
e.Content = []*Entry{} e.Content = []*Entry{}
} }
@ -74,7 +71,6 @@ func copyFromNode(e *Entry, n *yaml.Node) *Entry {
e.Line = n.Line e.Line = n.Line
e.Column = n.Column e.Column = n.Column
e.Type = n.ShortTag() e.Type = n.ShortTag()
return e
} }
func (e *Entry) String() string { func (e *Entry) String() string {
@ -91,7 +87,7 @@ func (e *Entry) ChildNamed(name string) *Entry {
} }
func (e *Entry) SetPath(parent []string, current string) { func (e *Entry) SetPath(parent []string, current string) {
e.Path = append(parent, current) e.Path = append(parent, current) // nolint: gocritic
switch e.Kind { switch e.Kind {
case yaml.MappingNode, yaml.DocumentNode: case yaml.MappingNode, yaml.DocumentNode:
for idx := 0; idx < len(e.Content); idx += 2 { for idx := 0; idx < len(e.Content); idx += 2 {
@ -101,19 +97,19 @@ func (e *Entry) SetPath(parent []string, current string) {
} }
case yaml.SequenceNode: case yaml.SequenceNode:
for idx, child := range e.Content { for idx, child := range e.Content {
child.Path = append(e.Path, fmt.Sprintf("%d", idx)) child.Path = append(e.Path, fmt.Sprintf("%d", idx)) // nolint: gocritic
} }
} }
} }
func (e *Entry) UnmarshalYAML(node *yaml.Node) error { func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
copyFromNode(e, node) e.copyFromNode(node)
switch node.Kind { switch node.Kind {
case yaml.SequenceNode, yaml.ScalarNode: case yaml.SequenceNode, yaml.ScalarNode:
for _, n := range node.Content { for _, n := range node.Content {
sub := &Entry{} sub := &Entry{}
copyFromNode(sub, n) sub.copyFromNode(n)
if err := n.Decode(&sub); err != nil { if err := n.Decode(&sub); err != nil {
return err return err
} }
@ -147,6 +143,10 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
return nil return nil
} }
func (e *Entry) IsScalar() bool {
return e.Kind != yaml.DocumentNode && e.Kind != yaml.MappingNode && e.Kind != yaml.SequenceNode
}
func (e *Entry) IsSecret() bool { func (e *Entry) IsSecret() bool {
return e.Tag == YAMLTypeSecret return e.Tag == YAMLTypeSecret
} }
@ -216,8 +216,7 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
annotations := map[string]string{} annotations := map[string]string{}
data := map[string]string{} data := map[string]string{}
for i := 0; i < len(fields); i++ { for _, field := range fields {
field := fields[i]
label := field.Label label := field.Label
if field.Section != nil { if field.Section != nil {
if field.Section.Label == "~annotations" { if field.Section.Label == "~annotations" {
@ -234,35 +233,12 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
} }
for label, valueStr := range data { for label, valueStr := range data {
var value any
var err error
var style yaml.Style var style yaml.Style
var tag string var tag string
switch annotations[label] { if annotations[label] == "secret" {
case "bool":
value, err = strconv.ParseBool(valueStr)
if err != nil {
return err
}
case "int":
value, err = strconv.ParseInt(valueStr, 10, 64)
if err != nil {
return err
}
case "float":
var err error
value, err = strconv.ParseFloat(valueStr, 64)
if err != nil {
return err
}
case "secret":
value = secretValue(value.(string))
style = yaml.TaggedStyle style = yaml.TaggedStyle
tag = YAMLTypeSecret tag = YAMLTypeSecret
default:
// either no annotation or an unknown value
value = valueStr
} }
path := strings.Split(label, ".") path := strings.Split(label, ".")
@ -270,7 +246,7 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
for idx, key := range path { for idx, key := range path {
if idx == len(path)-1 { if idx == len(path)-1 {
container.Content = append(container.Content, &Entry{ container.Content = append(container.Content, NewEntry(key, yaml.ScalarNode), &Entry{
Path: path, Path: path,
Kind: yaml.ScalarNode, Kind: yaml.ScalarNode,
Value: valueStr, Value: valueStr,
@ -289,8 +265,12 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
kind = yaml.SequenceNode kind = yaml.SequenceNode
} }
child := NewEntry(key, kind) child := NewEntry(key, kind)
child.Path = append(container.Path, key) child.Path = append(container.Path, key) // nolint: gocritic
if isNumeric(key) {
container.Content = append(container.Content, child) container.Content = append(container.Content, child)
} else {
container.Content = append(container.Content, NewEntry(child.Name(), child.Kind), child)
}
container = child container = child
} }
} }
@ -305,6 +285,7 @@ func (e *Entry) ToOP() []*op.ItemField {
if e.Kind == yaml.ScalarNode { if e.Kind == yaml.ScalarNode {
name := e.Path[len(e.Path)-1] name := e.Path[len(e.Path)-1]
fullPath := strings.Join(e.Path, ".")
if len(e.Path) > 1 { if len(e.Path) > 1 {
section = &op.ItemSection{ID: e.Path[0]} section = &op.ItemSection{ID: e.Path[0]}
name = strings.Join(e.Path[1:], ".") name = strings.Join(e.Path[1:], ".")
@ -313,20 +294,20 @@ func (e *Entry) ToOP() []*op.ItemField {
fieldType := "STRING" fieldType := "STRING"
if e.IsSecret() { if e.IsSecret() {
fieldType = "CONCEALED" fieldType = "CONCEALED"
} else { }
if annotationType := e.TypeStr(); annotationType != "" { if annotationType := e.TypeStr(); annotationType != "" {
ret = append(ret, &op.ItemField{ ret = append(ret, &op.ItemField{
ID: "~annotations." + strings.Join(e.Path, "."), ID: "~annotations." + fullPath,
Section: annotationsSection, Section: annotationsSection,
Label: name, Label: fullPath,
Type: "STRING", Type: "STRING",
Value: annotationType, Value: annotationType,
}) })
} }
}
ret = append(ret, &op.ItemField{ ret = append(ret, &op.ItemField{
ID: strings.Join(e.Path, "."), ID: fullPath,
Section: section, Section: section,
Label: name, Label: name,
Type: fieldType, Type: fieldType,
@ -385,3 +366,42 @@ func (e *Entry) AsMap() any {
} }
return ret return ret
} }
func (e *Entry) Merge(other *Entry) error {
if e.IsScalar() && other.IsScalar() {
e.Value = other.Value
e.Tag = other.Tag
e.Kind = other.Kind
e.Type = other.Type
return nil
}
if e.Kind == yaml.MappingNode || e.Kind == yaml.DocumentNode {
for i := 0; i < len(other.Content); i += 2 {
remote := other.Content[i+1]
local := e.ChildNamed(remote.Name())
if local != nil {
if err := local.Merge(remote); err != nil {
return err
}
} else {
e.Content = append(e.Content, NewEntry(remote.Name(), remote.Kind), remote)
}
}
return nil
}
for _, remote := range other.Content {
local := other.ChildNamed(remote.Name())
if local != nil {
if err := local.Merge(remote); err != nil {
return err
}
} else {
logrus.Debugf("adding new collection value at %s", remote.Path)
local.Content = append(local.Content, remote)
}
}
return nil
}

View File

@ -70,6 +70,9 @@ func FromOP(item *op.Item) (*Config, error) {
Tree: NewEntry("root", yaml.MappingNode), Tree: NewEntry("root", yaml.MappingNode),
} }
if cs := checksum(item.Fields); cs != item.GetValue("password") {
logrus.Warnf("1Password item changed and checksum was not updated. Expected %s, found %s", cs, item.GetValue("password"))
}
err := cfg.Tree.FromOP(item.Fields) err := cfg.Tree.FromOP(item.Fields)
return cfg, err return cfg, err
} }

View File

@ -28,12 +28,12 @@ import (
type opDetails struct { type opDetails struct {
Vault string `yaml:"vault"` Vault string `yaml:"vault"`
Name string `yaml:"name"` Name string `yaml:"name"`
NameTemplate string `yaml:"nameTemplate"` NameTemplate string `yaml:"nameTemplate"` // nolint: tagliatelle
Repo string Repo string
} }
type singleModeConfig struct { type singleModeConfig struct {
Config *opDetails `yaml:"_config,omitempty"` Config *opDetails `yaml:"_config,omitempty"` // nolint: tagliatelle
} }
func argIsYAMLFile(path string) bool { func argIsYAMLFile(path string) bool {