nidito
/
joao
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

implement fetch and flush

This commit is contained in:
Roberto Hidalgo 2022-12-18 12:16:46 -06:00
parent def0f4619e
commit ba63cc2419
8 changed files with 293 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
cmd/fetch.go
View File

@ -11,3 +11,82 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"fmt"
"io/fs"
"os"
"git.rob.mx/nidito/joao/internal/command"
"git.rob.mx/nidito/joao/internal/registry"
"git.rob.mx/nidito/joao/pkg/config"
"github.com/sirupsen/logrus"
)
func init() {
registry.Register(fetchCommand)
}
var fetchCommand = (&command.Command{
Path: []string{"fetch"},
Summary: "fetches configuration values from 1Password",
Description: `Fetches secrets for local ﹅CONFIG﹅ files from 1Password.`,
Arguments: command.Arguments{
{
Name: "config",
Description: "The configuration file(s) to fetch",
Required: false,
Variadic: true,
Values: &command.ValueSource{
Files: &[]string{"joao.yaml"},
},
},
},
Options: command.Options{
"dry-run": {
Description: "Don't persist to the filesystem",
Type: "bool",
},
},
Action: func(cmd *command.Command) error {
paths := cmd.Arguments[0].ToValue().([]string)
for _, path := range paths {
remote, err := config.Load(path, true)
if err != nil {
return err
}
local, err := config.Load(path, false)
if err != nil {
return err
}
if err = local.Merge(remote); err != nil {
return err
}
b, err := local.AsYAML(false)
if err != nil {
return err
}
if dryRun := cmd.Options["dry-run"].ToValue().(bool); dryRun {
logrus.Warnf("dry-run: would write to %s", path)
_, _ = cmd.Cobra.OutOrStdout().Write(b)
} else {
var mode fs.FileMode = 0644
if info, err := os.Stat(path); err == nil {
mode = info.Mode().Perm()
}
if err := os.WriteFile(path, b, mode); err != nil {
return fmt.Errorf("could not save changes to %s: %w", path, err)
}
}
logrus.Infof("Updated %s", path)
}
logrus.Info("Done")
return nil
},
}).SetBindings()

58
cmd/flush.go
View File

@ -11,3 +11,61 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"fmt"
"git.rob.mx/nidito/joao/internal/command"
opclient "git.rob.mx/nidito/joao/internal/op-client"
"git.rob.mx/nidito/joao/internal/registry"
"git.rob.mx/nidito/joao/pkg/config"
"github.com/sirupsen/logrus"
)
func init() {
registry.Register(flushCommand)
}
var flushCommand = (&command.Command{
Path: []string{"flush"},
Summary: "flush configuration values to 1Password",
Description: `Creates or updates existing items for every ﹅CONFIG﹅ file provided. Does not delete 1Password items.`,
Arguments: command.Arguments{
{
Name: "config",
Description: "The configuration file(s) to flush",
Required: false,
Variadic: true,
Values: &command.ValueSource{
Files: &[]string{"yaml"},
},
},
},
Options: command.Options{
"dry-run": {
Description: "Don't persist to 1Password",
Type: "bool",
},
},
Action: func(cmd *command.Command) error {
paths := cmd.Arguments[0].ToValue().([]string)
if dryRun := cmd.Options["dry-run"].ToValue().(bool); dryRun {
opclient.Use(&opclient.CLI{DryRun: true})
}
for _, path := range paths {
cfg, err := config.Load(path, false)
if err != nil {
return err
}
if err := opclient.Update(cfg.Vault, cfg.Name, cfg.ToOP()); err != nil {
return fmt.Errorf("could not flush to 1password: %w", err)
}
}
logrus.Info("Done")
return nil
},
}).SetBindings()

55
internal/op-client/cli.go
View File

@ -25,7 +25,9 @@ import (
"github.com/sirupsen/logrus"
)
type CLI struct{}
type CLI struct {
DryRun bool // Won't write to 1Password
}
func invoke(vault string, args ...string) (bytes.Buffer, error) {
if vault != "" {
@ -74,9 +76,9 @@ func (b *CLI) Get(vault, name string) (*op.Item, error) {
return item, nil
}
func (b *CLI) create(item *op.Item) error {
func (b *CLI) Create(item *op.Item) error {
logrus.Infof("Creating new item: %s/%s", item.Vault.ID, item.Title)
cmd := exec.Command("op", "--vault", shellescape.Quote(item.Vault.ID), "item", "create")
cmd := exec.Command("op", "--vault", shellescape.Quote(item.Vault.ID), "item", "create") // nolint: gosec
itemJSON, err := json.Marshal(item)
if err != nil {
@ -89,6 +91,10 @@ func (b *CLI) create(item *op.Item) error {
cmd.Stdout = &stdout
var stderr bytes.Buffer
cmd.Stderr = &stderr
if b.DryRun {
logrus.Warnf("dry-run: Would have invoked %v", cmd.Args)
return nil
}
if err := cmd.Run(); err != nil {
return fmt.Errorf("could not create item: %w", err)
}
@ -100,41 +106,8 @@ func (b *CLI) create(item *op.Item) error {
return nil
}
type hashResult int
const (
HashItemError hashResult = iota
HashItemMissing
HashMatch
HashMismatch
)
func keyForField(field *op.ItemField) string {
name := strings.ReplaceAll(field.Label, ".", "\\.")
if field.Section != nil {
name = field.Section.ID + "." + name
}
return name
}
func (b *CLI) Update(vault, name string, item *op.Item) error {
remote, err := b.Get(vault, name)
if err != nil {
if strings.Contains(err.Error(), fmt.Sprintf("\"%s\" isn't an item in the ", name)) {
return b.create(item)
}
return fmt.Errorf("could not fetch remote 1password item to compare against: %w", err)
}
if remote.GetValue("password") == item.GetValue("password") {
logrus.Warn("item is already up to date")
return nil
}
logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title)
args := []string{"item", "edit", name, "--"}
func (b *CLI) Update(item *op.Item, remote *op.Item) error {
args := []string{"item", "edit", item.Title, "--"}
localKeys := map[string]int{}
for _, field := range item.Fields {
@ -158,7 +131,11 @@ func (b *CLI) Update(vault, name string, item *op.Item) error {
}
}
stdout, err := invoke(vault, args...)
if b.DryRun {
logrus.Warnf("dry-run: Would have invoked op %v", args)
return nil
}
stdout, err := invoke(item.Vault.ID, args...)
if err != nil {
logrus.Errorf("op stderr: %s", stdout.String())
return err

32
internal/op-client/op.go
View File

@ -13,14 +13,19 @@
package opclient
import (
"fmt"
"strings"
op "github.com/1Password/connect-sdk-go/onepassword"
"github.com/sirupsen/logrus"
)
var client opClient
type opClient interface {
Get(vault, name string) (*op.Item, error)
Update(vault, name string, item *op.Item) error
Update(item *op.Item, remote *op.Item) error
Create(item *op.Item) error
List(vault, prefix string) ([]string, error)
}
@ -37,9 +42,32 @@ func Get(vault, name string) (*op.Item, error) {
}
func Update(vault, name string, item *op.Item) error {
return client.Update(vault, name, item)
remote, err := client.Get(vault, name)
if err != nil {
if strings.Contains(err.Error(), fmt.Sprintf("\"%s\" isn't an item in the ", name)) {
return client.Create(item)
}
return fmt.Errorf("could not fetch remote 1password item to compare against: %w", err)
}
if remote.GetValue("password") == item.GetValue("password") {
logrus.Warn("item is already up to date")
return nil
}
logrus.Infof("Item %s/%s already exists, updating", item.Vault.ID, item.Title)
return client.Update(item, remote)
}
func List(vault, prefix string) ([]string, error) {
return client.List(vault, prefix)
}
func keyForField(field *op.ItemField) string {
name := strings.ReplaceAll(field.Label, ".", "\\.")
if field.Section != nil {
name = field.Section.ID + "." + name
}
return name
}

45
pkg/config/config.go
View File

@ -14,13 +14,14 @@ package config
import (
"bytes"
"crypto/md5"
"encoding/json"
"fmt"
"sort"
"strings"
op "github.com/1Password/connect-sdk-go/onepassword"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/blake2b"
"gopkg.in/yaml.v3"
)
@ -66,17 +67,39 @@ func (cfg *Config) ToMap() map[string]any {
return ret
}
func checksum(fields []*op.ItemField) string {
newHash, err := blake2b.New256(nil)
if err != nil {
panic(err)
}
// newHash := md5.New()
df := []string{}
for _, field := range fields {
if field.ID == "password" || field.ID == "notesPlain" || (field.Section != nil && field.Section.ID == "~annotations") {
continue
}
label := field.Label
if field.Section != nil && field.Section.ID != "" {
label = field.Section.ID + "." + label
}
df = append(df, label+field.Value)
}
sort.Strings(df)
newHash.Write([]byte(strings.Join(df, "")))
checksum := newHash.Sum(nil)
return fmt.Sprintf("%x", checksum)
}
// ToOp turns a config into an 1Password Item.
func (cfg *Config) ToOP() *op.Item {
sections := []*op.ItemSection{annotationsSection}
fields := append([]*op.ItemField{}, defaultItemFields...)
newHash := md5.New()
datafields := cfg.Tree.ToOP()
for _, field := range datafields {
newHash.Write([]byte(field.ID + field.Value))
}
fields[0].Value = fmt.Sprintf("%x", newHash.Sum(nil))
cs := checksum(datafields)
fields[0].Value = cs
fields = append(fields, datafields...)
for i := 0; i < len(cfg.Tree.Content); i += 2 {
@ -107,7 +130,7 @@ func (cfg *Config) MarshalYAML() (any, error) {
return cfg.Tree.MarshalYAML()
}
// AsYAML returns the config encoded as YAML
// AsYAML returns the config encoded as YAML.
func (cfg *Config) AsYAML(redacted bool) ([]byte, error) {
redactOutput = redacted
var out bytes.Buffer
@ -221,21 +244,23 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er
}
if child := entry.ChildNamed(key); child != nil {
logrus.Infof("found child named %s, with len %v", key, len(child.Content))
entry = child
continue
}
logrus.Infof("no child named %s found in %s", key, entry.Name())
kind := yaml.MappingNode
if isNumeric(key) {
kind = yaml.SequenceNode
}
sub := NewEntry(key, kind)
sub.Path = append(entry.Path, key)
sub.Path = append(entry.Path, key) // nolint: gocritic
entry.Content = append(entry.Content, sub)
entry = sub
}
return nil
}
func (cfg *Config) Merge(other *Config) error {
return cfg.Tree.Merge(other.Tree)
}

106
pkg/config/entry.go
View File

@ -14,7 +14,6 @@ package config
import (
"fmt"
"strconv"
"strings"
op "github.com/1Password/connect-sdk-go/onepassword"
@ -31,10 +30,8 @@ func isNumeric(s string) bool {
return true
}
type secretValue string
// Entry is a configuration entry.
// Basically a copy of a yaml.Node with extra methods
// Basically a copy of a yaml.Node with extra methods.
type Entry struct {
Value string
Kind yaml.Kind
@ -59,7 +56,7 @@ func NewEntry(name string, kind yaml.Kind) *Entry {
}
}
func copyFromNode(e *Entry, n *yaml.Node) *Entry {
func (e *Entry) copyFromNode(n *yaml.Node) {
if e.Content == nil {
e.Content = []*Entry{}
}
@ -74,7 +71,6 @@ func copyFromNode(e *Entry, n *yaml.Node) *Entry {
e.Line = n.Line
e.Column = n.Column
e.Type = n.ShortTag()
return e
}
func (e *Entry) String() string {
@ -91,7 +87,7 @@ func (e *Entry) ChildNamed(name string) *Entry {
}
func (e *Entry) SetPath(parent []string, current string) {
e.Path = append(parent, current)
e.Path = append(parent, current) // nolint: gocritic
switch e.Kind {
case yaml.MappingNode, yaml.DocumentNode:
for idx := 0; idx < len(e.Content); idx += 2 {
@ -101,19 +97,19 @@ func (e *Entry) SetPath(parent []string, current string) {
}
case yaml.SequenceNode:
for idx, child := range e.Content {
child.Path = append(e.Path, fmt.Sprintf("%d", idx))
child.Path = append(e.Path, fmt.Sprintf("%d", idx)) // nolint: gocritic
}
}
}
func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
copyFromNode(e, node)
e.copyFromNode(node)
switch node.Kind {
case yaml.SequenceNode, yaml.ScalarNode:
for _, n := range node.Content {
sub := &Entry{}
copyFromNode(sub, n)
sub.copyFromNode(n)
if err := n.Decode(&sub); err != nil {
return err
}
@ -147,6 +143,10 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
return nil
}
func (e *Entry) IsScalar() bool {
return e.Kind != yaml.DocumentNode && e.Kind != yaml.MappingNode && e.Kind != yaml.SequenceNode
}
func (e *Entry) IsSecret() bool {
return e.Tag == YAMLTypeSecret
}
@ -216,8 +216,7 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
annotations := map[string]string{}
data := map[string]string{}
for i := 0; i < len(fields); i++ {
field := fields[i]
for _, field := range fields {
label := field.Label
if field.Section != nil {
if field.Section.Label == "~annotations" {
@ -234,35 +233,12 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
}
for label, valueStr := range data {
var value any
var err error
var style yaml.Style
var tag string
switch annotations[label] {
case "bool":
value, err = strconv.ParseBool(valueStr)
if err != nil {
return err
}
case "int":
value, err = strconv.ParseInt(valueStr, 10, 64)
if err != nil {
return err
}
case "float":
var err error
value, err = strconv.ParseFloat(valueStr, 64)
if err != nil {
return err
}
case "secret":
value = secretValue(value.(string))
if annotations[label] == "secret" {
style = yaml.TaggedStyle
tag = YAMLTypeSecret
default:
// either no annotation or an unknown value
value = valueStr
}
path := strings.Split(label, ".")
@ -270,7 +246,7 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
for idx, key := range path {
if idx == len(path)-1 {
container.Content = append(container.Content, &Entry{
container.Content = append(container.Content, NewEntry(key, yaml.ScalarNode), &Entry{
Path: path,
Kind: yaml.ScalarNode,
Value: valueStr,
@ -289,8 +265,12 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
kind = yaml.SequenceNode
}
child := NewEntry(key, kind)
child.Path = append(container.Path, key)
child.Path = append(container.Path, key) // nolint: gocritic
if isNumeric(key) {
container.Content = append(container.Content, child)
} else {
container.Content = append(container.Content, NewEntry(child.Name(), child.Kind), child)
}
container = child
}
}
@ -305,6 +285,7 @@ func (e *Entry) ToOP() []*op.ItemField {
if e.Kind == yaml.ScalarNode {
name := e.Path[len(e.Path)-1]
fullPath := strings.Join(e.Path, ".")
if len(e.Path) > 1 {
section = &op.ItemSection{ID: e.Path[0]}
name = strings.Join(e.Path[1:], ".")
@ -313,20 +294,20 @@ func (e *Entry) ToOP() []*op.ItemField {
fieldType := "STRING"
if e.IsSecret() {
fieldType = "CONCEALED"
} else {
}
if annotationType := e.TypeStr(); annotationType != "" {
ret = append(ret, &op.ItemField{
ID: "~annotations." + strings.Join(e.Path, "."),
ID: "~annotations." + fullPath,
Section: annotationsSection,
Label: name,
Label: fullPath,
Type: "STRING",
Value: annotationType,
})
}
}
ret = append(ret, &op.ItemField{
ID: strings.Join(e.Path, "."),
ID: fullPath,
Section: section,
Label: name,
Type: fieldType,
@ -385,3 +366,42 @@ func (e *Entry) AsMap() any {
}
return ret
}
func (e *Entry) Merge(other *Entry) error {
if e.IsScalar() && other.IsScalar() {
e.Value = other.Value
e.Tag = other.Tag
e.Kind = other.Kind
e.Type = other.Type
return nil
}
if e.Kind == yaml.MappingNode || e.Kind == yaml.DocumentNode {
for i := 0; i < len(other.Content); i += 2 {
remote := other.Content[i+1]
local := e.ChildNamed(remote.Name())
if local != nil {
if err := local.Merge(remote); err != nil {
return err
}
} else {
e.Content = append(e.Content, NewEntry(remote.Name(), remote.Kind), remote)
}
}
return nil
}
for _, remote := range other.Content {
local := other.ChildNamed(remote.Name())
if local != nil {
if err := local.Merge(remote); err != nil {
return err
}
} else {
logrus.Debugf("adding new collection value at %s", remote.Path)
local.Content = append(local.Content, remote)
}
}
return nil
}

3
pkg/config/readers.go
View File

@ -70,6 +70,9 @@ func FromOP(item *op.Item) (*Config, error) {
Tree: NewEntry("root", yaml.MappingNode),
}
if cs := checksum(item.Fields); cs != item.GetValue("password") {
logrus.Warnf("1Password item changed and checksum was not updated. Expected %s, found %s", cs, item.GetValue("password"))
}
err := cfg.Tree.FromOP(item.Fields)
return cfg, err
}

4
pkg/config/util.go
View File

@ -28,12 +28,12 @@ import (
type opDetails struct {
Vault string `yaml:"vault"`
Name string `yaml:"name"`
NameTemplate string `yaml:"nameTemplate"`
NameTemplate string `yaml:"nameTemplate"` // nolint: tagliatelle
Repo string
}
type singleModeConfig struct {
Config *opDetails `yaml:"_config,omitempty"`
Config *opDetails `yaml:"_config,omitempty"` // nolint: tagliatelle
}
func argIsYAMLFile(path string) bool {