From cfc2cdcc222dd5c629c38b82e934f63bad5dd877 Mon Sep 17 00:00:00 2001 From: Roberto Hidalgo Date: Fri, 9 Dec 2022 00:43:07 -0600 Subject: [PATCH] multi repo config --- README.md | 2 +- cmd/get.go | 4 +- cmd/set.go | 2 +- cmd/util.go | 72 ----------------- internal/command/arguments.go | 2 +- internal/command/arguments_test.go | 15 ++-- main.go | 2 + pkg/config/config.go | 96 +++++++++++++++++++++-- pkg/config/lookup.go | 25 +++++- pkg/config/util.go | 120 +++++++++++++++++++++++++++++ 10 files changed, 248 insertions(+), 92 deletions(-) delete mode 100644 cmd/util.go create mode 100644 pkg/config/util.go diff --git a/README.md b/README.md index ecc7868..84f1e77 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ smtp: ```sh # NAME can be either a filesystem path or a colon delimited item name -# for example: config/host/juazeiro.yaml or host:juazeiro +# for example: config/host/juazeiro.yaml or [op-vault-name/]host:juazeiro # DOT_DELIMITED_PATH is # for example: tls.cert, roles.0, dc diff --git a/cmd/get.go b/cmd/get.go index 019e415..1a92f45 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -119,13 +119,11 @@ looks at the filesystem or remotely, using 1password (over the CLI if available, path := cmd.Arguments[0].ToValue().(string) query := cmd.Arguments[1].ToValue().(string) - var cfg *config.Config - var err error remote := cmd.Options["remote"].ToValue().(bool) format := cmd.Options["output"].ToValue().(string) redacted := cmd.Options["redacted"].ToValue().(bool) - cfg, err = loadExisting(path, remote) + cfg, err := config.Load(path, remote) if err != nil { return err } diff --git a/cmd/set.go b/cmd/set.go index 669f78d..7475f5a 100644 --- a/cmd/set.go +++ b/cmd/set.go @@ -81,7 +81,7 @@ Will read from stdin (or ﹅--from﹅ a file) and store it at the ﹅PATH input := cmd.Options["input"].ToValue().(string) parseJSON := cmd.Options["json"].ToValue().(bool) - cfg, err = loadExisting(path, false) + cfg, err = config.Load(path, false) if err != nil { return err } diff --git a/cmd/util.go b/cmd/util.go deleted file mode 100644 index b85f249..0000000 --- a/cmd/util.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright © 2022 Roberto Hidalgo -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package cmd - -import ( - "fmt" - "io/ioutil" - "strings" - - opClient "git.rob.mx/nidito/joao/internal/op-client" - "git.rob.mx/nidito/joao/pkg/config" -) - -func argIsYAMLFile(path string) bool { - return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") -} - -func pathToName(path string) string { - comps := strings.Split(path, "config/") - return strings.ReplaceAll(strings.Replace(comps[len(comps)-1], ".yaml", "", 1), "/", ":") -} - -func nameToPath(name string) string { - return "config/" + strings.ReplaceAll(name, ":", "/") + ".yaml" -} - -func loadExisting(ref string, preferRemote bool) (*config.Config, error) { - isYaml := argIsYAMLFile(ref) - if preferRemote { - name := ref - if isYaml { - name = pathToName(ref) - } - - item, err := opClient.Get("nidito-admin", name) - if err != nil { - return nil, err - } - - return config.ConfigFromOP(item) - } - - path := ref - var name string - if !isYaml { - path = nameToPath(ref) - name = ref - } else { - name = pathToName(ref) - } - - buf, err := ioutil.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("could not read file %s", ref) - } - - if len(buf) == 0 { - buf = []byte("{}") - } - - return config.ConfigFromYAML(buf, name) -} diff --git a/internal/command/arguments.go b/internal/command/arguments.go index 05a2644..18b84c8 100644 --- a/internal/command/arguments.go +++ b/internal/command/arguments.go @@ -179,7 +179,7 @@ func (arg *Argument) ToString() string { if arg.Variadic { val := val.([]string) - return strings.Join(val, "") + return strings.Join(val, " ") } return val.(string) diff --git a/internal/command/arguments_test.go b/internal/command/arguments_test.go index 15f018c..7219e21 100644 --- a/internal/command/arguments_test.go +++ b/internal/command/arguments_test.go @@ -13,6 +13,7 @@ package command_test import ( + "reflect" "strings" "testing" @@ -70,7 +71,7 @@ func TestParse(t *testing.T) { t.Fatalf("variadic argument isn't on AllKnown map: %v", known) } - if val != "one two three" { + if !reflect.DeepEqual(val, []string{"one", "two", "three"}) { t.Fatalf("Known argument does not match. expected: %s, got %s", "one two three", val) } @@ -96,8 +97,9 @@ func TestParse(t *testing.T) { t.Fatalf("variadic argument isn't on AllKnown map: %v", known) } - if val != "defaultVariadic0 defaultVariadic1" { - t.Fatalf("variadic argument does not match. expected: %s, got %s", "defaultVariadic0 defaultVariadic1", val) + expected := []string{"defaultVariadic0", "defaultVariadic1"} + if !reflect.DeepEqual(val, expected) { + t.Fatalf("variadic argument does not match. expected: %s, got %s", expected, val) } } @@ -123,8 +125,9 @@ func TestBeforeParse(t *testing.T) { t.Fatalf("variadic argument isn't on AllKnown map: %v", known) } - if val != "defaultVariadic0 defaultVariadic1" { - t.Fatalf("variadic argument does not match. expected: %s, got %s", "defaultVariadic0 defaultVariadic1", val) + expected := []string{"defaultVariadic0", "defaultVariadic1"} + if !reflect.DeepEqual(val, expected) { + t.Fatalf("variadic argument does not match. expected: %s, got %s", expected, val) } } @@ -195,7 +198,7 @@ func TestArgumentsValidate(t *testing.T) { Args: []string{"good"}, ErrorSuffix: "could not validate argument for command test script bad-exit, ran", Command: (&Command{ - // Name: []string{"test", "script", "bad-exit"}, + Path: []string{"test", "script", "bad-exit"}, Arguments: []*Argument{ { Name: "first", diff --git a/main.go b/main.go index 34ca6cc..47edced 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,8 @@ func main() { ForceColors: runtime.ColorEnabled(), }) + logrus.SetLevel(logrus.DebugLevel) + err := registry.Execute(version) if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 8ebaf2a..1f4e592 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,6 +16,7 @@ import ( "bytes" "encoding/json" "fmt" + "io/ioutil" "sort" "strings" @@ -25,7 +26,6 @@ import ( ) type Config struct { - Path string Vault string Name string Tree *Entry @@ -85,17 +85,99 @@ func (cfg *Config) ToOP() *op.Item { return &op.Item{ Title: cfg.Name, Sections: sections, - Vault: op.ItemVault{ID: "nidito-admin"}, + Vault: op.ItemVault{ID: cfg.Vault}, Category: op.Password, Fields: fields, } } -func ConfigFromYAML(data []byte, name string) (*Config, error) { +type opDetails struct { + Vault string `yaml:"vault"` + Name string `yaml:"name"` + NameTemplate string `yaml:"nameTemplate"` +} + +type opConfig interface { + Name() string + Vault() string +} + +type inFileConfig struct { + *opDetails + *yaml.Node +} + +type virtualConfig struct { + *opDetails +} + +func (ifc *inFileConfig) MarshalYAML() (any, error) { + return ifc.Node, nil +} + +func (vc *virtualConfig) MarshalYAML() (any, error) { + return nil, nil +} + +func (ifc *inFileConfig) UnmarshalYAML(node *yaml.Node) error { + ifc.Node = node + d := &opDetails{} + + if err := node.Decode(&d); err != nil { + return err + } + ifc.opDetails = d + + return nil +} + +func (ifc *inFileConfig) Name() string { + return ifc.opDetails.Name +} + +func (ifc *inFileConfig) Vault() string { + return ifc.opDetails.Name +} + +type singleModeConfig struct { + Config *opDetails `yaml:"_config,omitempty"` +} + +type repoModeConfig struct { + Repo string + Vault string `yaml:"vault"` + NameTemplate string `yaml:"nameTemplate"` +} + +func ConfigFromFile(path string) (*Config, error) { + buf, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("could not read file %s", path) + } + + if len(buf) == 0 { + buf = []byte("{}") + } + + name, vault, err := vaultAndNameFrom(path, buf) + if err != nil { + return nil, err + } + logrus.Debugf("Found name: %s and vault: %s", name, vault) + + cfg, err := ConfigFromYAML(buf) + if err != nil { + return nil, err + } + cfg.Name = name + cfg.Vault = vault + + return cfg, nil +} + +func ConfigFromYAML(data []byte) (*Config, error) { cfg := &Config{ - Vault: "vault", - Name: name, - Tree: NewEntry("root", yaml.MappingNode), + Tree: NewEntry("root", yaml.MappingNode), } yaml.Unmarshal(data, &cfg.Tree) @@ -135,7 +217,7 @@ func scalarsIn(data map[string]yaml.Node, parents []string) ([]string, error) { } keys = append(keys, ret...) default: - logrus.Fatalf("found unknown %s at %s", leaf.Kind, key) + logrus.Fatalf("found unknown %v at %s", leaf.Kind, key) } } diff --git a/pkg/config/lookup.go b/pkg/config/lookup.go index 060cf78..3e2ff32 100644 --- a/pkg/config/lookup.go +++ b/pkg/config/lookup.go @@ -12,7 +12,13 @@ // limitations under the License. package config -import "fmt" +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) func (c *Config) Lookup(query []string) (*Entry, error) { if len(query) == 0 || len(query) == 1 && query[0] == "." { @@ -29,3 +35,20 @@ func (c *Config) Lookup(query []string) (*Entry, error) { return entry, nil } + +func findRepoConfig(from string) (*repoModeConfig, error) { + parts := strings.Split(from, "/") + for i := len(parts); i > 0; i -= 1 { + query := strings.Join(parts[0:i], "/") + if bytes, err := os.ReadFile(query + "/.joao.yaml"); err == nil { + rmc := &repoModeConfig{Repo: query} + err := yaml.Unmarshal(bytes, rmc) + if err != nil { + return nil, err + } + return rmc, nil + } + } + + return nil, nil +} diff --git a/pkg/config/util.go b/pkg/config/util.go new file mode 100644 index 0000000..91a63fb --- /dev/null +++ b/pkg/config/util.go @@ -0,0 +1,120 @@ +// Copyright © 2022 Roberto Hidalgo +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import ( + "bytes" + "fmt" + "io/ioutil" + "path/filepath" + "strings" + "text/template" + + opClient "git.rob.mx/nidito/joao/internal/op-client" + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" +) + +func argIsYAMLFile(path string) bool { + return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") +} + +// func pathToName(path string, prefix string) string { +// comps := strings.SplitN(path, prefix+"/", 2) +// return strings.ReplaceAll(strings.Replace(comps[1], ".yaml", "", 1), "/", ":") +// } + +// func nameToPath(name string) string { +// return "config/" + strings.ReplaceAll(name, ":", "/") + ".yaml" +// } + +func vaultAndNameFrom(path string, buf []byte) (string, string, error) { + smc := &singleModeConfig{} + if buf == nil { + var err error + buf, err = ioutil.ReadFile(path) + if err != nil { + return "", "", fmt.Errorf("could not read file %s", path) + } + } + + if err := yaml.Unmarshal(buf, &smc); err == nil && smc.Config != nil { + return smc.Config.Vault, smc.Config.Name, nil + } + + rmc, err := findRepoConfig(path) + if err != nil { + return "", "", err + } + + if rmc == nil { + return "", "", fmt.Errorf("could not find repo config for %s", path) + } + + if rmc.NameTemplate == "" { + rmc.NameTemplate = "{{ DirName }}:{{ FileName}}" + } + + logrus.Debugf("Found repo config at %s", rmc.Repo) + + tpl := template.Must(template.New("help").Funcs(template.FuncMap{ + "DirName": func() string { + return filepath.Base(filepath.Dir(path)) + }, + "FileName": func() string { + return strings.Split(filepath.Base(path), ".")[0] + }, + }).Parse(rmc.NameTemplate)) + + var nameBuf bytes.Buffer + err = tpl.Execute(&nameBuf, nil) + if err != nil { + return "", "", err + } + return nameBuf.String(), rmc.Vault, nil +} + +func Load(ref string, preferRemote bool) (*Config, error) { + isYaml := argIsYAMLFile(ref) + if preferRemote { + name := ref + vault := "" + + if isYaml { + var err error + name, vault, err = vaultAndNameFrom(ref, nil) + if err != nil { + return nil, err + } + } else { + parts := strings.SplitN(ref, "/", 2) + if len(parts) > 1 { + vault = parts[0] + name = parts[1] + } + } + + item, err := opClient.Get(vault, name) + if err != nil { + return nil, err + } + + return ConfigFromOP(item) + } + + if !isYaml { + return nil, fmt.Errorf("could not load %s from local as it's not a path", ref) + } + + return ConfigFromFile(ref) +}