From f011825e765c186f7a8e5e1bb74d69f222dee35d Mon Sep 17 00:00:00 2001 From: Roberto Hidalgo Date: Sun, 15 Jan 2023 01:17:34 -0600 Subject: [PATCH] lol, actually redact and encode deeply nested stuff properly --- cmd/fetch_test.go | 5 +- cmd/get_test.go | 134 ++++++++++++++++++++++++++++++++++------ cmd/redact.go | 2 +- deeply-nested.test.yaml | 9 +++ pkg/config/config.go | 6 +- pkg/config/entry.go | 47 +++++++------- pkg/config/input.go | 10 ++- pkg/config/output.go | 4 +- pkg/config/util.go | 1 - test.yaml | 5 +- 10 files changed, 167 insertions(+), 56 deletions(-) create mode 100644 deeply-nested.test.yaml diff --git a/cmd/fetch_test.go b/cmd/fetch_test.go index 17ccb0d..2090586 100644 --- a/cmd/fetch_test.go +++ b/cmd/fetch_test.go @@ -79,14 +79,15 @@ string: pato bool: false secret: !!secret very secret nested: - string: quem int: 1 - secret: !!secret very secret bool: true list: - 1 - 2 - 3 + secret: !!secret very secret + second_secret: !!secret very secret + string: quem list: - one - two diff --git a/cmd/get_test.go b/cmd/get_test.go index be18953..3b28aa3 100644 --- a/cmd/get_test.go +++ b/cmd/get_test.go @@ -4,6 +4,7 @@ package cmd_test import ( "bytes" + "encoding/json" "os" "path" "runtime" @@ -25,7 +26,7 @@ var testConfig = &onepassword.Item{ Category: "PASSWORD", Sections: []*onepassword.ItemSection{ {ID: "~annotations", Label: "~annotations"}, - // {ID: "nested", Label: "nested"}, + {ID: "nested", Label: "nested"}, {ID: "list", Label: "list"}, }, Fields: []*onepassword.ItemField{ @@ -34,7 +35,7 @@ var testConfig = &onepassword.Item{ Type: "CONCEALED", Purpose: "PASSWORD", Label: "password", - Value: "56615e9be5f0ce5f97d5b446faaa1d39f95a13a1ea8326ae933c3d29eb29735c", + Value: "8b23de7705b79b73d9f75b120651bc162859e45a732b764362feaefc882eab5d", }, { ID: "notesPlain", @@ -88,13 +89,6 @@ var testConfig = &onepassword.Item{ Label: "secret", Value: "very secret", }, - { - ID: "nested.string", - Section: &onepassword.ItemSection{ID: "nested", Label: "nested"}, - Type: "STRING", - Label: "string", - Value: "quem", - }, { ID: "~annotations.nested.int", Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, @@ -109,6 +103,62 @@ var testConfig = &onepassword.Item{ Label: "int", Value: "1", }, + { + ID: "~annotations.nested.bool", + Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, + Type: "STRING", + Label: "nested.bool", + Value: "bool", + }, + { + ID: "nested.bool", + Section: &onepassword.ItemSection{ID: "nested", Label: "nested"}, + Type: "STRING", + Label: "bool", + Value: "true", + }, + { + ID: "~annotations.nested.list.0", + Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, + Type: "STRING", + Label: "nested.list.0", + Value: "int", + }, + { + ID: "nested.list.0", + Section: &onepassword.ItemSection{ID: "nested", Label: "nested"}, + Type: "STRING", + Label: "list.0", + Value: "1", + }, + { + ID: "~annotations.nested.list.1", + Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, + Type: "STRING", + Label: "nested.list.1", + Value: "int", + }, + { + ID: "nested.list.1", + Section: &onepassword.ItemSection{ID: "nested", Label: "nested"}, + Type: "STRING", + Label: "list.1", + Value: "2", + }, + { + ID: "~annotations.nested.list.2", + Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, + Type: "STRING", + Label: "nested.list.2", + Value: "int", + }, + { + ID: "nested.list.2", + Section: &onepassword.ItemSection{ID: "nested", Label: "nested"}, + Type: "STRING", + Label: "list.2", + Value: "3", + }, { ID: "~annotations.nested.secret", Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, @@ -124,18 +174,25 @@ var testConfig = &onepassword.Item{ Value: "very secret", }, { - ID: "~annotations.nested.bool", + ID: "~annotations.nested.second_secret", Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, Type: "STRING", - Label: "nested.bool", - Value: "bool", + Label: "nested.second_secret", + Value: "secret", }, { - ID: "nested.bool", + ID: "nested.second_secret", + Section: &onepassword.ItemSection{ID: "nested", Label: "nested"}, + Type: "CONCEALED", + Label: "second_secret", + Value: "very secret", + }, + { + ID: "nested.string", Section: &onepassword.ItemSection{ID: "nested", Label: "nested"}, Type: "STRING", - Label: "bool", - Value: "true", + Label: "string", + Value: "quem", }, { ID: "list.0", @@ -306,6 +363,7 @@ list: - 1 - 2 - 3 +second_secret: very secret secret: very secret string: quem` @@ -336,6 +394,7 @@ list: - 1 - 2 - 3 +second_secret: very secret secret: very secret string: quem` @@ -376,6 +435,7 @@ nested: - 1 - 2 - 3 + second_secret: !!secret very secret secret: !!secret very secret string: quem secret: !!secret very secret @@ -402,7 +462,30 @@ func TestGetJSON(t *testing.T) { t.Fatalf("could not get: %s", err) } - expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"secret":"very secret","string":"quem"},"secret":"very secret","string":"pato"}` + expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"second_secret":"very secret","secret":"very secret","string":"quem"},"secret":"very secret","string":"pato"}` + + if got := out.String(); strings.TrimSpace(got) != expected { + t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got) + } +} + +func TestGetDeepJSON(t *testing.T) { + root := fromProjectRoot() + out := bytes.Buffer{} + Get.SetBindings() + cmd := &cobra.Command{} + cmd.Flags().Bool("redacted", false, "") + cmd.Flags().StringP("output", "o", "json", "") + cmd.SetOut(&out) + cmd.SetErr(&out) + Get.Cobra = cmd + err := Get.Run(cmd, []string{root + "/deeply-nested.test.yaml", ".", "--output", "json"}) + + if err != nil { + t.Fatalf("could not get: %s", err) + } + + expected := `{"root":{"of":{"deeply":{"nested_list":["10.42.31.42/32"],"nested_map":"asdf"}}}}` if got := out.String(); strings.TrimSpace(got) != expected { t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got) @@ -448,7 +531,7 @@ func TestGetJSONPathCollection(t *testing.T) { t.Fatalf("could not get: %s", err) } - expected := `{"bool":true,"int":1,"list":[1,2,3],"secret":"very secret","string":"quem"}` + expected := `{"bool":true,"int":1,"list":[1,2,3],"second_secret":"very secret","secret":"very secret","string":"quem"}` if got := out.String(); strings.TrimSpace(got) != expected { t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got) @@ -471,7 +554,7 @@ func TestGetJSONRedacted(t *testing.T) { t.Fatalf("could not get: %s", err) } - expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"secret":"","string":"quem"},"secret":"","string":"pato"}` + expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"second_secret":"","secret":"","string":"quem"},"secret":"","string":"pato"}` if got := out.String(); strings.TrimSpace(got) != expected { t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got) @@ -493,9 +576,15 @@ func TestGetJSONOP(t *testing.T) { t.Fatalf("could not get: %s", err) } - expected := `{"id":"","title":"some:test","vault":{"id":"example"},"category":"PASSWORD","sections":[{"id":"~annotations","label":"~annotations"},{"id":"nested","label":"nested"},{"id":"list","label":"list"}],"fields":[{"id":"password","type":"CONCEALED","purpose":"PASSWORD","label":"password","value":"cedbdf86fb15cf1237569e9b3188372d623aea9d6a707401aca656645590227c"},{"id":"notesPlain","type":"STRING","purpose":"NOTES","label":"notesPlain","value":"flushed by joao"},{"id":"~annotations.int","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"int","value":"int"},{"id":"int","type":"STRING","label":"int","value":"1"},{"id":"string","type":"STRING","label":"string","value":"pato"},{"id":"~annotations.bool","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"bool","value":"bool"},{"id":"bool","type":"STRING","label":"bool","value":"false"},{"id":"~annotations.secret","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"secret","value":"secret"},{"id":"secret","type":"CONCEALED","label":"secret","value":"very secret"},{"id":"nested.string","section":{"id":"nested"},"type":"STRING","label":"string","value":"quem"},{"id":"~annotations.nested.int","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.int","value":"int"},{"id":"nested.int","section":{"id":"nested"},"type":"STRING","label":"int","value":"1"},{"id":"~annotations.nested.secret","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.secret","value":"secret"},{"id":"nested.secret","section":{"id":"nested"},"type":"CONCEALED","label":"secret","value":"very secret"},{"id":"~annotations.nested.bool","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.bool","value":"bool"},{"id":"nested.bool","section":{"id":"nested"},"type":"STRING","label":"bool","value":"true"},{"id":"~annotations.nested.list.0","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.0","value":"int"},{"id":"nested.list.0","section":{"id":"nested"},"type":"STRING","label":"list.0","value":"1"},{"id":"~annotations.nested.list.1","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.1","value":"int"},{"id":"nested.list.1","section":{"id":"nested"},"type":"STRING","label":"list.1","value":"2"},{"id":"~annotations.nested.list.2","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.2","value":"int"},{"id":"nested.list.2","section":{"id":"nested"},"type":"STRING","label":"list.2","value":"3"},{"id":"list.0","section":{"id":"list"},"type":"STRING","label":"0","value":"one"},{"id":"list.1","section":{"id":"list"},"type":"STRING","label":"1","value":"two"},{"id":"list.2","section":{"id":"list"},"type":"STRING","label":"2","value":"three"}],"createdAt":"0001-01-01T00:00:00Z","updatedAt":"0001-01-01T00:00:00Z"}` + id := testConfig.ID + testConfig.ID = "" + defer func() { testConfig.ID = id }() + expected, err := json.Marshal(testConfig) + if err != nil { + t.Fatal(err) + } - if got := out.String(); strings.TrimSpace(got) != expected { + if got := out.String(); strings.TrimSpace(got) != strings.TrimSpace(string(expected)) { t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got) } } @@ -528,6 +617,11 @@ list: nested: bool: true int: 1 + list: + - 1 + - 2 + - 3 + second_secret: !!secret very secret secret: !!secret very secret string: quem secret: !!secret very secret diff --git a/cmd/redact.go b/cmd/redact.go index b12d6bb..e1b5d19 100644 --- a/cmd/redact.go +++ b/cmd/redact.go @@ -32,7 +32,7 @@ var Redact = &command.Command{ return err } - if err := cfg.AsFile(path); err != nil { + if err := cfg.AsFile(path, config.OutputModeRedacted); err != nil { return err } } diff --git a/deeply-nested.test.yaml b/deeply-nested.test.yaml new file mode 100644 index 0000000..0ff9df8 --- /dev/null +++ b/deeply-nested.test.yaml @@ -0,0 +1,9 @@ +_config: !!joao + name: some:deeply-nested-test + vault: example +root: + of: + deeply: + nested_map: asdf + nested_list: + - 10.42.31.42/32 diff --git a/pkg/config/config.go b/pkg/config/config.go index 01df03d..6c695e7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -117,15 +117,12 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er if dst := entry.ChildNamed(key); dst == nil { key := NewEntry(key, yaml.ScalarNode) if entry.Kind == yaml.MappingNode { - logrus.Infof("setting new map key %v", newEntry.Path) entry.Content = append(entry.Content, key, newEntry) } else { - logrus.Infof("setting new list key %v", newEntry.Path) entry.Kind = yaml.SequenceNode entry.Content = append(entry.Content, newEntry) } } else { - logrus.Infof("setting %v", newEntry.Path) dst.Value = newEntry.Value dst.Tag = newEntry.Tag dst.Style = newEntry.Style @@ -143,7 +140,8 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er kind = yaml.SequenceNode } sub := NewEntry(key, kind) - sub.Path = append(entry.Path, key) // nolint: gocritic + sub.Path = append([]string{}, entry.Path...) + sub.Path = append(sub.Path, key) keyEntry := NewEntry(sub.Name(), yaml.ScalarNode) keyEntry.Value = key diff --git a/pkg/config/entry.go b/pkg/config/entry.go index cef74cf..769682f 100644 --- a/pkg/config/entry.go +++ b/pkg/config/entry.go @@ -92,18 +92,25 @@ func (e *Entry) ChildNamed(name string) *Entry { } func (e *Entry) SetPath(parent []string, current string) { - e.Path = append(parent, current) // nolint: gocritic - switch e.Kind { - case yaml.MappingNode, yaml.DocumentNode: - for idx := 0; idx < len(e.Content); idx += 2 { - key := e.Content[idx] - child := e.Content[idx+1] - child.SetPath(e.Path, key.Value) - } - case yaml.SequenceNode: + if current != "." { + e.Path = append([]string{}, parent...) + e.Path = append(e.Path, current) + } + if e.IsScalar() { + return + } + + if e.Kind == yaml.SequenceNode { for idx, child := range e.Content { - child.Path = append(e.Path, fmt.Sprintf("%d", idx)) // nolint: gocritic + child.SetPath(e.Path, fmt.Sprintf("%d", idx)) } + return + } + + for idx := 0; idx < len(e.Content); idx += 2 { + key := e.Content[idx] + child := e.Content[idx+1] + child.SetPath(e.Path, key.Value) } } @@ -118,7 +125,6 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error { if err := n.Decode(&sub); err != nil { return err } - sub.SetPath(e.Path, n.Value) e.Content = append(e.Content, sub) } case yaml.DocumentNode, yaml.MappingNode: @@ -139,7 +145,6 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error { if valueNode.Tag == YAMLTypeMetaConfig { key.Type = YAMLTypeMetaConfig } - value.SetPath(e.Path, key.Value) e.Content = append(e.Content, key, value) } default: @@ -321,7 +326,8 @@ func (e *Entry) FromOP(fields []*op.ItemField) error { kind = yaml.SequenceNode } child := NewEntry(key, kind) - child.Path = append(container.Path, key) // nolint: gocritic + child.Path = append([]string{}, container.Path...) + child.Path = append(child.Path, key) keyEntry := NewEntry(child.Name(), yaml.ScalarNode) keyEntry.Value = key @@ -341,7 +347,7 @@ func (e *Entry) ToOP() []*op.ItemField { name := e.Path[len(e.Path)-1] fullPath := strings.Join(e.Path, ".") if len(e.Path) > 1 { - section = &op.ItemSection{ID: e.Path[0]} + section = &op.ItemSection{ID: e.Path[0], Label: e.Path[0]} name = strings.Join(e.Path[1:], ".") } @@ -371,7 +377,6 @@ func (e *Entry) ToOP() []*op.ItemField { } if e.Kind == yaml.SequenceNode { - ret := []*op.ItemField{} for _, child := range e.Content { ret = append(ret, child.ToOP()...) } @@ -396,7 +401,7 @@ func (e *Entry) Name() string { } func (e *Entry) AsMap() any { - if len(e.Content) == 0 { + if e.IsScalar() { switch e.TypeStr() { case "bool": var boolVal bool @@ -425,17 +430,15 @@ func (e *Entry) AsMap() any { if e.Kind == yaml.SequenceNode { ret := []any{} - for _, child := range e.Content { - ret = append(ret, child.AsMap()) + for _, sub := range e.Content { + ret = append(ret, sub.AsMap()) } return ret } ret := map[string]any{} - for idx, child := range e.Content { - if idx%2 == 0 { - continue - } + for idx := 1; idx < len(e.Content); idx += 2 { + child := e.Content[idx] ret[child.Name()] = child.AsMap() } return ret diff --git a/pkg/config/input.go b/pkg/config/input.go index 696321a..3631195 100644 --- a/pkg/config/input.go +++ b/pkg/config/input.go @@ -5,6 +5,7 @@ package config import ( "fmt" "io/ioutil" + "path/filepath" "strings" opClient "git.rob.mx/nidito/joao/internal/op-client" @@ -48,9 +49,13 @@ func Load(ref string, preferRemote bool) (*Config, error) { return nil, fmt.Errorf("could not load %s from local as it's not a path", ref) } - cfg, err := FromFile(ref) + path, err := filepath.Abs(ref) if err != nil { - return nil, fmt.Errorf("could not load file %s: %w", ref, err) + return nil, fmt.Errorf("could not find asbolute path to file %s: %w", ref, err) + } + cfg, err := FromFile(path) + if err != nil { + return nil, fmt.Errorf("could not load file %s: %w", path, err) } return cfg, nil } @@ -92,6 +97,7 @@ func FromYAML(data []byte) (*Config, error) { if err != nil { return nil, fmt.Errorf("could not parse %w", err) } + cfg.Tree.SetPath([]string{}, ".") return cfg, nil } diff --git a/pkg/config/output.go b/pkg/config/output.go index f512cbc..b640c63 100644 --- a/pkg/config/output.go +++ b/pkg/config/output.go @@ -143,8 +143,8 @@ func (cfg *Config) AsJSON(redacted bool, item bool) ([]byte, error) { return bytes, nil } -func (cfg *Config) AsFile(path string) error { - b, err := cfg.AsYAML() +func (cfg *Config) AsFile(path string, modes ...OutputMode) error { + b, err := cfg.AsYAML(modes...) if err != nil { return err } diff --git a/pkg/config/util.go b/pkg/config/util.go index a7ee8c7..44bfbaa 100644 --- a/pkg/config/util.go +++ b/pkg/config/util.go @@ -83,7 +83,6 @@ func checksum(fields []*op.ItemField) string { if err != nil { panic(err) } - // newHash := md5.New() df := []string{} for _, field := range fields { if field.ID == "password" || field.ID == "notesPlain" || (field.Section != nil && field.Section.ID == "~annotations") { diff --git a/test.yaml b/test.yaml index 8e964d8..d8560f7 100644 --- a/test.yaml +++ b/test.yaml @@ -8,14 +8,15 @@ string: pato bool: false secret: !!secret very secret nested: - string: quem int: 1 - secret: !!secret very secret bool: true list: - 1 - 2 - 3 + secret: !!secret very secret + second_secret: !!secret very secret + string: quem list: - one - two