lol, actually redact

and encode deeply nested stuff properly

cmd/fetch_test.go

@@ -79,14 +79,15 @@ string: pato
 bool: false
 secret: !!secret very secret
 nested:
-  string: quem
   int: 1
-  secret: !!secret very secret
   bool: true
   list:
     - 1
     - 2
     - 3
+  secret: !!secret very secret
+  second_secret: !!secret very secret
+  string: quem
 list:
   - one
   - two

cmd/get_test.go

@@ -4,6 +4,7 @@ package cmd_test
 
 import (
 	"bytes"
+	"encoding/json"
 	"os"
 	"path"
 	"runtime"
@@ -25,7 +26,7 @@ var testConfig = &onepassword.Item{
 	Category: "PASSWORD",
 	Sections: []*onepassword.ItemSection{
 		{ID: "~annotations", Label: "~annotations"},
-		// {ID: "nested", Label: "nested"},
+		{ID: "nested", Label: "nested"},
 		{ID: "list", Label: "list"},
 	},
 	Fields: []*onepassword.ItemField{
@@ -34,7 +35,7 @@ var testConfig = &onepassword.Item{
 			Type:    "CONCEALED",
 			Purpose: "PASSWORD",
 			Label:   "password",
-			Value:   "56615e9be5f0ce5f97d5b446faaa1d39f95a13a1ea8326ae933c3d29eb29735c",
+			Value:   "8b23de7705b79b73d9f75b120651bc162859e45a732b764362feaefc882eab5d",
 		},
 		{
 			ID:      "notesPlain",
@@ -88,13 +89,6 @@ var testConfig = &onepassword.Item{
 			Label: "secret",
 			Value: "very secret",
 		},
-		{
-			ID:      "nested.string",
-			Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
-			Type:    "STRING",
-			Label:   "string",
-			Value:   "quem",
-		},
 		{
 			ID:      "~annotations.nested.int",
 			Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
@@ -109,6 +103,62 @@ var testConfig = &onepassword.Item{
 			Label:   "int",
 			Value:   "1",
 		},
+		{
+			ID:      "~annotations.nested.bool",
+			Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
+			Type:    "STRING",
+			Label:   "nested.bool",
+			Value:   "bool",
+		},
+		{
+			ID:      "nested.bool",
+			Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
+			Type:    "STRING",
+			Label:   "bool",
+			Value:   "true",
+		},
+		{
+			ID:      "~annotations.nested.list.0",
+			Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
+			Type:    "STRING",
+			Label:   "nested.list.0",
+			Value:   "int",
+		},
+		{
+			ID:      "nested.list.0",
+			Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
+			Type:    "STRING",
+			Label:   "list.0",
+			Value:   "1",
+		},
+		{
+			ID:      "~annotations.nested.list.1",
+			Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
+			Type:    "STRING",
+			Label:   "nested.list.1",
+			Value:   "int",
+		},
+		{
+			ID:      "nested.list.1",
+			Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
+			Type:    "STRING",
+			Label:   "list.1",
+			Value:   "2",
+		},
+		{
+			ID:      "~annotations.nested.list.2",
+			Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
+			Type:    "STRING",
+			Label:   "nested.list.2",
+			Value:   "int",
+		},
+		{
+			ID:      "nested.list.2",
+			Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
+			Type:    "STRING",
+			Label:   "list.2",
+			Value:   "3",
+		},
 		{
 			ID:      "~annotations.nested.secret",
 			Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
@@ -124,18 +174,25 @@ var testConfig = &onepassword.Item{
 			Value:   "very secret",
 		},
 		{
-			ID:      "~annotations.nested.bool",
+			ID:      "~annotations.nested.second_secret",
 			Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
 			Type:    "STRING",
-			Label:   "nested.bool",
+			Label:   "nested.second_secret",
-			Value:   "bool",
+			Value:   "secret",
 		},
 		{
-			ID:      "nested.bool",
+			ID:      "nested.second_secret",
+			Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
+			Type:    "CONCEALED",
+			Label:   "second_secret",
+			Value:   "very secret",
+		},
+		{
+			ID:      "nested.string",
 			Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
 			Type:    "STRING",
-			Label:   "bool",
+			Label:   "string",
-			Value:   "true",
+			Value:   "quem",
 		},
 		{
 			ID:      "list.0",
@@ -306,6 +363,7 @@ list:
   - 1
   - 2
   - 3
+second_secret: very secret
 secret: very secret
 string: quem`
 
@@ -336,6 +394,7 @@ list:
   - 1
   - 2
   - 3
+second_secret: very secret
 secret: very secret
 string: quem`
 
@@ -376,6 +435,7 @@ nested:
     - 1
     - 2
     - 3
+  second_secret: !!secret very secret
   secret: !!secret very secret
   string: quem
 secret: !!secret very secret
@@ -402,7 +462,30 @@ func TestGetJSON(t *testing.T) {
 		t.Fatalf("could not get: %s", err)
 	}
 
-	expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"secret":"very secret","string":"quem"},"secret":"very secret","string":"pato"}`
+	expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"second_secret":"very secret","secret":"very secret","string":"quem"},"secret":"very secret","string":"pato"}`
+
+	if got := out.String(); strings.TrimSpace(got) != expected {
+		t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
+	}
+}
+
+func TestGetDeepJSON(t *testing.T) {
+	root := fromProjectRoot()
+	out := bytes.Buffer{}
+	Get.SetBindings()
+	cmd := &cobra.Command{}
+	cmd.Flags().Bool("redacted", false, "")
+	cmd.Flags().StringP("output", "o", "json", "")
+	cmd.SetOut(&out)
+	cmd.SetErr(&out)
+	Get.Cobra = cmd
+	err := Get.Run(cmd, []string{root + "/deeply-nested.test.yaml", ".", "--output", "json"})
+
+	if err != nil {
+		t.Fatalf("could not get: %s", err)
+	}
+
+	expected := `{"root":{"of":{"deeply":{"nested_list":["10.42.31.42/32"],"nested_map":"asdf"}}}}`
 
 	if got := out.String(); strings.TrimSpace(got) != expected {
 		t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
@@ -448,7 +531,7 @@ func TestGetJSONPathCollection(t *testing.T) {
 		t.Fatalf("could not get: %s", err)
 	}
 
-	expected := `{"bool":true,"int":1,"list":[1,2,3],"secret":"very secret","string":"quem"}`
+	expected := `{"bool":true,"int":1,"list":[1,2,3],"second_secret":"very secret","secret":"very secret","string":"quem"}`
 
 	if got := out.String(); strings.TrimSpace(got) != expected {
 		t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
@@ -471,7 +554,7 @@ func TestGetJSONRedacted(t *testing.T) {
 		t.Fatalf("could not get: %s", err)
 	}
 
-	expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"secret":"","string":"quem"},"secret":"","string":"pato"}`
+	expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"second_secret":"","secret":"","string":"quem"},"secret":"","string":"pato"}`
 
 	if got := out.String(); strings.TrimSpace(got) != expected {
 		t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
@@ -493,9 +576,15 @@ func TestGetJSONOP(t *testing.T) {
 		t.Fatalf("could not get: %s", err)
 	}
 
-	expected := `{"id":"","title":"some:test","vault":{"id":"example"},"category":"PASSWORD","sections":[{"id":"~annotations","label":"~annotations"},{"id":"nested","label":"nested"},{"id":"list","label":"list"}],"fields":[{"id":"password","type":"CONCEALED","purpose":"PASSWORD","label":"password","value":"cedbdf86fb15cf1237569e9b3188372d623aea9d6a707401aca656645590227c"},{"id":"notesPlain","type":"STRING","purpose":"NOTES","label":"notesPlain","value":"flushed by joao"},{"id":"~annotations.int","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"int","value":"int"},{"id":"int","type":"STRING","label":"int","value":"1"},{"id":"string","type":"STRING","label":"string","value":"pato"},{"id":"~annotations.bool","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"bool","value":"bool"},{"id":"bool","type":"STRING","label":"bool","value":"false"},{"id":"~annotations.secret","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"secret","value":"secret"},{"id":"secret","type":"CONCEALED","label":"secret","value":"very secret"},{"id":"nested.string","section":{"id":"nested"},"type":"STRING","label":"string","value":"quem"},{"id":"~annotations.nested.int","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.int","value":"int"},{"id":"nested.int","section":{"id":"nested"},"type":"STRING","label":"int","value":"1"},{"id":"~annotations.nested.secret","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.secret","value":"secret"},{"id":"nested.secret","section":{"id":"nested"},"type":"CONCEALED","label":"secret","value":"very secret"},{"id":"~annotations.nested.bool","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.bool","value":"bool"},{"id":"nested.bool","section":{"id":"nested"},"type":"STRING","label":"bool","value":"true"},{"id":"~annotations.nested.list.0","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.0","value":"int"},{"id":"nested.list.0","section":{"id":"nested"},"type":"STRING","label":"list.0","value":"1"},{"id":"~annotations.nested.list.1","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.1","value":"int"},{"id":"nested.list.1","section":{"id":"nested"},"type":"STRING","label":"list.1","value":"2"},{"id":"~annotations.nested.list.2","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.2","value":"int"},{"id":"nested.list.2","section":{"id":"nested"},"type":"STRING","label":"list.2","value":"3"},{"id":"list.0","section":{"id":"list"},"type":"STRING","label":"0","value":"one"},{"id":"list.1","section":{"id":"list"},"type":"STRING","label":"1","value":"two"},{"id":"list.2","section":{"id":"list"},"type":"STRING","label":"2","value":"three"}],"createdAt":"0001-01-01T00:00:00Z","updatedAt":"0001-01-01T00:00:00Z"}`
+	id := testConfig.ID
+	testConfig.ID = ""
+	defer func() { testConfig.ID = id }()
+	expected, err := json.Marshal(testConfig)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	if got := out.String(); strings.TrimSpace(got) != expected {
+	if got := out.String(); strings.TrimSpace(got) != strings.TrimSpace(string(expected)) {
 		t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
 	}
 }
@@ -528,6 +617,11 @@ list:
 nested:
   bool: true
   int: 1
+  list:
+    - 1
+    - 2
+    - 3
+  second_secret: !!secret very secret
   secret: !!secret very secret
   string: quem
 secret: !!secret very secret

cmd/redact.go

@@ -32,7 +32,7 @@ var Redact = &command.Command{
 				return err
 			}
 
-			if err := cfg.AsFile(path); err != nil {
+			if err := cfg.AsFile(path, config.OutputModeRedacted); err != nil {
 				return err
 			}
 		}

deeply-nested.test.yaml

@@ -0,0 +1,9 @@
+_config: !!joao
+  name: some:deeply-nested-test
+  vault: example
+root:
+  of:
+    deeply:
+      nested_map: asdf
+      nested_list:
+        - 10.42.31.42/32

pkg/config/config.go

@@ -117,15 +117,12 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er
 			if dst := entry.ChildNamed(key); dst == nil {
 				key := NewEntry(key, yaml.ScalarNode)
 				if entry.Kind == yaml.MappingNode {
-					logrus.Infof("setting new map key %v", newEntry.Path)
 					entry.Content = append(entry.Content, key, newEntry)
 				} else {
-					logrus.Infof("setting new list key %v", newEntry.Path)
 					entry.Kind = yaml.SequenceNode
 					entry.Content = append(entry.Content, newEntry)
 				}
 			} else {
-				logrus.Infof("setting %v", newEntry.Path)
 				dst.Value = newEntry.Value
 				dst.Tag = newEntry.Tag
 				dst.Style = newEntry.Style
@@ -143,7 +140,8 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er
 			kind = yaml.SequenceNode
 		}
 		sub := NewEntry(key, kind)
-		sub.Path = append(entry.Path, key) // nolint: gocritic
+		sub.Path = append([]string{}, entry.Path...)
+		sub.Path = append(sub.Path, key)
 
 		keyEntry := NewEntry(sub.Name(), yaml.ScalarNode)
 		keyEntry.Value = key

pkg/config/entry.go

@@ -92,19 +92,26 @@ func (e *Entry) ChildNamed(name string) *Entry {
 }
 
 func (e *Entry) SetPath(parent []string, current string) {
-	e.Path = append(parent, current) // nolint: gocritic
+	if current != "." {
-	switch e.Kind {
+		e.Path = append([]string{}, parent...)
-	case yaml.MappingNode, yaml.DocumentNode:
+		e.Path = append(e.Path, current)
+	}
+	if e.IsScalar() {
+		return
+	}
+
+	if e.Kind == yaml.SequenceNode {
+		for idx, child := range e.Content {
+			child.SetPath(e.Path, fmt.Sprintf("%d", idx))
+		}
+		return
+	}
+
 	for idx := 0; idx < len(e.Content); idx += 2 {
 		key := e.Content[idx]
 		child := e.Content[idx+1]
 		child.SetPath(e.Path, key.Value)
 	}
-	case yaml.SequenceNode:
-		for idx, child := range e.Content {
-			child.Path = append(e.Path, fmt.Sprintf("%d", idx)) // nolint: gocritic
-		}
-	}
 }
 
 func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
@@ -118,7 +125,6 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
 			if err := n.Decode(&sub); err != nil {
 				return err
 			}
-			sub.SetPath(e.Path, n.Value)
 			e.Content = append(e.Content, sub)
 		}
 	case yaml.DocumentNode, yaml.MappingNode:
@@ -139,7 +145,6 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
 			if valueNode.Tag == YAMLTypeMetaConfig {
 				key.Type = YAMLTypeMetaConfig
 			}
-			value.SetPath(e.Path, key.Value)
 			e.Content = append(e.Content, key, value)
 		}
 	default:
@@ -321,7 +326,8 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
 				kind = yaml.SequenceNode
 			}
 			child := NewEntry(key, kind)
-			child.Path = append(container.Path, key) // nolint: gocritic
+			child.Path = append([]string{}, container.Path...)
+			child.Path = append(child.Path, key)
 
 			keyEntry := NewEntry(child.Name(), yaml.ScalarNode)
 			keyEntry.Value = key
@@ -341,7 +347,7 @@ func (e *Entry) ToOP() []*op.ItemField {
 		name := e.Path[len(e.Path)-1]
 		fullPath := strings.Join(e.Path, ".")
 		if len(e.Path) > 1 {
-			section = &op.ItemSection{ID: e.Path[0]}
+			section = &op.ItemSection{ID: e.Path[0], Label: e.Path[0]}
 			name = strings.Join(e.Path[1:], ".")
 		}
 
@@ -371,7 +377,6 @@ func (e *Entry) ToOP() []*op.ItemField {
 	}
 
 	if e.Kind == yaml.SequenceNode {
-		ret := []*op.ItemField{}
 		for _, child := range e.Content {
 			ret = append(ret, child.ToOP()...)
 		}
@@ -396,7 +401,7 @@ func (e *Entry) Name() string {
 }
 
 func (e *Entry) AsMap() any {
-	if len(e.Content) == 0 {
+	if e.IsScalar() {
 		switch e.TypeStr() {
 		case "bool":
 			var boolVal bool
@@ -425,17 +430,15 @@ func (e *Entry) AsMap() any {
 
 	if e.Kind == yaml.SequenceNode {
 		ret := []any{}
-		for _, child := range e.Content {
+		for _, sub := range e.Content {
-			ret = append(ret, child.AsMap())
+			ret = append(ret, sub.AsMap())
 		}
 		return ret
 	}
 
 	ret := map[string]any{}
-	for idx, child := range e.Content {
+	for idx := 1; idx < len(e.Content); idx += 2 {
-		if idx%2 == 0 {
+		child := e.Content[idx]
-			continue
-		}
 		ret[child.Name()] = child.AsMap()
 	}
 	return ret

pkg/config/input.go

@@ -5,6 +5,7 @@ package config
 import (
 	"fmt"
 	"io/ioutil"
+	"path/filepath"
 	"strings"
 
 	opClient "git.rob.mx/nidito/joao/internal/op-client"
@@ -48,9 +49,13 @@ func Load(ref string, preferRemote bool) (*Config, error) {
 		return nil, fmt.Errorf("could not load %s from local as it's not a path", ref)
 	}
 
-	cfg, err := FromFile(ref)
+	path, err := filepath.Abs(ref)
 	if err != nil {
-		return nil, fmt.Errorf("could not load file %s: %w", ref, err)
+		return nil, fmt.Errorf("could not find asbolute path to file %s: %w", ref, err)
+	}
+	cfg, err := FromFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("could not load file %s: %w", path, err)
 	}
 	return cfg, nil
 }
@@ -92,6 +97,7 @@ func FromYAML(data []byte) (*Config, error) {
 	if err != nil {
 		return nil, fmt.Errorf("could not parse %w", err)
 	}
+	cfg.Tree.SetPath([]string{}, ".")
 
 	return cfg, nil
 }

pkg/config/output.go

@@ -143,8 +143,8 @@ func (cfg *Config) AsJSON(redacted bool, item bool) ([]byte, error) {
 	return bytes, nil
 }
 
-func (cfg *Config) AsFile(path string) error {
+func (cfg *Config) AsFile(path string, modes ...OutputMode) error {
-	b, err := cfg.AsYAML()
+	b, err := cfg.AsYAML(modes...)
 	if err != nil {
 		return err
 	}

pkg/config/util.go

@@ -83,7 +83,6 @@ func checksum(fields []*op.ItemField) string {
 	if err != nil {
 		panic(err)
 	}
-	// newHash := md5.New()
 	df := []string{}
 	for _, field := range fields {
 		if field.ID == "password" || field.ID == "notesPlain" || (field.Section != nil && field.Section.ID == "~annotations") {

test.yaml

@@ -8,14 +8,15 @@ string: pato
 bool: false
 secret: !!secret very secret
 nested:
-  string: quem
   int: 1
-  secret: !!secret very secret
   bool: true
   list:
     - 1
     - 2
     - 3
+  secret: !!secret very secret
+  second_secret: !!secret very secret
+  string: quem
 list:
   - one
   - two