lol, actually redact

and encode deeply nested stuff properly
This commit is contained in:
Roberto Hidalgo 2023-01-15 01:17:34 -06:00
parent c94f1cda76
commit f011825e76
10 changed files with 167 additions and 56 deletions

View File

@ -79,14 +79,15 @@ string: pato
bool: false bool: false
secret: !!secret very secret secret: !!secret very secret
nested: nested:
string: quem
int: 1 int: 1
secret: !!secret very secret
bool: true bool: true
list: list:
- 1 - 1
- 2 - 2
- 3 - 3
secret: !!secret very secret
second_secret: !!secret very secret
string: quem
list: list:
- one - one
- two - two

View File

@ -4,6 +4,7 @@ package cmd_test
import ( import (
"bytes" "bytes"
"encoding/json"
"os" "os"
"path" "path"
"runtime" "runtime"
@ -25,7 +26,7 @@ var testConfig = &onepassword.Item{
Category: "PASSWORD", Category: "PASSWORD",
Sections: []*onepassword.ItemSection{ Sections: []*onepassword.ItemSection{
{ID: "~annotations", Label: "~annotations"}, {ID: "~annotations", Label: "~annotations"},
// {ID: "nested", Label: "nested"}, {ID: "nested", Label: "nested"},
{ID: "list", Label: "list"}, {ID: "list", Label: "list"},
}, },
Fields: []*onepassword.ItemField{ Fields: []*onepassword.ItemField{
@ -34,7 +35,7 @@ var testConfig = &onepassword.Item{
Type: "CONCEALED", Type: "CONCEALED",
Purpose: "PASSWORD", Purpose: "PASSWORD",
Label: "password", Label: "password",
Value: "56615e9be5f0ce5f97d5b446faaa1d39f95a13a1ea8326ae933c3d29eb29735c", Value: "8b23de7705b79b73d9f75b120651bc162859e45a732b764362feaefc882eab5d",
}, },
{ {
ID: "notesPlain", ID: "notesPlain",
@ -88,13 +89,6 @@ var testConfig = &onepassword.Item{
Label: "secret", Label: "secret",
Value: "very secret", Value: "very secret",
}, },
{
ID: "nested.string",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "string",
Value: "quem",
},
{ {
ID: "~annotations.nested.int", ID: "~annotations.nested.int",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
@ -109,6 +103,62 @@ var testConfig = &onepassword.Item{
Label: "int", Label: "int",
Value: "1", Value: "1",
}, },
{
ID: "~annotations.nested.bool",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING",
Label: "nested.bool",
Value: "bool",
},
{
ID: "nested.bool",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "bool",
Value: "true",
},
{
ID: "~annotations.nested.list.0",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING",
Label: "nested.list.0",
Value: "int",
},
{
ID: "nested.list.0",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "list.0",
Value: "1",
},
{
ID: "~annotations.nested.list.1",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING",
Label: "nested.list.1",
Value: "int",
},
{
ID: "nested.list.1",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "list.1",
Value: "2",
},
{
ID: "~annotations.nested.list.2",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING",
Label: "nested.list.2",
Value: "int",
},
{
ID: "nested.list.2",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "list.2",
Value: "3",
},
{ {
ID: "~annotations.nested.secret", ID: "~annotations.nested.secret",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
@ -124,18 +174,25 @@ var testConfig = &onepassword.Item{
Value: "very secret", Value: "very secret",
}, },
{ {
ID: "~annotations.nested.bool", ID: "~annotations.nested.second_secret",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"}, Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING", Type: "STRING",
Label: "nested.bool", Label: "nested.second_secret",
Value: "bool", Value: "secret",
}, },
{ {
ID: "nested.bool", ID: "nested.second_secret",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "CONCEALED",
Label: "second_secret",
Value: "very secret",
},
{
ID: "nested.string",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"}, Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING", Type: "STRING",
Label: "bool", Label: "string",
Value: "true", Value: "quem",
}, },
{ {
ID: "list.0", ID: "list.0",
@ -306,6 +363,7 @@ list:
- 1 - 1
- 2 - 2
- 3 - 3
second_secret: very secret
secret: very secret secret: very secret
string: quem` string: quem`
@ -336,6 +394,7 @@ list:
- 1 - 1
- 2 - 2
- 3 - 3
second_secret: very secret
secret: very secret secret: very secret
string: quem` string: quem`
@ -376,6 +435,7 @@ nested:
- 1 - 1
- 2 - 2
- 3 - 3
second_secret: !!secret very secret
secret: !!secret very secret secret: !!secret very secret
string: quem string: quem
secret: !!secret very secret secret: !!secret very secret
@ -402,7 +462,30 @@ func TestGetJSON(t *testing.T) {
t.Fatalf("could not get: %s", err) t.Fatalf("could not get: %s", err)
} }
expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"secret":"very secret","string":"quem"},"secret":"very secret","string":"pato"}` expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"second_secret":"very secret","secret":"very secret","string":"quem"},"secret":"very secret","string":"pato"}`
if got := out.String(); strings.TrimSpace(got) != expected {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
}
}
func TestGetDeepJSON(t *testing.T) {
root := fromProjectRoot()
out := bytes.Buffer{}
Get.SetBindings()
cmd := &cobra.Command{}
cmd.Flags().Bool("redacted", false, "")
cmd.Flags().StringP("output", "o", "json", "")
cmd.SetOut(&out)
cmd.SetErr(&out)
Get.Cobra = cmd
err := Get.Run(cmd, []string{root + "/deeply-nested.test.yaml", ".", "--output", "json"})
if err != nil {
t.Fatalf("could not get: %s", err)
}
expected := `{"root":{"of":{"deeply":{"nested_list":["10.42.31.42/32"],"nested_map":"asdf"}}}}`
if got := out.String(); strings.TrimSpace(got) != expected { if got := out.String(); strings.TrimSpace(got) != expected {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got) t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
@ -448,7 +531,7 @@ func TestGetJSONPathCollection(t *testing.T) {
t.Fatalf("could not get: %s", err) t.Fatalf("could not get: %s", err)
} }
expected := `{"bool":true,"int":1,"list":[1,2,3],"secret":"very secret","string":"quem"}` expected := `{"bool":true,"int":1,"list":[1,2,3],"second_secret":"very secret","secret":"very secret","string":"quem"}`
if got := out.String(); strings.TrimSpace(got) != expected { if got := out.String(); strings.TrimSpace(got) != expected {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got) t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
@ -471,7 +554,7 @@ func TestGetJSONRedacted(t *testing.T) {
t.Fatalf("could not get: %s", err) t.Fatalf("could not get: %s", err)
} }
expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"secret":"","string":"quem"},"secret":"","string":"pato"}` expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"second_secret":"","secret":"","string":"quem"},"secret":"","string":"pato"}`
if got := out.String(); strings.TrimSpace(got) != expected { if got := out.String(); strings.TrimSpace(got) != expected {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got) t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
@ -493,9 +576,15 @@ func TestGetJSONOP(t *testing.T) {
t.Fatalf("could not get: %s", err) t.Fatalf("could not get: %s", err)
} }
expected := `{"id":"","title":"some:test","vault":{"id":"example"},"category":"PASSWORD","sections":[{"id":"~annotations","label":"~annotations"},{"id":"nested","label":"nested"},{"id":"list","label":"list"}],"fields":[{"id":"password","type":"CONCEALED","purpose":"PASSWORD","label":"password","value":"cedbdf86fb15cf1237569e9b3188372d623aea9d6a707401aca656645590227c"},{"id":"notesPlain","type":"STRING","purpose":"NOTES","label":"notesPlain","value":"flushed by joao"},{"id":"~annotations.int","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"int","value":"int"},{"id":"int","type":"STRING","label":"int","value":"1"},{"id":"string","type":"STRING","label":"string","value":"pato"},{"id":"~annotations.bool","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"bool","value":"bool"},{"id":"bool","type":"STRING","label":"bool","value":"false"},{"id":"~annotations.secret","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"secret","value":"secret"},{"id":"secret","type":"CONCEALED","label":"secret","value":"very secret"},{"id":"nested.string","section":{"id":"nested"},"type":"STRING","label":"string","value":"quem"},{"id":"~annotations.nested.int","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.int","value":"int"},{"id":"nested.int","section":{"id":"nested"},"type":"STRING","label":"int","value":"1"},{"id":"~annotations.nested.secret","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.secret","value":"secret"},{"id":"nested.secret","section":{"id":"nested"},"type":"CONCEALED","label":"secret","value":"very secret"},{"id":"~annotations.nested.bool","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.bool","value":"bool"},{"id":"nested.bool","section":{"id":"nested"},"type":"STRING","label":"bool","value":"true"},{"id":"~annotations.nested.list.0","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.0","value":"int"},{"id":"nested.list.0","section":{"id":"nested"},"type":"STRING","label":"list.0","value":"1"},{"id":"~annotations.nested.list.1","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.1","value":"int"},{"id":"nested.list.1","section":{"id":"nested"},"type":"STRING","label":"list.1","value":"2"},{"id":"~annotations.nested.list.2","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.2","value":"int"},{"id":"nested.list.2","section":{"id":"nested"},"type":"STRING","label":"list.2","value":"3"},{"id":"list.0","section":{"id":"list"},"type":"STRING","label":"0","value":"one"},{"id":"list.1","section":{"id":"list"},"type":"STRING","label":"1","value":"two"},{"id":"list.2","section":{"id":"list"},"type":"STRING","label":"2","value":"three"}],"createdAt":"0001-01-01T00:00:00Z","updatedAt":"0001-01-01T00:00:00Z"}` id := testConfig.ID
testConfig.ID = ""
defer func() { testConfig.ID = id }()
expected, err := json.Marshal(testConfig)
if err != nil {
t.Fatal(err)
}
if got := out.String(); strings.TrimSpace(got) != expected { if got := out.String(); strings.TrimSpace(got) != strings.TrimSpace(string(expected)) {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got) t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
} }
} }
@ -528,6 +617,11 @@ list:
nested: nested:
bool: true bool: true
int: 1 int: 1
list:
- 1
- 2
- 3
second_secret: !!secret very secret
secret: !!secret very secret secret: !!secret very secret
string: quem string: quem
secret: !!secret very secret secret: !!secret very secret

View File

@ -32,7 +32,7 @@ var Redact = &command.Command{
return err return err
} }
if err := cfg.AsFile(path); err != nil { if err := cfg.AsFile(path, config.OutputModeRedacted); err != nil {
return err return err
} }
} }

9
deeply-nested.test.yaml Normal file
View File

@ -0,0 +1,9 @@
_config: !!joao
name: some:deeply-nested-test
vault: example
root:
of:
deeply:
nested_map: asdf
nested_list:
- 10.42.31.42/32

View File

@ -117,15 +117,12 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er
if dst := entry.ChildNamed(key); dst == nil { if dst := entry.ChildNamed(key); dst == nil {
key := NewEntry(key, yaml.ScalarNode) key := NewEntry(key, yaml.ScalarNode)
if entry.Kind == yaml.MappingNode { if entry.Kind == yaml.MappingNode {
logrus.Infof("setting new map key %v", newEntry.Path)
entry.Content = append(entry.Content, key, newEntry) entry.Content = append(entry.Content, key, newEntry)
} else { } else {
logrus.Infof("setting new list key %v", newEntry.Path)
entry.Kind = yaml.SequenceNode entry.Kind = yaml.SequenceNode
entry.Content = append(entry.Content, newEntry) entry.Content = append(entry.Content, newEntry)
} }
} else { } else {
logrus.Infof("setting %v", newEntry.Path)
dst.Value = newEntry.Value dst.Value = newEntry.Value
dst.Tag = newEntry.Tag dst.Tag = newEntry.Tag
dst.Style = newEntry.Style dst.Style = newEntry.Style
@ -143,7 +140,8 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er
kind = yaml.SequenceNode kind = yaml.SequenceNode
} }
sub := NewEntry(key, kind) sub := NewEntry(key, kind)
sub.Path = append(entry.Path, key) // nolint: gocritic sub.Path = append([]string{}, entry.Path...)
sub.Path = append(sub.Path, key)
keyEntry := NewEntry(sub.Name(), yaml.ScalarNode) keyEntry := NewEntry(sub.Name(), yaml.ScalarNode)
keyEntry.Value = key keyEntry.Value = key

View File

@ -92,18 +92,25 @@ func (e *Entry) ChildNamed(name string) *Entry {
} }
func (e *Entry) SetPath(parent []string, current string) { func (e *Entry) SetPath(parent []string, current string) {
e.Path = append(parent, current) // nolint: gocritic if current != "." {
switch e.Kind { e.Path = append([]string{}, parent...)
case yaml.MappingNode, yaml.DocumentNode: e.Path = append(e.Path, current)
for idx := 0; idx < len(e.Content); idx += 2 { }
key := e.Content[idx] if e.IsScalar() {
child := e.Content[idx+1] return
child.SetPath(e.Path, key.Value) }
}
case yaml.SequenceNode: if e.Kind == yaml.SequenceNode {
for idx, child := range e.Content { for idx, child := range e.Content {
child.Path = append(e.Path, fmt.Sprintf("%d", idx)) // nolint: gocritic child.SetPath(e.Path, fmt.Sprintf("%d", idx))
} }
return
}
for idx := 0; idx < len(e.Content); idx += 2 {
key := e.Content[idx]
child := e.Content[idx+1]
child.SetPath(e.Path, key.Value)
} }
} }
@ -118,7 +125,6 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
if err := n.Decode(&sub); err != nil { if err := n.Decode(&sub); err != nil {
return err return err
} }
sub.SetPath(e.Path, n.Value)
e.Content = append(e.Content, sub) e.Content = append(e.Content, sub)
} }
case yaml.DocumentNode, yaml.MappingNode: case yaml.DocumentNode, yaml.MappingNode:
@ -139,7 +145,6 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
if valueNode.Tag == YAMLTypeMetaConfig { if valueNode.Tag == YAMLTypeMetaConfig {
key.Type = YAMLTypeMetaConfig key.Type = YAMLTypeMetaConfig
} }
value.SetPath(e.Path, key.Value)
e.Content = append(e.Content, key, value) e.Content = append(e.Content, key, value)
} }
default: default:
@ -321,7 +326,8 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
kind = yaml.SequenceNode kind = yaml.SequenceNode
} }
child := NewEntry(key, kind) child := NewEntry(key, kind)
child.Path = append(container.Path, key) // nolint: gocritic child.Path = append([]string{}, container.Path...)
child.Path = append(child.Path, key)
keyEntry := NewEntry(child.Name(), yaml.ScalarNode) keyEntry := NewEntry(child.Name(), yaml.ScalarNode)
keyEntry.Value = key keyEntry.Value = key
@ -341,7 +347,7 @@ func (e *Entry) ToOP() []*op.ItemField {
name := e.Path[len(e.Path)-1] name := e.Path[len(e.Path)-1]
fullPath := strings.Join(e.Path, ".") fullPath := strings.Join(e.Path, ".")
if len(e.Path) > 1 { if len(e.Path) > 1 {
section = &op.ItemSection{ID: e.Path[0]} section = &op.ItemSection{ID: e.Path[0], Label: e.Path[0]}
name = strings.Join(e.Path[1:], ".") name = strings.Join(e.Path[1:], ".")
} }
@ -371,7 +377,6 @@ func (e *Entry) ToOP() []*op.ItemField {
} }
if e.Kind == yaml.SequenceNode { if e.Kind == yaml.SequenceNode {
ret := []*op.ItemField{}
for _, child := range e.Content { for _, child := range e.Content {
ret = append(ret, child.ToOP()...) ret = append(ret, child.ToOP()...)
} }
@ -396,7 +401,7 @@ func (e *Entry) Name() string {
} }
func (e *Entry) AsMap() any { func (e *Entry) AsMap() any {
if len(e.Content) == 0 { if e.IsScalar() {
switch e.TypeStr() { switch e.TypeStr() {
case "bool": case "bool":
var boolVal bool var boolVal bool
@ -425,17 +430,15 @@ func (e *Entry) AsMap() any {
if e.Kind == yaml.SequenceNode { if e.Kind == yaml.SequenceNode {
ret := []any{} ret := []any{}
for _, child := range e.Content { for _, sub := range e.Content {
ret = append(ret, child.AsMap()) ret = append(ret, sub.AsMap())
} }
return ret return ret
} }
ret := map[string]any{} ret := map[string]any{}
for idx, child := range e.Content { for idx := 1; idx < len(e.Content); idx += 2 {
if idx%2 == 0 { child := e.Content[idx]
continue
}
ret[child.Name()] = child.AsMap() ret[child.Name()] = child.AsMap()
} }
return ret return ret

View File

@ -5,6 +5,7 @@ package config
import ( import (
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"path/filepath"
"strings" "strings"
opClient "git.rob.mx/nidito/joao/internal/op-client" opClient "git.rob.mx/nidito/joao/internal/op-client"
@ -48,9 +49,13 @@ func Load(ref string, preferRemote bool) (*Config, error) {
return nil, fmt.Errorf("could not load %s from local as it's not a path", ref) return nil, fmt.Errorf("could not load %s from local as it's not a path", ref)
} }
cfg, err := FromFile(ref) path, err := filepath.Abs(ref)
if err != nil { if err != nil {
return nil, fmt.Errorf("could not load file %s: %w", ref, err) return nil, fmt.Errorf("could not find asbolute path to file %s: %w", ref, err)
}
cfg, err := FromFile(path)
if err != nil {
return nil, fmt.Errorf("could not load file %s: %w", path, err)
} }
return cfg, nil return cfg, nil
} }
@ -92,6 +97,7 @@ func FromYAML(data []byte) (*Config, error) {
if err != nil { if err != nil {
return nil, fmt.Errorf("could not parse %w", err) return nil, fmt.Errorf("could not parse %w", err)
} }
cfg.Tree.SetPath([]string{}, ".")
return cfg, nil return cfg, nil
} }

View File

@ -143,8 +143,8 @@ func (cfg *Config) AsJSON(redacted bool, item bool) ([]byte, error) {
return bytes, nil return bytes, nil
} }
func (cfg *Config) AsFile(path string) error { func (cfg *Config) AsFile(path string, modes ...OutputMode) error {
b, err := cfg.AsYAML() b, err := cfg.AsYAML(modes...)
if err != nil { if err != nil {
return err return err
} }

View File

@ -83,7 +83,6 @@ func checksum(fields []*op.ItemField) string {
if err != nil { if err != nil {
panic(err) panic(err)
} }
// newHash := md5.New()
df := []string{} df := []string{}
for _, field := range fields { for _, field := range fields {
if field.ID == "password" || field.ID == "notesPlain" || (field.Section != nil && field.Section.ID == "~annotations") { if field.ID == "password" || field.ID == "notesPlain" || (field.Section != nil && field.Section.ID == "~annotations") {

View File

@ -8,14 +8,15 @@ string: pato
bool: false bool: false
secret: !!secret very secret secret: !!secret very secret
nested: nested:
string: quem
int: 1 int: 1
secret: !!secret very secret
bool: true bool: true
list: list:
- 1 - 1
- 2 - 2
- 3 - 3
secret: !!secret very secret
second_secret: !!secret very secret
string: quem
list: list:
- one - one
- two - two