lol, actually redact

and encode deeply nested stuff properly
This commit is contained in:
Roberto Hidalgo 2023-01-15 01:17:34 -06:00
parent c94f1cda76
commit f011825e76
10 changed files with 167 additions and 56 deletions

View File

@ -79,14 +79,15 @@ string: pato
bool: false
secret: !!secret very secret
nested:
string: quem
int: 1
secret: !!secret very secret
bool: true
list:
- 1
- 2
- 3
secret: !!secret very secret
second_secret: !!secret very secret
string: quem
list:
- one
- two

View File

@ -4,6 +4,7 @@ package cmd_test
import (
"bytes"
"encoding/json"
"os"
"path"
"runtime"
@ -25,7 +26,7 @@ var testConfig = &onepassword.Item{
Category: "PASSWORD",
Sections: []*onepassword.ItemSection{
{ID: "~annotations", Label: "~annotations"},
// {ID: "nested", Label: "nested"},
{ID: "nested", Label: "nested"},
{ID: "list", Label: "list"},
},
Fields: []*onepassword.ItemField{
@ -34,7 +35,7 @@ var testConfig = &onepassword.Item{
Type: "CONCEALED",
Purpose: "PASSWORD",
Label: "password",
Value: "56615e9be5f0ce5f97d5b446faaa1d39f95a13a1ea8326ae933c3d29eb29735c",
Value: "8b23de7705b79b73d9f75b120651bc162859e45a732b764362feaefc882eab5d",
},
{
ID: "notesPlain",
@ -88,13 +89,6 @@ var testConfig = &onepassword.Item{
Label: "secret",
Value: "very secret",
},
{
ID: "nested.string",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "string",
Value: "quem",
},
{
ID: "~annotations.nested.int",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
@ -109,6 +103,62 @@ var testConfig = &onepassword.Item{
Label: "int",
Value: "1",
},
{
ID: "~annotations.nested.bool",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING",
Label: "nested.bool",
Value: "bool",
},
{
ID: "nested.bool",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "bool",
Value: "true",
},
{
ID: "~annotations.nested.list.0",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING",
Label: "nested.list.0",
Value: "int",
},
{
ID: "nested.list.0",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "list.0",
Value: "1",
},
{
ID: "~annotations.nested.list.1",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING",
Label: "nested.list.1",
Value: "int",
},
{
ID: "nested.list.1",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "list.1",
Value: "2",
},
{
ID: "~annotations.nested.list.2",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING",
Label: "nested.list.2",
Value: "int",
},
{
ID: "nested.list.2",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "list.2",
Value: "3",
},
{
ID: "~annotations.nested.secret",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
@ -124,18 +174,25 @@ var testConfig = &onepassword.Item{
Value: "very secret",
},
{
ID: "~annotations.nested.bool",
ID: "~annotations.nested.second_secret",
Section: &onepassword.ItemSection{ID: "~annotations", Label: "~annotations"},
Type: "STRING",
Label: "nested.bool",
Value: "bool",
Label: "nested.second_secret",
Value: "secret",
},
{
ID: "nested.bool",
ID: "nested.second_secret",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "CONCEALED",
Label: "second_secret",
Value: "very secret",
},
{
ID: "nested.string",
Section: &onepassword.ItemSection{ID: "nested", Label: "nested"},
Type: "STRING",
Label: "bool",
Value: "true",
Label: "string",
Value: "quem",
},
{
ID: "list.0",
@ -306,6 +363,7 @@ list:
- 1
- 2
- 3
second_secret: very secret
secret: very secret
string: quem`
@ -336,6 +394,7 @@ list:
- 1
- 2
- 3
second_secret: very secret
secret: very secret
string: quem`
@ -376,6 +435,7 @@ nested:
- 1
- 2
- 3
second_secret: !!secret very secret
secret: !!secret very secret
string: quem
secret: !!secret very secret
@ -402,7 +462,30 @@ func TestGetJSON(t *testing.T) {
t.Fatalf("could not get: %s", err)
}
expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"secret":"very secret","string":"quem"},"secret":"very secret","string":"pato"}`
expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"second_secret":"very secret","secret":"very secret","string":"quem"},"secret":"very secret","string":"pato"}`
if got := out.String(); strings.TrimSpace(got) != expected {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
}
}
func TestGetDeepJSON(t *testing.T) {
root := fromProjectRoot()
out := bytes.Buffer{}
Get.SetBindings()
cmd := &cobra.Command{}
cmd.Flags().Bool("redacted", false, "")
cmd.Flags().StringP("output", "o", "json", "")
cmd.SetOut(&out)
cmd.SetErr(&out)
Get.Cobra = cmd
err := Get.Run(cmd, []string{root + "/deeply-nested.test.yaml", ".", "--output", "json"})
if err != nil {
t.Fatalf("could not get: %s", err)
}
expected := `{"root":{"of":{"deeply":{"nested_list":["10.42.31.42/32"],"nested_map":"asdf"}}}}`
if got := out.String(); strings.TrimSpace(got) != expected {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
@ -448,7 +531,7 @@ func TestGetJSONPathCollection(t *testing.T) {
t.Fatalf("could not get: %s", err)
}
expected := `{"bool":true,"int":1,"list":[1,2,3],"secret":"very secret","string":"quem"}`
expected := `{"bool":true,"int":1,"list":[1,2,3],"second_secret":"very secret","secret":"very secret","string":"quem"}`
if got := out.String(); strings.TrimSpace(got) != expected {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
@ -471,7 +554,7 @@ func TestGetJSONRedacted(t *testing.T) {
t.Fatalf("could not get: %s", err)
}
expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"secret":"","string":"quem"},"secret":"","string":"pato"}`
expected := `{"bool":false,"int":1,"list":["one","two","three"],"nested":{"bool":true,"int":1,"list":[1,2,3],"second_secret":"","secret":"","string":"quem"},"secret":"","string":"pato"}`
if got := out.String(); strings.TrimSpace(got) != expected {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
@ -493,9 +576,15 @@ func TestGetJSONOP(t *testing.T) {
t.Fatalf("could not get: %s", err)
}
expected := `{"id":"","title":"some:test","vault":{"id":"example"},"category":"PASSWORD","sections":[{"id":"~annotations","label":"~annotations"},{"id":"nested","label":"nested"},{"id":"list","label":"list"}],"fields":[{"id":"password","type":"CONCEALED","purpose":"PASSWORD","label":"password","value":"cedbdf86fb15cf1237569e9b3188372d623aea9d6a707401aca656645590227c"},{"id":"notesPlain","type":"STRING","purpose":"NOTES","label":"notesPlain","value":"flushed by joao"},{"id":"~annotations.int","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"int","value":"int"},{"id":"int","type":"STRING","label":"int","value":"1"},{"id":"string","type":"STRING","label":"string","value":"pato"},{"id":"~annotations.bool","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"bool","value":"bool"},{"id":"bool","type":"STRING","label":"bool","value":"false"},{"id":"~annotations.secret","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"secret","value":"secret"},{"id":"secret","type":"CONCEALED","label":"secret","value":"very secret"},{"id":"nested.string","section":{"id":"nested"},"type":"STRING","label":"string","value":"quem"},{"id":"~annotations.nested.int","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.int","value":"int"},{"id":"nested.int","section":{"id":"nested"},"type":"STRING","label":"int","value":"1"},{"id":"~annotations.nested.secret","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.secret","value":"secret"},{"id":"nested.secret","section":{"id":"nested"},"type":"CONCEALED","label":"secret","value":"very secret"},{"id":"~annotations.nested.bool","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.bool","value":"bool"},{"id":"nested.bool","section":{"id":"nested"},"type":"STRING","label":"bool","value":"true"},{"id":"~annotations.nested.list.0","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.0","value":"int"},{"id":"nested.list.0","section":{"id":"nested"},"type":"STRING","label":"list.0","value":"1"},{"id":"~annotations.nested.list.1","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.1","value":"int"},{"id":"nested.list.1","section":{"id":"nested"},"type":"STRING","label":"list.1","value":"2"},{"id":"~annotations.nested.list.2","section":{"id":"~annotations","label":"~annotations"},"type":"STRING","label":"nested.list.2","value":"int"},{"id":"nested.list.2","section":{"id":"nested"},"type":"STRING","label":"list.2","value":"3"},{"id":"list.0","section":{"id":"list"},"type":"STRING","label":"0","value":"one"},{"id":"list.1","section":{"id":"list"},"type":"STRING","label":"1","value":"two"},{"id":"list.2","section":{"id":"list"},"type":"STRING","label":"2","value":"three"}],"createdAt":"0001-01-01T00:00:00Z","updatedAt":"0001-01-01T00:00:00Z"}`
id := testConfig.ID
testConfig.ID = ""
defer func() { testConfig.ID = id }()
expected, err := json.Marshal(testConfig)
if err != nil {
t.Fatal(err)
}
if got := out.String(); strings.TrimSpace(got) != expected {
if got := out.String(); strings.TrimSpace(got) != strings.TrimSpace(string(expected)) {
t.Fatalf("did not get expected output:\nwanted: %s\ngot: %s", expected, got)
}
}
@ -528,6 +617,11 @@ list:
nested:
bool: true
int: 1
list:
- 1
- 2
- 3
second_secret: !!secret very secret
secret: !!secret very secret
string: quem
secret: !!secret very secret

View File

@ -32,7 +32,7 @@ var Redact = &command.Command{
return err
}
if err := cfg.AsFile(path); err != nil {
if err := cfg.AsFile(path, config.OutputModeRedacted); err != nil {
return err
}
}

9
deeply-nested.test.yaml Normal file
View File

@ -0,0 +1,9 @@
_config: !!joao
name: some:deeply-nested-test
vault: example
root:
of:
deeply:
nested_map: asdf
nested_list:
- 10.42.31.42/32

View File

@ -117,15 +117,12 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er
if dst := entry.ChildNamed(key); dst == nil {
key := NewEntry(key, yaml.ScalarNode)
if entry.Kind == yaml.MappingNode {
logrus.Infof("setting new map key %v", newEntry.Path)
entry.Content = append(entry.Content, key, newEntry)
} else {
logrus.Infof("setting new list key %v", newEntry.Path)
entry.Kind = yaml.SequenceNode
entry.Content = append(entry.Content, newEntry)
}
} else {
logrus.Infof("setting %v", newEntry.Path)
dst.Value = newEntry.Value
dst.Tag = newEntry.Tag
dst.Style = newEntry.Style
@ -143,7 +140,8 @@ func (cfg *Config) Set(path []string, data []byte, isSecret, parseEntry bool) er
kind = yaml.SequenceNode
}
sub := NewEntry(key, kind)
sub.Path = append(entry.Path, key) // nolint: gocritic
sub.Path = append([]string{}, entry.Path...)
sub.Path = append(sub.Path, key)
keyEntry := NewEntry(sub.Name(), yaml.ScalarNode)
keyEntry.Value = key

View File

@ -92,18 +92,25 @@ func (e *Entry) ChildNamed(name string) *Entry {
}
func (e *Entry) SetPath(parent []string, current string) {
e.Path = append(parent, current) // nolint: gocritic
switch e.Kind {
case yaml.MappingNode, yaml.DocumentNode:
for idx := 0; idx < len(e.Content); idx += 2 {
key := e.Content[idx]
child := e.Content[idx+1]
child.SetPath(e.Path, key.Value)
}
case yaml.SequenceNode:
if current != "." {
e.Path = append([]string{}, parent...)
e.Path = append(e.Path, current)
}
if e.IsScalar() {
return
}
if e.Kind == yaml.SequenceNode {
for idx, child := range e.Content {
child.Path = append(e.Path, fmt.Sprintf("%d", idx)) // nolint: gocritic
child.SetPath(e.Path, fmt.Sprintf("%d", idx))
}
return
}
for idx := 0; idx < len(e.Content); idx += 2 {
key := e.Content[idx]
child := e.Content[idx+1]
child.SetPath(e.Path, key.Value)
}
}
@ -118,7 +125,6 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
if err := n.Decode(&sub); err != nil {
return err
}
sub.SetPath(e.Path, n.Value)
e.Content = append(e.Content, sub)
}
case yaml.DocumentNode, yaml.MappingNode:
@ -139,7 +145,6 @@ func (e *Entry) UnmarshalYAML(node *yaml.Node) error {
if valueNode.Tag == YAMLTypeMetaConfig {
key.Type = YAMLTypeMetaConfig
}
value.SetPath(e.Path, key.Value)
e.Content = append(e.Content, key, value)
}
default:
@ -321,7 +326,8 @@ func (e *Entry) FromOP(fields []*op.ItemField) error {
kind = yaml.SequenceNode
}
child := NewEntry(key, kind)
child.Path = append(container.Path, key) // nolint: gocritic
child.Path = append([]string{}, container.Path...)
child.Path = append(child.Path, key)
keyEntry := NewEntry(child.Name(), yaml.ScalarNode)
keyEntry.Value = key
@ -341,7 +347,7 @@ func (e *Entry) ToOP() []*op.ItemField {
name := e.Path[len(e.Path)-1]
fullPath := strings.Join(e.Path, ".")
if len(e.Path) > 1 {
section = &op.ItemSection{ID: e.Path[0]}
section = &op.ItemSection{ID: e.Path[0], Label: e.Path[0]}
name = strings.Join(e.Path[1:], ".")
}
@ -371,7 +377,6 @@ func (e *Entry) ToOP() []*op.ItemField {
}
if e.Kind == yaml.SequenceNode {
ret := []*op.ItemField{}
for _, child := range e.Content {
ret = append(ret, child.ToOP()...)
}
@ -396,7 +401,7 @@ func (e *Entry) Name() string {
}
func (e *Entry) AsMap() any {
if len(e.Content) == 0 {
if e.IsScalar() {
switch e.TypeStr() {
case "bool":
var boolVal bool
@ -425,17 +430,15 @@ func (e *Entry) AsMap() any {
if e.Kind == yaml.SequenceNode {
ret := []any{}
for _, child := range e.Content {
ret = append(ret, child.AsMap())
for _, sub := range e.Content {
ret = append(ret, sub.AsMap())
}
return ret
}
ret := map[string]any{}
for idx, child := range e.Content {
if idx%2 == 0 {
continue
}
for idx := 1; idx < len(e.Content); idx += 2 {
child := e.Content[idx]
ret[child.Name()] = child.AsMap()
}
return ret

View File

@ -5,6 +5,7 @@ package config
import (
"fmt"
"io/ioutil"
"path/filepath"
"strings"
opClient "git.rob.mx/nidito/joao/internal/op-client"
@ -48,9 +49,13 @@ func Load(ref string, preferRemote bool) (*Config, error) {
return nil, fmt.Errorf("could not load %s from local as it's not a path", ref)
}
cfg, err := FromFile(ref)
path, err := filepath.Abs(ref)
if err != nil {
return nil, fmt.Errorf("could not load file %s: %w", ref, err)
return nil, fmt.Errorf("could not find asbolute path to file %s: %w", ref, err)
}
cfg, err := FromFile(path)
if err != nil {
return nil, fmt.Errorf("could not load file %s: %w", path, err)
}
return cfg, nil
}
@ -92,6 +97,7 @@ func FromYAML(data []byte) (*Config, error) {
if err != nil {
return nil, fmt.Errorf("could not parse %w", err)
}
cfg.Tree.SetPath([]string{}, ".")
return cfg, nil
}

View File

@ -143,8 +143,8 @@ func (cfg *Config) AsJSON(redacted bool, item bool) ([]byte, error) {
return bytes, nil
}
func (cfg *Config) AsFile(path string) error {
b, err := cfg.AsYAML()
func (cfg *Config) AsFile(path string, modes ...OutputMode) error {
b, err := cfg.AsYAML(modes...)
if err != nil {
return err
}

View File

@ -83,7 +83,6 @@ func checksum(fields []*op.ItemField) string {
if err != nil {
panic(err)
}
// newHash := md5.New()
df := []string{}
for _, field := range fields {
if field.ID == "password" || field.ID == "notesPlain" || (field.Section != nil && field.Section.ID == "~annotations") {

View File

@ -8,14 +8,15 @@ string: pato
bool: false
secret: !!secret very secret
nested:
string: quem
int: 1
secret: !!secret very secret
bool: true
list:
- 1
- 2
- 3
secret: !!secret very secret
second_secret: !!secret very secret
string: quem
list:
- one
- two