add git filters, reorganize readme

README.md

@@ -2,6 +2,36 @@
 
 A very wip configuration manager. Keeps config entries encoded as YAML in the filesystem, backs it up to 1Password, and syncs scrubbed copies to git. robots consume entries via 1Password Connect + Vault.
 
+## Usage
+
+```sh
+# PATH refers to a filesystem path
+# examples: config/host/juazeiro.yaml, service/gitea/config.joao.yaml
+
+# QUERY refers to a sequence of keys delimited by dots
+# examples: tls.cert, roles.0, dc, . (literal dot meaning the whole thing)
+
+# there's better help available within each command, try:
+joao get --help
+
+# get a single value/tree from a single item/file
+joao get [--output|-o=(raw|json|yaml|op)] [--remote] PATH [QUERY]
+# set/update a single value in a single item/file
+joao set [--secret] [--flush] [--input=/path/to/input|<<<"value"] PATH QUERY
+# sync local changes upstream
+joao flush [--dry-run] [--redact] PATH
+# sync remote secrets to filesystem
+joao fetch [--dry-run] PATH
+# check for differences between local and remote items
+joao diff [--cache] PATH
+
+# show information on the git integration
+joao git-filter
+
+# show information on the vault integration
+joao vault server --help
+```
+
 ## Why
 
 So I wanted to operate on my configuration mess...
@@ -94,37 +124,85 @@ smtp:
 
 ```
 
-## Usage
+## git integration
+
+In order to store configuration files within a git repository while keeping secrets off remote copies, `joao` provides git filters.
+
+To install them, **every collaborator** would need to run:
 
 ```sh
-# NAME can be either a filesystem path or a colon delimited item name
+# setup filters in your local copy of the repo:
-# for example: config/host/juazeiro.yaml or [op-vault-name/]host:juazeiro
+# this runs when you check in a file (i.e. about to commit a config file)
+# it will flush secrets to 1password before removing secrets from the file on disk
+git config filter.joao.clean "joao git-filter clean --flush %f"
+# this step runs after checkout (i.e. pulling changes)
+# it simply outputs the file as-is on disk
+git config filter.joao.smudge cat
+# let's enforce these filters
+git config filter.joao.required true
 
-# DOT_DELIMITED_PATH is
+# optionally, configure a diff filter to show changes as would be commited to git
-# for example: tls.cert, roles.0, dc
+# this does not modify the original file on disk
-
+git config diff.joao.textconv "joao git-filter diff"
-# get a single value/tree from a single item/file
-joao get NAME [--output|-o=(raw|json|yaml|op)] [--remote] [jq expr]
-# set/update a single value in a single item/file
-joao set NAME DOT_DELIMITED_PATH [--secret] [--flush] [--input=/path/to/input|<<<"value"]
-# sync local changes upstream
-joao flush NAME [--dry-run] [--redact]
-# sync remote secrets to filesystem
-joao fetch NAME [--dry-run]
-# check for differences between local and remote items
-joao diff PATH [--cache]
-# print the repo config root
-# tbd
-# initialize a new joao repo
-# joao repo init [PATH]
-# list the item names within prefix
-# joao repo list [PREFIX]
-# joao repo root
-# joao repo status
-# joao repo filter clean FILE
-# joao repo filter diff PATH OLD_FILE OLD_SHA OLD_MODE NEW_FILE NEW_SHA NEW_MODE
-# joao repo filter smudge FILE
-
-# get instructions to run as a vault plugin:
-joao vault server --help
 ```
+
+Then, **only once**, we need to specify which files to apply the filters and diff commands to:
+
+```sh
+# adds diff and filter attributes for config files ending with .joao.yaml
+echo '**/*.joao.yaml filter=joao diff=joao' >> .gitattributes
+# finally, commit and push these attributes
+git add .gitattributes
+git commit -m "installing joao attributes"
+git push origin main
+```
+
+See:
+  - https://git-scm.com/docs/gitattributes#_filter
+  - https://git-scm.com/docs/gitattributes#_diff
+
+## vault integration
+
+`joao` can run as a plugin to Hashicorp Vault, and make whole configuration entries available—secrets and all—through the Vault API.
+
+To install, download `joao` to the machine running `vault` at the `plugin_directory`, as specified by vault's config. The installed `joao` executable needs to be executable for the user running vault only.
+
+### Configuration
+```sh
+export VAULT_PLUGIN_DIR=/var/lib/vault/plugins
+chmod 700 "$VAULT_PLUGIN_DIR/joao"
+export PLUGIN_SHA="$(openssl dgst -sha256 -hex "$VAULT_PLUGIN_DIR/joao" | awk '{print $2}')"
+export VERSION="$($VAULT_PLUGIN_DIR/joao --version)"
+
+# register
+vault plugin register -sha256="$PLUGIN_SHA" -command=joao -args="vault,server" -version="$VERSION" secret joao
+
+# configure, add `vault` to set a default vault for querying
+vault write config/1password "host=$OP_CONNECT_HOST" "token=$OP_CONNECT_TOKEN" # vault=my-default-vault
+
+if !vault plugin list secret | grep -c -m1 '^joao ' >/dev/null; then
+  # first time, let's enable the secrets backend
+  vault secrets enable --path=config joao
+else
+  # updating from a previous version
+  vault secrets tune -plugin-version="$VERSION" config/
+  vault plugin reload -plugin joao
+fi
+```
+
+### Vault API
+
+```sh
+# VAULT is optional if configured with a default `vault`. See above
+
+# vault read config/tree/[VAULT/]ITEM
+vault read config/tree/service:api
+vault read config/tree/prod/service:api
+
+# vault list config/trees/[VAULT/]
+vault list config/trees
+vault list config/trees/prod
+```
+
+See:
+  - https://developer.hashicorp.com/vault/docs/plugins

cmd/git-filters.go

@@ -0,0 +1,145 @@
+// Copyright © 2022 Roberto Hidalgo <joao@un.rob.mx>
+// SPDX-License-Identifier: Apache-2.0
+package cmd
+
+import (
+	"os"
+
+	"git.rob.mx/nidito/chinampa/pkg/command"
+	"git.rob.mx/nidito/joao/pkg/config"
+)
+
+var GitFilters = []*command.Command{
+	FilterDiff,
+	FilterClean,
+	FilterGroup,
+}
+
+func redactedData(cmd *command.Command) error {
+	path := cmd.Arguments[0].ToValue().(string)
+
+	flush := false
+	if opt, ok := cmd.Options["flush"]; ok {
+		flush = opt.ToValue().(bool)
+	}
+
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		return err
+	}
+
+	cfg, err := config.FromYAML(contents)
+	if err != nil {
+		return err
+	}
+
+	if flush {
+		name, vault, err := config.VaultAndNameFrom(path, contents)
+		if err != nil {
+			return err
+		}
+
+		cfg.Name = name
+		cfg.Vault = vault
+	}
+
+	res, err := cfg.AsYAML(config.OutputModeRedacted)
+	if err != nil {
+		return err
+	}
+
+	_, err = cmd.Cobra.OutOrStdout().Write(res)
+	return err
+}
+
+var FilterGroup = &command.Command{
+	Path:    []string{"git-filter"},
+	Summary: "Subcommands used by `git` as filters",
+	Description: `In order to store configuration files within a git repository while keeping secrets off remote copies, ﹅joao﹅ provides git filters.
+
+To install them, **every collaborator** would need to run:
+
+﹅﹅﹅sh
+# setup filters in your local copy of the repo:
+# this runs when you check in a file (i.e. about to commit a config file)
+# it will flush secrets to 1password before removing secrets from the file on disk
+git config filter.joao.clean "joao git-filter clean --flush %f"
+# this step runs after checkout (i.e. pulling changes)
+# it simply outputs the file as-is on disk
+git config filter.joao.smudge cat
+# let's enforce these filters
+git config filter.joao.required true
+
+# optionally, configure a diff filter to show changes as would be commited to git
+# this does not modify the original file on disk
+git config diff.joao.textconv "joao git-filter diff"
+﹅﹅﹅
+
+Then, **only once**, we need to specify which files to apply the filters and diff commands to:
+
+﹅﹅﹅sh
+# adds diff and filter attributes for config files ending with .joao.yaml
+echo '**/*.joao.yaml filter=joao diff=joao' >> .gitattributes
+# finally, commit and push these attributes
+git add .gitattributes
+git commit -m "installing joao attributes"
+git push origin main
+﹅﹅﹅
+
+See:
+  - https://git-scm.com/docs/gitattributes#_filter
+  - https://git-scm.com/docs/gitattributes#_diff`,
+	Arguments: command.Arguments{},
+	Options:   command.Options{},
+	Action: func(cmd *command.Command) error {
+		data, err := cmd.ShowHelp(command.Root.Options, os.Args)
+		if err != nil {
+			return err
+		}
+		_, err = cmd.Cobra.OutOrStderr().Write(data)
+		return err
+	},
+}
+
+var FilterDiff = &command.Command{
+	Path:        []string{"git-filter", "diff"},
+	Summary:     "a filter for git to call during `git diff`",
+	Description: `see ﹅joao git-filter﹅ for instructions to install this filter`,
+	Arguments: command.Arguments{
+		{
+			Name:        "path",
+			Description: "The git staged path to read from",
+			Required:    true,
+			Values: &command.ValueSource{
+				Files: &[]string{"yaml", "yml"},
+			},
+		},
+	},
+	Options: command.Options{},
+	Action:  redactedData,
+}
+
+var FilterClean = &command.Command{
+	Path:    []string{"git-filter", "clean"},
+	Summary: "a filter for git to call when a file is checked in",
+	Description: `see ﹅joao git-filter﹅ for instructions to install this filter
+
+Use ﹅--flush﹅ to save changes to 1password before redacting file.`,
+	Arguments: command.Arguments{
+		{
+			Name:        "path",
+			Description: "The git staged path to read from",
+			Required:    true,
+			Values: &command.ValueSource{
+				Files: &[]string{"yaml", "yml"},
+			},
+		},
+	},
+	Options: command.Options{
+		"flush": {
+			Description: "Save to 1Password after before redacting",
+			Type:        "bool",
+		},
+	},
+	Action: redactedData,
+}

cmd/list.go

@@ -1,3 +0,0 @@
-// Copyright © 2022 Roberto Hidalgo <joao@un.rob.mx>
-// SPDX-License-Identifier: Apache-2.0
-package cmd

cmd/vault-plugin.go

@@ -12,7 +12,7 @@ import (
 var Plugin = &command.Command{
 	Path:    []string{"vault", "server"},
 	Summary: "Starts a vault-joao-plugin server",
-	Description: `Runs ﹅joao﹅ as a vault plugin. See https://developer.hashicorp.com/vault/docs/plugins
+	Description: `Runs ﹅joao﹅ as a vault plugin.
 
 You'll need to install ﹅joao﹅ in the machine running ﹅vault﹅ to ﹅plugin_directory﹅ as specified by vault's config. The installed ﹅joao﹅ executable needs to be executable for the user running vault only.
 
@@ -52,6 +52,9 @@ vault read config/tree/prod/service:api
 vault list config/trees
 vault list config/trees/prod
 ﹅﹅﹅
+
+See:
+  - https://developer.hashicorp.com/vault/docs/plugins
 `,
 	Options: command.Options{
 		"ca-cert": {

go.mod

@@ -5,7 +5,7 @@ go 1.18
 // replace git.rob.mx/nidito/chinampa => /Users/roberto/src/chinampa
 
 require (
-	git.rob.mx/nidito/chinampa v0.0.0-20230102065449-d9b257e145ce
+	git.rob.mx/nidito/chinampa v0.0.0-20230111054043-db21a53b2d20
 	github.com/1Password/connect-sdk-go v1.5.0
 	github.com/alessio/shellescape v1.4.1
 	github.com/hashicorp/go-hclog v1.4.0
@@ -86,8 +86,8 @@ require (
 	golang.org/x/term v0.4.0 // indirect
 	golang.org/x/text v0.6.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6 // indirect
+	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
-	google.golang.org/grpc v1.51.0 // indirect
+	google.golang.org/grpc v1.52.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 )

go.sum

@@ -1,6 +1,6 @@
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-git.rob.mx/nidito/chinampa v0.0.0-20230102065449-d9b257e145ce h1:fKG3wUdPgsviY2mE79vhXL4CalNdvhkL6vDAtdyVt0I=
+git.rob.mx/nidito/chinampa v0.0.0-20230111054043-db21a53b2d20 h1:hHzbCNAFx9wdi3NyliugN6dTEX8dGsuup6wk2+8TmnI=
-git.rob.mx/nidito/chinampa v0.0.0-20230102065449-d9b257e145ce/go.mod h1:obhWsLkUIlKJyhfa7uunrSs2O44JBqsegSAtAvY2LRM=
+git.rob.mx/nidito/chinampa v0.0.0-20230111054043-db21a53b2d20/go.mod h1:obhWsLkUIlKJyhfa7uunrSs2O44JBqsegSAtAvY2LRM=
 github.com/1Password/connect-sdk-go v1.5.0 h1:F0WJcLSzGg3iXEDY49/ULdszYKsQLGTzn+2cyYXqiyk=
 github.com/1Password/connect-sdk-go v1.5.0/go.mod h1:TdynFeyvaRoackENbJ8RfJokH+WAowAu1MLmUbdMq6s=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -83,7 +83,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -351,16 +351,15 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
 gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
 gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
-google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6 h1:uUn6GsgKK2eCI0bWeRMgRCcqDaQXYDuB+5tXA5Xeg/8=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
-google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
-google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
+google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk=
-google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
+google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=

main.go

@@ -24,12 +24,15 @@ func main() {
 		logrus.Debug("Debugging enabled")
 	}
 
-	chinampa.Register(cmd.Get)
+	chinampa.Register(
-	chinampa.Register(cmd.Set)
+		cmd.Get,
-	chinampa.Register(cmd.Diff)
+		cmd.Set,
-	chinampa.Register(cmd.Fetch)
+		cmd.Diff,
-	chinampa.Register(cmd.Flush)
+		cmd.Fetch,
-	chinampa.Register(cmd.Plugin)
+		cmd.Flush,
+		cmd.Plugin,
+	)
+	chinampa.Register(cmd.GitFilters...)
 
 	if err := chinampa.Execute(chinampa.Config{
 		Name:    "joao",

pkg/config/config.go

@@ -193,6 +193,12 @@ func (cfg *Config) DiffRemote(path string, stdout io.Writer, stderr io.Writer) e
 	diff.Stderr = stderr
 
 	if err := diff.Run(); err != nil {
+		if _, ok := err.(*exec.ExitError); ok {
+			if diff.ProcessState.ExitCode() == 1 {
+				return nil
+			}
+
+		}
 		return fmt.Errorf("diff could not run: %w", err)
 	}
 

pkg/config/input.go

@@ -24,7 +24,7 @@ func Load(ref string, preferRemote bool) (*Config, error) {
 
 		if argIsYAMLFile(ref) {
 			var err error
-			name, vault, err = vaultAndNameFrom(ref, nil)
+			name, vault, err = VaultAndNameFrom(ref, nil)
 			if err != nil {
 				return nil, err
 			}
@@ -66,7 +66,7 @@ func FromFile(path string) (*Config, error) {
 		buf = []byte("{}")
 	}
 
-	name, vault, err := vaultAndNameFrom(path, buf)
+	name, vault, err := VaultAndNameFrom(path, buf)
 	if err != nil {
 		return nil, err
 	}

pkg/config/util.go

@@ -32,7 +32,7 @@ func argIsYAMLFile(path string) bool {
 	return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
 }
 
-func vaultAndNameFrom(path string, buf []byte) (name string, vault string, err error) {
+func VaultAndNameFrom(path string, buf []byte) (name string, vault string, err error) {
 	smc := &singleModeConfig{}
 	if buf == nil {
 		var err error