From 7bc47f6a9cd2a56af98c748915e15c28984afa8c Mon Sep 17 00:00:00 2001 From: Roberto Hidalgo Date: Wed, 11 Jan 2023 00:57:58 -0600 Subject: [PATCH] add git filters, reorganize readme --- README.md | 138 +++++++++++++++++++++++++++++++--------- cmd/git-filters.go | 145 +++++++++++++++++++++++++++++++++++++++++++ cmd/list.go | 3 - cmd/vault-plugin.go | 5 +- go.mod | 6 +- go.sum | 15 +++-- main.go | 15 +++-- pkg/config/config.go | 6 ++ pkg/config/input.go | 4 +- pkg/config/util.go | 2 +- 10 files changed, 285 insertions(+), 54 deletions(-) create mode 100644 cmd/git-filters.go delete mode 100644 cmd/list.go diff --git a/README.md b/README.md index c44e61e..c6faa33 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,36 @@ A very wip configuration manager. Keeps config entries encoded as YAML in the filesystem, backs it up to 1Password, and syncs scrubbed copies to git. robots consume entries via 1Password Connect + Vault. +## Usage + +```sh +# PATH refers to a filesystem path +# examples: config/host/juazeiro.yaml, service/gitea/config.joao.yaml + +# QUERY refers to a sequence of keys delimited by dots +# examples: tls.cert, roles.0, dc, . (literal dot meaning the whole thing) + +# there's better help available within each command, try: +joao get --help + +# get a single value/tree from a single item/file +joao get [--output|-o=(raw|json|yaml|op)] [--remote] PATH [QUERY] +# set/update a single value in a single item/file +joao set [--secret] [--flush] [--input=/path/to/input|<<<"value"] PATH QUERY +# sync local changes upstream +joao flush [--dry-run] [--redact] PATH +# sync remote secrets to filesystem +joao fetch [--dry-run] PATH +# check for differences between local and remote items +joao diff [--cache] PATH + +# show information on the git integration +joao git-filter + +# show information on the vault integration +joao vault server --help +``` + ## Why So I wanted to operate on my configuration mess... @@ -94,37 +124,85 @@ smtp: ``` -## Usage +## git integration + +In order to store configuration files within a git repository while keeping secrets off remote copies, `joao` provides git filters. + +To install them, **every collaborator** would need to run: ```sh -# NAME can be either a filesystem path or a colon delimited item name -# for example: config/host/juazeiro.yaml or [op-vault-name/]host:juazeiro +# setup filters in your local copy of the repo: +# this runs when you check in a file (i.e. about to commit a config file) +# it will flush secrets to 1password before removing secrets from the file on disk +git config filter.joao.clean "joao git-filter clean --flush %f" +# this step runs after checkout (i.e. pulling changes) +# it simply outputs the file as-is on disk +git config filter.joao.smudge cat +# let's enforce these filters +git config filter.joao.required true -# DOT_DELIMITED_PATH is -# for example: tls.cert, roles.0, dc - -# get a single value/tree from a single item/file -joao get NAME [--output|-o=(raw|json|yaml|op)] [--remote] [jq expr] -# set/update a single value in a single item/file -joao set NAME DOT_DELIMITED_PATH [--secret] [--flush] [--input=/path/to/input|<<<"value"] -# sync local changes upstream -joao flush NAME [--dry-run] [--redact] -# sync remote secrets to filesystem -joao fetch NAME [--dry-run] -# check for differences between local and remote items -joao diff PATH [--cache] -# print the repo config root -# tbd -# initialize a new joao repo -# joao repo init [PATH] -# list the item names within prefix -# joao repo list [PREFIX] -# joao repo root -# joao repo status -# joao repo filter clean FILE -# joao repo filter diff PATH OLD_FILE OLD_SHA OLD_MODE NEW_FILE NEW_SHA NEW_MODE -# joao repo filter smudge FILE - -# get instructions to run as a vault plugin: -joao vault server --help +# optionally, configure a diff filter to show changes as would be commited to git +# this does not modify the original file on disk +git config diff.joao.textconv "joao git-filter diff" ``` + +Then, **only once**, we need to specify which files to apply the filters and diff commands to: + +```sh +# adds diff and filter attributes for config files ending with .joao.yaml +echo '**/*.joao.yaml filter=joao diff=joao' >> .gitattributes +# finally, commit and push these attributes +git add .gitattributes +git commit -m "installing joao attributes" +git push origin main +``` + +See: + - https://git-scm.com/docs/gitattributes#_filter + - https://git-scm.com/docs/gitattributes#_diff + +## vault integration + +`joao` can run as a plugin to Hashicorp Vault, and make whole configuration entries available—secrets and all—through the Vault API. + +To install, download `joao` to the machine running `vault` at the `plugin_directory`, as specified by vault's config. The installed `joao` executable needs to be executable for the user running vault only. + +### Configuration +```sh +export VAULT_PLUGIN_DIR=/var/lib/vault/plugins +chmod 700 "$VAULT_PLUGIN_DIR/joao" +export PLUGIN_SHA="$(openssl dgst -sha256 -hex "$VAULT_PLUGIN_DIR/joao" | awk '{print $2}')" +export VERSION="$($VAULT_PLUGIN_DIR/joao --version)" + +# register +vault plugin register -sha256="$PLUGIN_SHA" -command=joao -args="vault,server" -version="$VERSION" secret joao + +# configure, add `vault` to set a default vault for querying +vault write config/1password "host=$OP_CONNECT_HOST" "token=$OP_CONNECT_TOKEN" # vault=my-default-vault + +if !vault plugin list secret | grep -c -m1 '^joao ' >/dev/null; then + # first time, let's enable the secrets backend + vault secrets enable --path=config joao +else + # updating from a previous version + vault secrets tune -plugin-version="$VERSION" config/ + vault plugin reload -plugin joao +fi +``` + +### Vault API + +```sh +# VAULT is optional if configured with a default `vault`. See above + +# vault read config/tree/[VAULT/]ITEM +vault read config/tree/service:api +vault read config/tree/prod/service:api + +# vault list config/trees/[VAULT/] +vault list config/trees +vault list config/trees/prod +``` + +See: + - https://developer.hashicorp.com/vault/docs/plugins diff --git a/cmd/git-filters.go b/cmd/git-filters.go new file mode 100644 index 0000000..bdb9870 --- /dev/null +++ b/cmd/git-filters.go @@ -0,0 +1,145 @@ +// Copyright © 2022 Roberto Hidalgo +// SPDX-License-Identifier: Apache-2.0 +package cmd + +import ( + "os" + + "git.rob.mx/nidito/chinampa/pkg/command" + "git.rob.mx/nidito/joao/pkg/config" +) + +var GitFilters = []*command.Command{ + FilterDiff, + FilterClean, + FilterGroup, +} + +func redactedData(cmd *command.Command) error { + path := cmd.Arguments[0].ToValue().(string) + + flush := false + if opt, ok := cmd.Options["flush"]; ok { + flush = opt.ToValue().(bool) + } + + contents, err := os.ReadFile(path) + if err != nil { + return err + } + + cfg, err := config.FromYAML(contents) + if err != nil { + return err + } + + if flush { + name, vault, err := config.VaultAndNameFrom(path, contents) + if err != nil { + return err + } + + cfg.Name = name + cfg.Vault = vault + } + + res, err := cfg.AsYAML(config.OutputModeRedacted) + if err != nil { + return err + } + + _, err = cmd.Cobra.OutOrStdout().Write(res) + return err +} + +var FilterGroup = &command.Command{ + Path: []string{"git-filter"}, + Summary: "Subcommands used by `git` as filters", + Description: `In order to store configuration files within a git repository while keeping secrets off remote copies, ﹅joao﹅ provides git filters. + +To install them, **every collaborator** would need to run: + +﹅﹅﹅sh +# setup filters in your local copy of the repo: +# this runs when you check in a file (i.e. about to commit a config file) +# it will flush secrets to 1password before removing secrets from the file on disk +git config filter.joao.clean "joao git-filter clean --flush %f" +# this step runs after checkout (i.e. pulling changes) +# it simply outputs the file as-is on disk +git config filter.joao.smudge cat +# let's enforce these filters +git config filter.joao.required true + +# optionally, configure a diff filter to show changes as would be commited to git +# this does not modify the original file on disk +git config diff.joao.textconv "joao git-filter diff" +﹅﹅﹅ + +Then, **only once**, we need to specify which files to apply the filters and diff commands to: + +﹅﹅﹅sh +# adds diff and filter attributes for config files ending with .joao.yaml +echo '**/*.joao.yaml filter=joao diff=joao' >> .gitattributes +# finally, commit and push these attributes +git add .gitattributes +git commit -m "installing joao attributes" +git push origin main +﹅﹅﹅ + +See: + - https://git-scm.com/docs/gitattributes#_filter + - https://git-scm.com/docs/gitattributes#_diff`, + Arguments: command.Arguments{}, + Options: command.Options{}, + Action: func(cmd *command.Command) error { + data, err := cmd.ShowHelp(command.Root.Options, os.Args) + if err != nil { + return err + } + _, err = cmd.Cobra.OutOrStderr().Write(data) + return err + }, +} + +var FilterDiff = &command.Command{ + Path: []string{"git-filter", "diff"}, + Summary: "a filter for git to call during `git diff`", + Description: `see ﹅joao git-filter﹅ for instructions to install this filter`, + Arguments: command.Arguments{ + { + Name: "path", + Description: "The git staged path to read from", + Required: true, + Values: &command.ValueSource{ + Files: &[]string{"yaml", "yml"}, + }, + }, + }, + Options: command.Options{}, + Action: redactedData, +} + +var FilterClean = &command.Command{ + Path: []string{"git-filter", "clean"}, + Summary: "a filter for git to call when a file is checked in", + Description: `see ﹅joao git-filter﹅ for instructions to install this filter + +Use ﹅--flush﹅ to save changes to 1password before redacting file.`, + Arguments: command.Arguments{ + { + Name: "path", + Description: "The git staged path to read from", + Required: true, + Values: &command.ValueSource{ + Files: &[]string{"yaml", "yml"}, + }, + }, + }, + Options: command.Options{ + "flush": { + Description: "Save to 1Password after before redacting", + Type: "bool", + }, + }, + Action: redactedData, +} diff --git a/cmd/list.go b/cmd/list.go deleted file mode 100644 index 8da5b24..0000000 --- a/cmd/list.go +++ /dev/null @@ -1,3 +0,0 @@ -// Copyright © 2022 Roberto Hidalgo -// SPDX-License-Identifier: Apache-2.0 -package cmd diff --git a/cmd/vault-plugin.go b/cmd/vault-plugin.go index daabc6f..4356f24 100644 --- a/cmd/vault-plugin.go +++ b/cmd/vault-plugin.go @@ -12,7 +12,7 @@ import ( var Plugin = &command.Command{ Path: []string{"vault", "server"}, Summary: "Starts a vault-joao-plugin server", - Description: `Runs ﹅joao﹅ as a vault plugin. See https://developer.hashicorp.com/vault/docs/plugins + Description: `Runs ﹅joao﹅ as a vault plugin. You'll need to install ﹅joao﹅ in the machine running ﹅vault﹅ to ﹅plugin_directory﹅ as specified by vault's config. The installed ﹅joao﹅ executable needs to be executable for the user running vault only. @@ -52,6 +52,9 @@ vault read config/tree/prod/service:api vault list config/trees vault list config/trees/prod ﹅﹅﹅ + +See: + - https://developer.hashicorp.com/vault/docs/plugins `, Options: command.Options{ "ca-cert": { diff --git a/go.mod b/go.mod index 64630f2..fd291cd 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 // replace git.rob.mx/nidito/chinampa => /Users/roberto/src/chinampa require ( - git.rob.mx/nidito/chinampa v0.0.0-20230102065449-d9b257e145ce + git.rob.mx/nidito/chinampa v0.0.0-20230111054043-db21a53b2d20 github.com/1Password/connect-sdk-go v1.5.0 github.com/alessio/shellescape v1.4.1 github.com/hashicorp/go-hclog v1.4.0 @@ -86,8 +86,8 @@ require ( golang.org/x/term v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6 // indirect - google.golang.org/grpc v1.51.0 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/grpc v1.52.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect ) diff --git a/go.sum b/go.sum index d294938..cd574d4 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.rob.mx/nidito/chinampa v0.0.0-20230102065449-d9b257e145ce h1:fKG3wUdPgsviY2mE79vhXL4CalNdvhkL6vDAtdyVt0I= -git.rob.mx/nidito/chinampa v0.0.0-20230102065449-d9b257e145ce/go.mod h1:obhWsLkUIlKJyhfa7uunrSs2O44JBqsegSAtAvY2LRM= +git.rob.mx/nidito/chinampa v0.0.0-20230111054043-db21a53b2d20 h1:hHzbCNAFx9wdi3NyliugN6dTEX8dGsuup6wk2+8TmnI= +git.rob.mx/nidito/chinampa v0.0.0-20230111054043-db21a53b2d20/go.mod h1:obhWsLkUIlKJyhfa7uunrSs2O44JBqsegSAtAvY2LRM= github.com/1Password/connect-sdk-go v1.5.0 h1:F0WJcLSzGg3iXEDY49/ULdszYKsQLGTzn+2cyYXqiyk= github.com/1Password/connect-sdk-go v1.5.0/go.mod h1:TdynFeyvaRoackENbJ8RfJokH+WAowAu1MLmUbdMq6s= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -83,7 +83,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -351,16 +351,15 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6 h1:uUn6GsgKK2eCI0bWeRMgRCcqDaQXYDuB+5tXA5Xeg/8= -google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= +google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= diff --git a/main.go b/main.go index 3067523..6f1f184 100644 --- a/main.go +++ b/main.go @@ -24,12 +24,15 @@ func main() { logrus.Debug("Debugging enabled") } - chinampa.Register(cmd.Get) - chinampa.Register(cmd.Set) - chinampa.Register(cmd.Diff) - chinampa.Register(cmd.Fetch) - chinampa.Register(cmd.Flush) - chinampa.Register(cmd.Plugin) + chinampa.Register( + cmd.Get, + cmd.Set, + cmd.Diff, + cmd.Fetch, + cmd.Flush, + cmd.Plugin, + ) + chinampa.Register(cmd.GitFilters...) if err := chinampa.Execute(chinampa.Config{ Name: "joao", diff --git a/pkg/config/config.go b/pkg/config/config.go index cc978ac..3e96627 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -193,6 +193,12 @@ func (cfg *Config) DiffRemote(path string, stdout io.Writer, stderr io.Writer) e diff.Stderr = stderr if err := diff.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + if diff.ProcessState.ExitCode() == 1 { + return nil + } + + } return fmt.Errorf("diff could not run: %w", err) } diff --git a/pkg/config/input.go b/pkg/config/input.go index 2dfd11b..696321a 100644 --- a/pkg/config/input.go +++ b/pkg/config/input.go @@ -24,7 +24,7 @@ func Load(ref string, preferRemote bool) (*Config, error) { if argIsYAMLFile(ref) { var err error - name, vault, err = vaultAndNameFrom(ref, nil) + name, vault, err = VaultAndNameFrom(ref, nil) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func FromFile(path string) (*Config, error) { buf = []byte("{}") } - name, vault, err := vaultAndNameFrom(path, buf) + name, vault, err := VaultAndNameFrom(path, buf) if err != nil { return nil, err } diff --git a/pkg/config/util.go b/pkg/config/util.go index 9290182..a7ee8c7 100644 --- a/pkg/config/util.go +++ b/pkg/config/util.go @@ -32,7 +32,7 @@ func argIsYAMLFile(path string) bool { return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") } -func vaultAndNameFrom(path string, buf []byte) (name string, vault string, err error) { +func VaultAndNameFrom(path string, buf []byte) (name string, vault string, err error) { smc := &singleModeConfig{} if buf == nil { var err error