add git filters, reorganize readme

This commit is contained in:
Roberto Hidalgo 2023-01-11 00:57:58 -06:00
parent 632af1a2be
commit 7bc47f6a9c
10 changed files with 285 additions and 54 deletions

138
README.md
View File

@ -2,6 +2,36 @@
A very wip configuration manager. Keeps config entries encoded as YAML in the filesystem, backs it up to 1Password, and syncs scrubbed copies to git. robots consume entries via 1Password Connect + Vault.
## Usage
```sh
# PATH refers to a filesystem path
# examples: config/host/juazeiro.yaml, service/gitea/config.joao.yaml
# QUERY refers to a sequence of keys delimited by dots
# examples: tls.cert, roles.0, dc, . (literal dot meaning the whole thing)
# there's better help available within each command, try:
joao get --help
# get a single value/tree from a single item/file
joao get [--output|-o=(raw|json|yaml|op)] [--remote] PATH [QUERY]
# set/update a single value in a single item/file
joao set [--secret] [--flush] [--input=/path/to/input|<<<"value"] PATH QUERY
# sync local changes upstream
joao flush [--dry-run] [--redact] PATH
# sync remote secrets to filesystem
joao fetch [--dry-run] PATH
# check for differences between local and remote items
joao diff [--cache] PATH
# show information on the git integration
joao git-filter
# show information on the vault integration
joao vault server --help
```
## Why
So I wanted to operate on my configuration mess...
@ -94,37 +124,85 @@ smtp:
```
## Usage
## git integration
In order to store configuration files within a git repository while keeping secrets off remote copies, `joao` provides git filters.
To install them, **every collaborator** would need to run:
```sh
# NAME can be either a filesystem path or a colon delimited item name
# for example: config/host/juazeiro.yaml or [op-vault-name/]host:juazeiro
# setup filters in your local copy of the repo:
# this runs when you check in a file (i.e. about to commit a config file)
# it will flush secrets to 1password before removing secrets from the file on disk
git config filter.joao.clean "joao git-filter clean --flush %f"
# this step runs after checkout (i.e. pulling changes)
# it simply outputs the file as-is on disk
git config filter.joao.smudge cat
# let's enforce these filters
git config filter.joao.required true
# DOT_DELIMITED_PATH is
# for example: tls.cert, roles.0, dc
# get a single value/tree from a single item/file
joao get NAME [--output|-o=(raw|json|yaml|op)] [--remote] [jq expr]
# set/update a single value in a single item/file
joao set NAME DOT_DELIMITED_PATH [--secret] [--flush] [--input=/path/to/input|<<<"value"]
# sync local changes upstream
joao flush NAME [--dry-run] [--redact]
# sync remote secrets to filesystem
joao fetch NAME [--dry-run]
# check for differences between local and remote items
joao diff PATH [--cache]
# print the repo config root
# tbd
# initialize a new joao repo
# joao repo init [PATH]
# list the item names within prefix
# joao repo list [PREFIX]
# joao repo root
# joao repo status
# joao repo filter clean FILE
# joao repo filter diff PATH OLD_FILE OLD_SHA OLD_MODE NEW_FILE NEW_SHA NEW_MODE
# joao repo filter smudge FILE
# get instructions to run as a vault plugin:
joao vault server --help
# optionally, configure a diff filter to show changes as would be commited to git
# this does not modify the original file on disk
git config diff.joao.textconv "joao git-filter diff"
```
Then, **only once**, we need to specify which files to apply the filters and diff commands to:
```sh
# adds diff and filter attributes for config files ending with .joao.yaml
echo '**/*.joao.yaml filter=joao diff=joao' >> .gitattributes
# finally, commit and push these attributes
git add .gitattributes
git commit -m "installing joao attributes"
git push origin main
```
See:
- https://git-scm.com/docs/gitattributes#_filter
- https://git-scm.com/docs/gitattributes#_diff
## vault integration
`joao` can run as a plugin to Hashicorp Vault, and make whole configuration entries available—secrets and all—through the Vault API.
To install, download `joao` to the machine running `vault` at the `plugin_directory`, as specified by vault's config. The installed `joao` executable needs to be executable for the user running vault only.
### Configuration
```sh
export VAULT_PLUGIN_DIR=/var/lib/vault/plugins
chmod 700 "$VAULT_PLUGIN_DIR/joao"
export PLUGIN_SHA="$(openssl dgst -sha256 -hex "$VAULT_PLUGIN_DIR/joao" | awk '{print $2}')"
export VERSION="$($VAULT_PLUGIN_DIR/joao --version)"
# register
vault plugin register -sha256="$PLUGIN_SHA" -command=joao -args="vault,server" -version="$VERSION" secret joao
# configure, add `vault` to set a default vault for querying
vault write config/1password "host=$OP_CONNECT_HOST" "token=$OP_CONNECT_TOKEN" # vault=my-default-vault
if !vault plugin list secret | grep -c -m1 '^joao ' >/dev/null; then
# first time, let's enable the secrets backend
vault secrets enable --path=config joao
else
# updating from a previous version
vault secrets tune -plugin-version="$VERSION" config/
vault plugin reload -plugin joao
fi
```
### Vault API
```sh
# VAULT is optional if configured with a default `vault`. See above
# vault read config/tree/[VAULT/]ITEM
vault read config/tree/service:api
vault read config/tree/prod/service:api
# vault list config/trees/[VAULT/]
vault list config/trees
vault list config/trees/prod
```
See:
- https://developer.hashicorp.com/vault/docs/plugins

145
cmd/git-filters.go Normal file
View File

@ -0,0 +1,145 @@
// Copyright © 2022 Roberto Hidalgo <joao@un.rob.mx>
// SPDX-License-Identifier: Apache-2.0
package cmd
import (
"os"
"git.rob.mx/nidito/chinampa/pkg/command"
"git.rob.mx/nidito/joao/pkg/config"
)
var GitFilters = []*command.Command{
FilterDiff,
FilterClean,
FilterGroup,
}
func redactedData(cmd *command.Command) error {
path := cmd.Arguments[0].ToValue().(string)
flush := false
if opt, ok := cmd.Options["flush"]; ok {
flush = opt.ToValue().(bool)
}
contents, err := os.ReadFile(path)
if err != nil {
return err
}
cfg, err := config.FromYAML(contents)
if err != nil {
return err
}
if flush {
name, vault, err := config.VaultAndNameFrom(path, contents)
if err != nil {
return err
}
cfg.Name = name
cfg.Vault = vault
}
res, err := cfg.AsYAML(config.OutputModeRedacted)
if err != nil {
return err
}
_, err = cmd.Cobra.OutOrStdout().Write(res)
return err
}
var FilterGroup = &command.Command{
Path: []string{"git-filter"},
Summary: "Subcommands used by `git` as filters",
Description: `In order to store configuration files within a git repository while keeping secrets off remote copies, joao provides git filters.
To install them, **every collaborator** would need to run:
sh
# setup filters in your local copy of the repo:
# this runs when you check in a file (i.e. about to commit a config file)
# it will flush secrets to 1password before removing secrets from the file on disk
git config filter.joao.clean "joao git-filter clean --flush %f"
# this step runs after checkout (i.e. pulling changes)
# it simply outputs the file as-is on disk
git config filter.joao.smudge cat
# let's enforce these filters
git config filter.joao.required true
# optionally, configure a diff filter to show changes as would be commited to git
# this does not modify the original file on disk
git config diff.joao.textconv "joao git-filter diff"
Then, **only once**, we need to specify which files to apply the filters and diff commands to:
sh
# adds diff and filter attributes for config files ending with .joao.yaml
echo '**/*.joao.yaml filter=joao diff=joao' >> .gitattributes
# finally, commit and push these attributes
git add .gitattributes
git commit -m "installing joao attributes"
git push origin main
See:
- https://git-scm.com/docs/gitattributes#_filter
- https://git-scm.com/docs/gitattributes#_diff`,
Arguments: command.Arguments{},
Options: command.Options{},
Action: func(cmd *command.Command) error {
data, err := cmd.ShowHelp(command.Root.Options, os.Args)
if err != nil {
return err
}
_, err = cmd.Cobra.OutOrStderr().Write(data)
return err
},
}
var FilterDiff = &command.Command{
Path: []string{"git-filter", "diff"},
Summary: "a filter for git to call during `git diff`",
Description: `see ﹅joao git-filter﹅ for instructions to install this filter`,
Arguments: command.Arguments{
{
Name: "path",
Description: "The git staged path to read from",
Required: true,
Values: &command.ValueSource{
Files: &[]string{"yaml", "yml"},
},
},
},
Options: command.Options{},
Action: redactedData,
}
var FilterClean = &command.Command{
Path: []string{"git-filter", "clean"},
Summary: "a filter for git to call when a file is checked in",
Description: `see joao git-filter for instructions to install this filter
Use --flush to save changes to 1password before redacting file.`,
Arguments: command.Arguments{
{
Name: "path",
Description: "The git staged path to read from",
Required: true,
Values: &command.ValueSource{
Files: &[]string{"yaml", "yml"},
},
},
},
Options: command.Options{
"flush": {
Description: "Save to 1Password after before redacting",
Type: "bool",
},
},
Action: redactedData,
}

View File

@ -1,3 +0,0 @@
// Copyright © 2022 Roberto Hidalgo <joao@un.rob.mx>
// SPDX-License-Identifier: Apache-2.0
package cmd

View File

@ -12,7 +12,7 @@ import (
var Plugin = &command.Command{
Path: []string{"vault", "server"},
Summary: "Starts a vault-joao-plugin server",
Description: `Runs joao as a vault plugin. See https://developer.hashicorp.com/vault/docs/plugins
Description: `Runs joao as a vault plugin.
You'll need to install joao in the machine running vault to plugin_directory as specified by vault's config. The installed joao executable needs to be executable for the user running vault only.
@ -52,6 +52,9 @@ vault read config/tree/prod/service:api
vault list config/trees
vault list config/trees/prod
See:
- https://developer.hashicorp.com/vault/docs/plugins
`,
Options: command.Options{
"ca-cert": {

6
go.mod
View File

@ -5,7 +5,7 @@ go 1.18
// replace git.rob.mx/nidito/chinampa => /Users/roberto/src/chinampa
require (
git.rob.mx/nidito/chinampa v0.0.0-20230102065449-d9b257e145ce
git.rob.mx/nidito/chinampa v0.0.0-20230111054043-db21a53b2d20
github.com/1Password/connect-sdk-go v1.5.0
github.com/alessio/shellescape v1.4.1
github.com/hashicorp/go-hclog v1.4.0
@ -86,8 +86,8 @@ require (
golang.org/x/term v0.4.0 // indirect
golang.org/x/text v0.6.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6 // indirect
google.golang.org/grpc v1.51.0 // indirect
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
google.golang.org/grpc v1.52.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
)

15
go.sum
View File

@ -1,6 +1,6 @@
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
git.rob.mx/nidito/chinampa v0.0.0-20230102065449-d9b257e145ce h1:fKG3wUdPgsviY2mE79vhXL4CalNdvhkL6vDAtdyVt0I=
git.rob.mx/nidito/chinampa v0.0.0-20230102065449-d9b257e145ce/go.mod h1:obhWsLkUIlKJyhfa7uunrSs2O44JBqsegSAtAvY2LRM=
git.rob.mx/nidito/chinampa v0.0.0-20230111054043-db21a53b2d20 h1:hHzbCNAFx9wdi3NyliugN6dTEX8dGsuup6wk2+8TmnI=
git.rob.mx/nidito/chinampa v0.0.0-20230111054043-db21a53b2d20/go.mod h1:obhWsLkUIlKJyhfa7uunrSs2O44JBqsegSAtAvY2LRM=
github.com/1Password/connect-sdk-go v1.5.0 h1:F0WJcLSzGg3iXEDY49/ULdszYKsQLGTzn+2cyYXqiyk=
github.com/1Password/connect-sdk-go v1.5.0/go.mod h1:TdynFeyvaRoackENbJ8RfJokH+WAowAu1MLmUbdMq6s=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@ -83,7 +83,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -351,16 +351,15 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6 h1:uUn6GsgKK2eCI0bWeRMgRCcqDaQXYDuB+5tXA5Xeg/8=
google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk=
google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=

15
main.go
View File

@ -24,12 +24,15 @@ func main() {
logrus.Debug("Debugging enabled")
}
chinampa.Register(cmd.Get)
chinampa.Register(cmd.Set)
chinampa.Register(cmd.Diff)
chinampa.Register(cmd.Fetch)
chinampa.Register(cmd.Flush)
chinampa.Register(cmd.Plugin)
chinampa.Register(
cmd.Get,
cmd.Set,
cmd.Diff,
cmd.Fetch,
cmd.Flush,
cmd.Plugin,
)
chinampa.Register(cmd.GitFilters...)
if err := chinampa.Execute(chinampa.Config{
Name: "joao",

View File

@ -193,6 +193,12 @@ func (cfg *Config) DiffRemote(path string, stdout io.Writer, stderr io.Writer) e
diff.Stderr = stderr
if err := diff.Run(); err != nil {
if _, ok := err.(*exec.ExitError); ok {
if diff.ProcessState.ExitCode() == 1 {
return nil
}
}
return fmt.Errorf("diff could not run: %w", err)
}

View File

@ -24,7 +24,7 @@ func Load(ref string, preferRemote bool) (*Config, error) {
if argIsYAMLFile(ref) {
var err error
name, vault, err = vaultAndNameFrom(ref, nil)
name, vault, err = VaultAndNameFrom(ref, nil)
if err != nil {
return nil, err
}
@ -66,7 +66,7 @@ func FromFile(path string) (*Config, error) {
buf = []byte("{}")
}
name, vault, err := vaultAndNameFrom(path, buf)
name, vault, err := VaultAndNameFrom(path, buf)
if err != nil {
return nil, err
}

View File

@ -32,7 +32,7 @@ func argIsYAMLFile(path string) bool {
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
}
func vaultAndNameFrom(path string, buf []byte) (name string, vault string, err error) {
func VaultAndNameFrom(path string, buf []byte) (name string, vault string, err error) {
smc := &singleModeConfig{}
if buf == nil {
var err error