[Backend] New filesystem backend

.travis.yml

@@ -3,7 +3,7 @@ go:
   - 1.x
   - tip
 env:
-  - VAULT_ADDR='http://127.0.0.1:8200' CONSUL_VERSION=0.9.3 ETCD_VERSION=3.3.1 DYNAMODB_VERSION=2017-02-16 VAULT_VERSION=0.10.1 ZOOKEEPER_VERSION=3.4.10 RANCHER_VERSION=0.6.0
+  - VAULT_ADDR='http://127.0.0.1:8200' CONSUL_VERSION=0.9.3 ETCD_VERSION=3.3.1 DYNAMODB_VERSION=2017-02-16 VAULT_VERSION=0.10.1 ZOOKEEPER_VERSION=3.4.14 RANCHER_VERSION=0.6.0
 services:
   - redis
 before_install:

backends/client.go

@@ -10,6 +10,7 @@ import (
 	"github.com/kelseyhightower/confd/backends/etcd"
 	"github.com/kelseyhightower/confd/backends/etcdv3"
 	"github.com/kelseyhightower/confd/backends/file"
+	"github.com/kelseyhightower/confd/backends/filesystem"
 	"github.com/kelseyhightower/confd/backends/rancher"
 	"github.com/kelseyhightower/confd/backends/redis"
 	"github.com/kelseyhightower/confd/backends/ssm"
@@ -64,6 +65,8 @@ func New(config Config) (StoreClient, error) {
 		return env.NewEnvClient()
 	case "file":
 		return file.NewFileClient(config.YAMLFile, config.Filter)
+	case "filesystem":
+		return filesystem.NewFileSystemClient(config.MaxFileSize)
 	case "vault":
 		vaultConfig := map[string]string{
 			"app-id":    config.AppID,

backends/config.go

@@ -5,26 +5,27 @@ import (
 )
 
 type Config struct {
-	AuthToken    string     `toml:"auth_token"`
-	AuthType     string     `toml:"auth_type"`
-	Backend      string     `toml:"backend"`
-	BasicAuth    bool       `toml:"basic_auth"`
-	ClientCaKeys string     `toml:"client_cakeys"`
-	ClientCert   string     `toml:"client_cert"`
-	ClientKey    string     `toml:"client_key"`
-        ClientInsecure bool     `toml:"client_insecure"`
-	BackendNodes util.Nodes `toml:"nodes"`
-	Password     string     `toml:"password"`
-	Scheme       string     `toml:"scheme"`
-	Table        string     `toml:"table"`
-	Separator    string     `toml:"separator"`
-	Username     string     `toml:"username"`
-	AppID        string     `toml:"app_id"`
-	UserID       string     `toml:"user_id"`
-	RoleID       string     `toml:"role_id"`
-	SecretID     string     `toml:"secret_id"`
-	YAMLFile     util.Nodes `toml:"file"`
-	Filter       string     `toml:"filter"`
-	Path         string     `toml:"path"`
-	Role         string
+	AuthToken      string     `toml:"auth_token"`
+	AuthType       string     `toml:"auth_type"`
+	Backend        string     `toml:"backend"`
+	BasicAuth      bool       `toml:"basic_auth"`
+	ClientCaKeys   string     `toml:"client_cakeys"`
+	ClientCert     string     `toml:"client_cert"`
+	ClientKey      string     `toml:"client_key"`
+	ClientInsecure bool       `toml:"client_insecure"`
+	BackendNodes   util.Nodes `toml:"nodes"`
+	Password       string     `toml:"password"`
+	Scheme         string     `toml:"scheme"`
+	Table          string     `toml:"table"`
+	Separator      string     `toml:"separator"`
+	Username       string     `toml:"username"`
+	AppID          string     `toml:"app_id"`
+	UserID         string     `toml:"user_id"`
+	RoleID         string     `toml:"role_id"`
+	SecretID       string     `toml:"secret_id"`
+	YAMLFile       util.Nodes `toml:"file"`
+	Filter         string     `toml:"filter"`
+	Path           string     `toml:"path"`
+	Role           string
+	MaxFileSize    int64      `toml:"max_file_size"`
 }

backends/etcd/client.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/coreos/etcd/client"
 	"golang.org/x/net/context"
-        "github.com/kelseyhightower/confd/log"
+    "github.com/kelseyhightower/confd/log"
 )
 
 // Client is a wrapper around the etcd client

backends/filesystem/client.go

@@ -0,0 +1,77 @@
+package filesystem
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+
+	util "github.com/kelseyhightower/confd/util"
+)
+
+// Client provides a shell for the filesystem client
+type Client struct {
+	MaxFileSize int64
+}
+
+// NewFilesystemClient returns a new client
+func NewFileSystemClient(max int64) (*Client, error) {
+	return &Client{MaxFileSize: max}, nil
+}
+
+// GetValues queries the filesystem for keys
+func (c *Client) GetValues(keys []string) (map[string]string, error) {
+	vars := make(map[string]string)
+	for _, key := range keys {
+		fpath := toPath(key)
+		stat, err := os.Stat(fpath)
+		if err != nil {
+			return nil, err
+		}
+
+		if stat.IsDir() {
+			// Walk subdirs
+			flist, err := util.RecursiveFilesLookup(fpath, "*")
+			if err != nil {
+				return nil, err
+			}
+
+			klist := make([]string, 0)
+			for _, v := range flist {
+				klist = append(klist, toKey(v))
+			}
+
+			// Recursively get values for sub-keys (fles)
+			recurseVars, err := c.GetValues(klist)
+			if err != nil {
+				return nil, err
+			}
+
+			// Add to our vars map
+			for k, v := range recurseVars {
+				vars[k] = v
+			}
+		} else {
+			if stat.Size() > c.MaxFileSize {
+				return nil, fmt.Errorf("size of %s too large", key)
+			}
+
+			// Read contents of file
+			b, err := ioutil.ReadFile(fpath)
+			if err != nil {
+				return nil, err
+			}
+
+			// Add to vars
+			vars[key] = strings.TrimSpace(string(b))
+
+		}
+	}
+
+	return vars, nil
+}
+
+func (c *Client) WatchPrefix(prefix string, keys []string, waitIndex uint64, stopChan chan bool) (uint64, error) {
+	<-stopChan
+	return 0, nil
+}

backends/filesystem/helpers_posix.go

@@ -0,0 +1,15 @@
+// +build !windows
+
+package filesystem
+
+import (
+	"path/filepath"
+)
+
+func toPath(key string) string {
+	return filepath.FromSlash(key)
+}
+
+func toKey(path string) string {
+	return filepath.ToSlash(path)
+}

backends/filesystem/helpers_windows.go

@@ -0,0 +1,37 @@
+package filesystem
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func toPath(key string) string {
+	sep := string(os.PathSeparator)
+
+	// Replace slashes with path seperator
+	key = filepath.FromSlash(key)
+
+	// Trim leading backslash, so \c\path\to\file becomes c\path\to\file
+	key = strings.TrimPrefix(key, sep)
+
+	// Replace first slash with colon-slash so c\path\to\file becomes c:\path\to\file
+	key = strings.Replace(key, sep, ":"+sep, 1)
+
+	return key
+}
+
+func toKey(path string) string {
+
+	// Replace backslash with forward slash
+	path = filepath.ToSlash(path)
+
+	// Get rid of colon
+	path = strings.Replace(path, ":", "", 1)
+
+	if !strings.HasPrefix(path, "/") {
+		path = "/" + path
+	}
+
+	return path
+}

config.go

@@ -44,7 +44,7 @@ func init() {
 	flag.StringVar(&config.ClientCaKeys, "client-ca-keys", "", "client ca keys")
 	flag.StringVar(&config.ClientCert, "client-cert", "", "the client cert")
 	flag.StringVar(&config.ClientKey, "client-key", "", "the client key")
-        flag.BoolVar(&config.ClientInsecure, "client-insecure", false, "Allow connections to SSL sites without certs (only used with -backend=etcd)")
+    flag.BoolVar(&config.ClientInsecure, "client-insecure", false, "Allow connections to SSL sites without certs (only used with -backend=etcd)")
 	flag.StringVar(&config.ConfDir, "confdir", "/etc/confd", "confd conf directory")
 	flag.StringVar(&config.ConfigFile, "config-file", "/etc/confd/confd.toml", "the confd config file")
 	flag.Var(&config.YAMLFile, "file", "the YAML file to watch for changes (only used with -backend=file)")
@@ -72,6 +72,7 @@ func init() {
 	flag.StringVar(&config.Separator, "separator", "", "the separator to replace '/' with when looking up keys in the backend, prefixed '/' will also be removed (only used with -backend=redis)")
 	flag.StringVar(&config.Username, "username", "", "the username to authenticate as (only used with vault and etcd backends)")
 	flag.StringVar(&config.Password, "password", "", "the password to authenticate with (only used with vault and etcd backends)")
+	flag.Int64Var(&config.MaxFileSize, "max-file-size", 16384, "the maximum file size to read (only used with -backend=filesystem)")
 	flag.BoolVar(&config.Watch, "watch", false, "enable watch support")
 }
 

config_test.go

@@ -25,6 +25,7 @@ func TestInitConfigDefaultConfig(t *testing.T) {
 		ConfigFile: "/etc/confd/confd.toml",
 		Interval:   600,
 	}
+	want.MaxFileSize = 16384
 	if err := initConfig(); err != nil {
 		t.Errorf(err.Error())
 	}

docs/command-line-flags.md

@@ -38,6 +38,8 @@ Usage of confd:
       keep staged files
   -log-level string
       level which confd should log messages
+  -max-file-size integer
+      maximum file size to read (only used with -backend=filesystem) (default 16384)
   -node value
       list of backend nodes
   -noop

docs/configuration-guide.md

@@ -36,6 +36,7 @@ Optional:
 * `file` (array of strings) - The YAML file to watch for changes (only used with -backend=file).
 * `filter` (string) - Files filter (only used with -backend=file) (default "*").
 * `path` (string) - Vault mount path of the auth method (only used with -backend=vault).
+* `max-file-size` (int64) - Maximum file size to read (only used with -backend=filesystem) (default 16384).
 
 Example:
 

docs/quick-start-guide.md

@@ -11,6 +11,7 @@ confd supports the following backends:
 * vault
 * environment variables
 * file
+* filesystem
 * redis
 * zookeeper
 * dynamodb
@@ -58,6 +59,13 @@ myapp:
     user: rob
 ```
 
+#### filesystem
+
+```
+echo "db.example.com" > /myapp/database/url
+echo "rob" > /myapp/database/user
+```
+
 #### redis
 
 ```
@@ -182,6 +190,12 @@ confd -onetime -backend env
 confd -onetime -backend file -file myapp.yaml
 ```
 
+#### filesystem
+
+```
+confd -onetime -backend filesystem
+```
+
 #### redis
 
 ```

integration/filesystem/test.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+export HOSTNAME="localhost"
+
+sudo mkdir -p /database /upstream /nested/{east,west} /prefix/database /prefix/nested/{east,west} /prefix/upstream
+
+echo "foobar" | sudo tee /key
+echo "127.0.0.1" | sudo tee /database/host
+echo "p@sSw0rd" | sudo tee /database/password
+echo "3306" | sudo tee /database/port
+echo "confd" | sudo tee /database/username
+echo "confd" | sudo tee /database/username
+echo "10.0.1.10:8080" | sudo tee /upstream/app1
+echo "10.0.1.11:8080" | sudo tee /upstream/app2
+echo "10.0.1.10:8080" | sudo tee /nested/east/app1
+echo "10.0.1.11:8080" | sudo tee /nested/west/app2
+echo "127.0.0.1" | sudo tee /prefix/database/host
+echo "p@sSw0rd" | sudo tee /prefix/database/password
+echo "3306" | sudo tee /prefix/database/port
+echo "confd" | sudo tee /prefix/database/username
+echo "10.0.1.10:8080" | sudo tee /prefix/upstream/app1
+echo "10.0.1.11:8080" | sudo tee /prefix/upstream/app2
+echo "10.0.1.10:8080" | sudo tee /prefix/nested/east/app1
+echo "10.0.1.11:8080" | sudo tee /prefix/nested/west/app2
+
+# Run confd
+confd --onetime --log-level debug --confdir ./integration/confdir --backend filesystem --watch