Hack week: integrate with HCP Vault Secrets aka HVS

config/config.go

@@ -44,6 +44,12 @@ const (
 // homePath is the location to the user's home directory.
 var homePath, _ = homedir.Dir()
 
+type HCPVSConfig struct {
+	OrgID   string
+	ProjID  string
+	WIPName string
+}
+
 // Config is used to configure Consul Template
 type Config struct {
 	// Consul is the configuration for connecting to a Consul cluster.
@@ -92,6 +98,8 @@ type Config struct {
 	// Vault is the configuration for connecting to a vault server.
 	Vault *VaultConfig `mapstructure:"vault"`
 
+	HCPVS *HCPVSConfig
+
 	// Nomad is the configuration for connecting to a Nomad agent.
 	Nomad *NomadConfig `mapstructure:"nomad"`
 

dependency/client_set.go

@@ -10,11 +10,18 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"os"
 	"sync"
 	"time"
 
+	httptransport "github.com/go-openapi/runtime/client"
 	consulapi "github.com/hashicorp/consul/api"
 	rootcerts "github.com/hashicorp/go-rootcerts"
+	"github.com/hashicorp/hcp-sdk-go/auth"
+	"github.com/hashicorp/hcp-sdk-go/auth/workload"
+	secretspreview "github.com/hashicorp/hcp-sdk-go/clients/cloud-vault-secrets/preview/2023-11-28/client/secret_service"
+	hcpconf "github.com/hashicorp/hcp-sdk-go/config"
+	"github.com/hashicorp/hcp-sdk-go/httpclient"
 	nomadapi "github.com/hashicorp/nomad/api"
 	vaultapi "github.com/hashicorp/vault/api"
 	vaultkubernetesauth "github.com/hashicorp/vault/api/auth/kubernetes"
@@ -28,6 +35,7 @@ type ClientSet struct {
 	vault  *vaultClient
 	consul *consulClient
 	nomad  *nomadClient
+	hcpvs  *hcpvsClient
 }
 
 // consulClient is a wrapper around a real Consul API client.
@@ -48,6 +56,13 @@ type nomadClient struct {
 	httpClient *http.Client
 }
 
+type hcpvsClient struct {
+	client    secretspreview.ClientService
+	transport *httptransport.Runtime
+	orgID     string
+	projID    string
+}
+
 // TransportDialer is an interface that allows passing a custom dialer function
 // to an HTTP client's transport config
 type TransportDialer interface {
@@ -141,6 +156,13 @@ type CreateNomadClientInput struct {
 	TransportTLSHandshakeTimeout time.Duration
 }
 
+type CreateHVSClientInput struct {
+	JWTToken string
+	WIPName  string
+	OrgID    string
+	ProjID   string
+}
+
 // NewClientSet creates a new client set that is ready to accept clients.
 func NewClientSet() *ClientSet {
 	return &ClientSet{}
@@ -487,6 +509,47 @@ func (c *ClientSet) CreateNomadClient(i *CreateNomadClientInput) error {
 	return nil
 }
 
+func (c *ClientSet) CreateHCPVaultSecretsClient(i *CreateHVSClientInput) error {
+	opts := []hcpconf.HCPConfigOption{
+		//hcpconf.FromEnv(),
+		hcpconf.WithoutBrowserLogin(),
+	}
+	os.Setenv("NOMAD_TOKEN", i.JWTToken)
+	cf := &auth.CredentialFile{
+		Scheme: "workload",
+		Workload: &workload.IdentityProviderConfig{
+			ProviderResourceName: i.WIPName,
+			EnvironmentVariable: &workload.EnvironmentVariableCredentialSource{
+				Var: "NOMAD_TOKEN",
+			},
+		},
+	}
+	opts = append(opts, hcpconf.WithCredentialFile(cf))
+	//if creds := os.Getenv("HCP_CREDENTIALS_FILE"); creds != "" {
+	//	opts = append(opts, hcpconf.WithCredentialFilePath(creds))
+	//}
+	hcp, err := hcpconf.NewHCPConfig(opts...)
+	if err != nil {
+		return fmt.Errorf("error creating HCP config: %v", err)
+	}
+
+	transport, err := httpclient.New(httpclient.Config{HCPConfig: hcp})
+	if err != nil {
+		return fmt.Errorf("error creating HTTP transport for HCP: %v", err)
+	}
+
+	c.Lock()
+	c.hcpvs = &hcpvsClient{
+		client:    secretspreview.New(transport, nil),
+		transport: transport,
+		projID:    i.ProjID,
+		orgID:     i.OrgID,
+	}
+	c.Unlock()
+
+	return nil
+}
+
 // Consul returns the Consul client for this set.
 func (c *ClientSet) Consul() *consulapi.Client {
 	c.RLock()

dependency/dependency.go

@@ -40,6 +40,7 @@ const (
 	TypeVault
 	TypeLocal
 	TypeNomad
+	TypeHCPVS
 )
 
 const (

dependency/dependency_test.go

@@ -19,12 +19,13 @@ import (
 	"github.com/hashicorp/consul-template/test"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/sdk/testutil"
+	secretspreview "github.com/hashicorp/hcp-sdk-go/clients/cloud-vault-secrets/preview/2023-11-28/client/secret_service"
 	nomadapi "github.com/hashicorp/nomad/api"
 	vapi "github.com/hashicorp/vault/api"
 )
 
 const (
-	vaultAddr  = "http://127.0.0.1:8200"
+	vaultAddr  = "http://127.0.0.2:8200"
 	vaultToken = "a_token"
 )
 
@@ -70,14 +71,21 @@ func TestMain(m *testing.M) {
 		Fatalf("failed to create vault client: %v\n", err)
 	}
 	if err := clients.CreateNomadClient(&CreateNomadClientInput{
-		Address: "http://127.0.0.1:4646",
+		Address: "http://127.0.0.2:4646",
 	}); err != nil {
 		testConsul.Stop()
 		testVault.Stop()
 		testNomad.Stop()
 		Fatalf("failed to create nomad client: %v\n", err)
 	}
 
+	if err := clients.CreateHCPVaultSecretsClient("", nil); err != nil {
+		testConsul.Stop()
+		testVault.Stop()
+		testNomad.Stop()
+		Fatalf("failed to create HCP vault secrets client: %v\n", err)
+	}
+
 	testClients = clients
 
 	if err := testClients.createConsulPartitions(); err != nil {
@@ -286,6 +294,7 @@ func runTestNomad() <-chan error {
 		"-consul-client-auto-join=false", "-consul-server-auto-join=false",
 		"-network-speed=100",
 		"-log-level=error", // We're just discarding it anyway
+		"-bind=127.0.0.2",
 	)
 	cmd.Stdout = io.Discard
 	cmd.Stderr = io.Discard
@@ -321,6 +330,7 @@ func initTestNomad(errCh chan<- error) {
 	}
 
 	config := nomadapi.DefaultConfig()
+	config.Address = "http://127.0.0.2:4646"
 	client, err := nomadapi.NewClient(config)
 	if err != nil {
 		errCh <- fmt.Errorf("failed to create nomad client: %w", err)
@@ -428,7 +438,7 @@ func runTestVault() {
 	}
 	args := []string{
 		"server", "-dev", "-dev-root-token-id", vaultToken,
-		"-dev-no-store-token",
+		"-dev-no-store-token", "-dev-listen-address=127.0.0.2:8200",
 	}
 	cmd := exec.Command("vault", args...)
 	cmd.Stdout = io.Discard
@@ -571,3 +581,26 @@ func (c *ClientSet) createConsulNs() error {
 
 	return nil
 }
+
+func testHCPVS(t *testing.T, appName string) *ClientSet {
+	{
+		p := secretspreview.NewDeleteAppParams()
+		p.Name = appName
+		_, err := testClients.hcpvs.client.DeleteApp(p, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	{
+		p := secretspreview.NewCreateAppParams()
+		p.Body = secretspreview.CreateAppBody{
+			Name: appName,
+		}
+		_, err := testClients.hcpvs.client.CreateApp(p, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	return testClients
+}

dependency/hcpvs_list_secrets.go

@@ -0,0 +1,77 @@
+package dependency
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/hashicorp/hcp-sdk-go/clients/cloud-vault-secrets/preview/2023-11-28/client/secret_service"
+	"github.com/pkg/errors"
+)
+
+var _ Dependency = (*HCPVSListQuery)(nil)
+
+type HCPVSListQuery struct {
+	stopCh  chan struct{}
+	appName string
+}
+
+func NewHCPVSListQuery(s string) (*HCPVSListQuery, error) {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return nil, fmt.Errorf("hcpvs.list: invalid format: %q", s)
+	}
+
+	return &HCPVSListQuery{
+		stopCh:  make(chan struct{}, 1),
+		appName: s,
+	}, nil
+}
+
+func (d *HCPVSListQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
+	select {
+	case <-d.stopCh:
+		return nil, nil, ErrStopped
+	default:
+	}
+
+	opts = opts.Merge(&QueryOptions{})
+
+	// If we got this far, we either didn't have a secret to renew, the secret was
+	// not renewable, or the renewal failed, so attempt a fresh list.
+	log.Printf("[TRACE] HCPVS: LIST %q", d.appName)
+
+	p := secret_service.NewListAppSecretsParams()
+	p.AppName = d.appName
+	p.OrganizationID = clients.hcpvs.orgID
+	p.ProjectID = clients.hcpvs.projID
+	res, err := clients.hcpvs.client.ListAppSecrets(p, nil)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, d.String())
+	}
+
+	var result []string
+	for _, v := range res.GetPayload().Secrets {
+		result = append(result, v.Name)
+	}
+
+	log.Printf("[TRACE] %s: returned %d results", d, len(result))
+
+	return respWithMetadata(result)
+}
+
+func (d *HCPVSListQuery) CanShare() bool {
+	return false
+}
+
+func (d *HCPVSListQuery) String() string {
+	return fmt.Sprintf("hcpvs.list(%s)", d.appName)
+}
+
+func (d *HCPVSListQuery) Stop() {
+	close(d.stopCh)
+}
+
+func (d *HCPVSListQuery) Type() Type {
+	return TypeHCPVS
+}

dependency/hcpvs_open_secret.go

@@ -0,0 +1,80 @@
+package dependency
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/hashicorp/hcp-sdk-go/clients/cloud-vault-secrets/preview/2023-11-28/client/secret_service"
+	"github.com/pkg/errors"
+)
+
+var _ Dependency = (*HCPVSOpenSecretQuery)(nil)
+
+type HCPVSOpenSecretQuery struct {
+	stopCh     chan struct{}
+	appName    string
+	secretName string
+}
+
+func NewHCPVSOpenSecretQuery(app, secret string) (*HCPVSOpenSecretQuery, error) {
+	app = strings.TrimSpace(app)
+	if app == "" {
+		return nil, fmt.Errorf("hcpvs.opensecret: invalid format for app: %q", app)
+	}
+	secret = strings.TrimSpace(secret)
+	if secret == "" {
+		return nil, fmt.Errorf("hcpvs.opensecret: invalid format for secret: %q", secret)
+	}
+
+	return &HCPVSOpenSecretQuery{
+		stopCh:     make(chan struct{}, 1),
+		appName:    app,
+		secretName: secret,
+	}, nil
+}
+
+func (d *HCPVSOpenSecretQuery) Fetch(clients *ClientSet, opts *QueryOptions) (interface{}, *ResponseMetadata, error) {
+	select {
+	case <-d.stopCh:
+		return nil, nil, ErrStopped
+	default:
+	}
+
+	opts = opts.Merge(&QueryOptions{})
+
+	// If we got this far, we either didn't have a secret to renew, the secret was
+	// not renewable, or the renewal failed, so attempt a fresh list.
+	log.Printf("[TRACE] %s", d)
+
+	p := secret_service.NewOpenAppSecretParams()
+	p.AppName, p.SecretName = d.appName, d.secretName
+	p.OrganizationID = clients.hcpvs.orgID
+	p.ProjectID = clients.hcpvs.projID
+	res, err := clients.hcpvs.client.OpenAppSecret(p, nil)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, d.String())
+	}
+
+	result := res.GetPayload().Secret.StaticVersion.Value
+
+	log.Printf("[TRACE] %s: returned 1 result", d)
+
+	return respWithMetadata(result)
+}
+
+func (d *HCPVSOpenSecretQuery) CanShare() bool {
+	return false
+}
+
+func (d *HCPVSOpenSecretQuery) String() string {
+	return fmt.Sprintf("hcpvs.open(%s, %s)", d.appName, d.secretName)
+}
+
+func (d *HCPVSOpenSecretQuery) Stop() {
+	close(d.stopCh)
+}
+
+func (d *HCPVSOpenSecretQuery) Type() Type {
+	return TypeHCPVS
+}