feat: Add componentKind cdktf (#328)

- Add templates for TS cdktf components with eslint and prettier configuration
- Add templates for pnpm workspaces and turborepo task execution and caching for synth
- Add cdktf FoggStack construct to parse fogg generated Backend and provider configurations
- Add Component config options for npm dependencies

.gitignore

@@ -2,10 +2,11 @@ fogg
 coverage.out
 dist
 .vscode
+!templates/templates/turbo/root/.vscode
+!testdata/**/.vscode
 
 coverage.txt
 
 bin
 node_modules/
 package-lock.json
-package.json

apply/apply.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/tkrajina/typescriptify-golang-structs/typescriptify"
 	"golang.org/x/exp/slices"
 
 	v2 "github.com/chanzuckerberg/fogg/config/v2"
@@ -27,6 +28,7 @@ import (
 	"github.com/hashicorp/terraform/registry"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
+	yaml "gopkg.in/yaml.v3"
 )
 
 // Apply will run a plan and apply all the changes to the current repo.
@@ -71,6 +73,13 @@ func Apply(fs afero.Fs, conf *v2.Config, tmpl *templates.T, upgrade bool) error
 		}
 	}
 
+	if plan.Turbo.Enabled {
+		err = applyTree(fs, tmpl.TurboRoot, tmpl.Common, "", plan.Turbo)
+		if err != nil {
+			return errs.WrapUser(err, "unable to apply Turbo config")
+		}
+	}
+
 	tfBox := tmpl.Components[v2.ComponentKindTerraform]
 	err = applyAccounts(fs, plan, tfBox, tmpl.Common)
 	if err != nil {
@@ -370,13 +379,24 @@ func applyEnvs(
 	logrus.Debug("applying envs")
 	pathModuleConfigs = make(PathModuleConfigs)
 	for env, envPlan := range p.Envs {
+
+		foggTSConverter := typescriptify.New().WithInterface(true).WithCustomJsonTag("yaml").Add(plan.Component{})
+
+		// TODO: Handle tscriptify customCode?
+		customCode := map[string]string{}
+		// TODO: move to fogg npm package
+		foggTS, err := foggTSConverter.Convert(customCode)
+		if err != nil {
+			return nil, errs.WrapUserf(err, "unable to convert Golang structs")
+		}
+
 		logrus.Debugf("applying %s", env)
 		path := fmt.Sprintf("%s/envs/%s", util.RootPath, env)
 		err = fs.MkdirAll(path, 0755)
 		if err != nil {
 			return nil, errs.WrapUserf(err, "unable to make directory %s", path)
 		}
-		err := applyTree(fs, envBox, commonBox, path, envPlan)
+		err = applyTree(fs, envBox, commonBox, path, envPlan)
 		if err != nil {
 			return nil, errs.WrapUser(err, "unable to apply templates to env")
 		}
@@ -388,12 +408,10 @@ func applyEnvs(
 				return nil, errs.WrapUser(err, "unable to make directories for component")
 			}
 
-			// NOTE(el): component kind only support TF now
-			// 					 add a dynamic check to make sure.
 			kind := componentPlan.Kind.GetOrDefault()
 			componentBox, ok := componentBoxes[kind]
 			if !ok {
-				return nil, errs.NewUserf("component of kind '%s' not supported, must be 'terraform'", kind)
+				return nil, errs.NewUserf("component of kind '%s' not supported", kind)
 			}
 
 			if componentPlan.AutoplanFiles != nil {
@@ -420,44 +438,53 @@ func applyEnvs(
 				return nil, errs.WrapUser(err, "unable to apply templates for component")
 			}
 
-			mi := make([]moduleInvocation, 0)
-			if componentPlan.ModuleSource != nil {
-				downloader, err := util.MakeDownloader(*componentPlan.ModuleSource, "", reg)
-				if err != nil {
-					return nil, errs.WrapUser(err, "unable to make a downloader")
+			if kind == v2.ComponentKindTerraform {
+				mi := make([]moduleInvocation, 0)
+				if componentPlan.ModuleSource != nil {
+					downloader, err := util.MakeDownloader(*componentPlan.ModuleSource, "", reg)
+					if err != nil {
+						return nil, errs.WrapUser(err, "unable to make a downloader")
+					}
+					mi = append(mi, moduleInvocation{
+						module: v2.ComponentModule{
+							Name:         componentPlan.ModuleName,
+							Source:       componentPlan.ModuleSource,
+							ForEach:      componentPlan.ModuleForEach,
+							Version:      nil,
+							Variables:    componentPlan.Variables,
+							Outputs:      componentPlan.Outputs,
+							Prefix:       nil,
+							ProvidersMap: componentPlan.ProvidersMap,
+						},
+						downloadFunc: downloader,
+					})
 				}
-				mi = append(mi, moduleInvocation{
-					module: v2.ComponentModule{
-						Name:         componentPlan.ModuleName,
-						Source:       componentPlan.ModuleSource,
-						ForEach:      componentPlan.ModuleForEach,
-						Version:      nil,
-						Variables:    componentPlan.Variables,
-						Outputs:      componentPlan.Outputs,
-						Prefix:       nil,
-						ProvidersMap: componentPlan.ProvidersMap,
-					},
-					downloadFunc: downloader,
-				})
-			}
 
-			for _, m := range componentPlan.Modules {
-				moduleVersion := ""
-				if m.Version != nil {
-					moduleVersion = *m.Version
+				for _, m := range componentPlan.Modules {
+					moduleVersion := ""
+					if m.Version != nil {
+						moduleVersion = *m.Version
+					}
+					downloader, err := util.MakeDownloader(*m.Source, moduleVersion, reg)
+					if err != nil {
+						return nil, errs.WrapUser(err, "unable to make a downloader")
+					}
+					mi = append(mi, moduleInvocation{
+						module:       m,
+						downloadFunc: downloader,
+					})
 				}
-				downloader, err := util.MakeDownloader(*m.Source, moduleVersion, reg)
+				pathModuleConfigs[path], err = applyModuleInvocation(fs, path, templates.Templates.ModuleInvocation, commonBox, mi, componentPlan.IntegrationRegistry)
 				if err != nil {
-					return nil, errs.WrapUser(err, "unable to make a downloader")
+					return nil, errs.WrapUser(err, "unable to apply module invocation")
 				}
-				mi = append(mi, moduleInvocation{
-					module:       m,
-					downloadFunc: downloader,
-				})
-			}
-			pathModuleConfigs[path], err = applyModuleInvocation(fs, path, templates.Templates.ModuleInvocation, commonBox, mi, componentPlan.IntegrationRegistry)
-			if err != nil {
-				return nil, errs.WrapUser(err, "unable to apply module invocation")
+			} else if kind == v2.ComponentKindCDKTF {
+				logrus.Warn("module invocations not templated for kind CDKTF")
+				err := writeStructToTS(fs, foggTS, fmt.Sprintf("%s/src/helpers/fogg-types.generated.ts", path))
+				if err != nil {
+					panic(err.Error())
+				}
+				writeYamlFile(fs, componentPlan, fmt.Sprintf("%s/.fogg-component.yaml", path))
 			}
 		}
 	}
@@ -691,6 +718,45 @@ func createFile(dest afero.Fs, path string, sourceFile io.Reader) error {
 	return nil
 }
 
+func writeYamlFile(dest afero.Fs, in interface{}, path string) error {
+	out, err := yaml.Marshal(in)
+	if err != nil {
+		return errs.WrapInternal(err, "yaml: could not marshal")
+	}
+	dir, _ := filepath.Split(path)
+	ospath := filepath.FromSlash(dir)
+	err = dest.MkdirAll(ospath, 0775)
+	if err != nil {
+		return errs.WrapUserf(err, "couldn't create %s directory", dir)
+	}
+	return afero.WriteFile(dest, path, out, 0644)
+}
+
+// helper method supporting afero.Fs to write converted golang structs to a file
+func writeStructToTS(dest afero.Fs, converted string, path string) error {
+	dir, _ := filepath.Split(path)
+	ospath := filepath.FromSlash(dir)
+	err := dest.MkdirAll(ospath, 0775)
+	if err != nil {
+		return errs.WrapUserf(err, "couldn't create %s directory", dir)
+	}
+
+	f, err := dest.Create(path)
+	if err != nil {
+		return errs.WrapUserf(err, "unable to open %q", path)
+	}
+	defer f.Close()
+	if _, err := f.WriteString("/* Do not change, this code is generated from Golang structs */\n\n"); err != nil {
+		return errs.WrapUserf(err, "unable to write to %q", path)
+	}
+	if _, err := f.WriteString(converted); err != nil {
+		return errs.WrapUserf(err, "unable to write to %q", path)
+	}
+
+	logrus.Infof("%s updated", path)
+	return nil
+}
+
 func removeExtension(path string) string {
 	return strings.TrimSuffix(path, filepath.Ext(path))
 }

apply/golden_file_test.go

@@ -46,6 +46,7 @@ func TestIntegration(t *testing.T) {
 		{"v2_integration_registry"},
 		{"v2_github_actions_with_pre_commit"},
 		{"v2_atlantis_depends_on"},
+		{"v2_cdktf_components"},
 		{"generic_providers_yaml"},
 	}
 

config/v2/config.go

@@ -150,6 +150,7 @@ type Config struct {
 	Version  int                `validate:"required,eq=2"`
 	TFE      *TFE               `yaml:"tfe,omitempty"`
 	ConfDir  *string            `yaml:"conf_dir,omitempty"`
+	Turbo    *TurboConfig       `yaml:"turbo,omitempty"`
 }
 
 type TFE struct {
@@ -229,18 +230,21 @@ type Env struct {
 	Components map[string]Component `yaml:"components,omitempty"`
 }
 
+// TODO: Support cdktf depedencies from a private registry/scope
 type Component struct {
 	Common `yaml:",inline"`
 
-	EKS           *EKSConfig        `yaml:"eks,omitempty"`
-	Kind          *ComponentKind    `yaml:"kind,omitempty"`
-	ModuleSource  *string           `yaml:"module_source,omitempty"`
-	ModuleName    *string           `yaml:"module_name,omitempty"`
-	ModuleForEach *string           `yaml:"module_for_each,omitempty"`
-	ProvidersMap  map[string]string `yaml:"module_providers,omitempty"`
-	Variables     []string          `yaml:"variables,omitempty"`
-	Outputs       []string          `yaml:"outputs,omitempty"`
-	Modules       []ComponentModule `yaml:"modules,omitempty"`
+	EKS                  *EKSConfig             `yaml:"eks,omitempty"`
+	Kind                 *ComponentKind         `yaml:"kind,omitempty"`
+	ModuleSource         *string                `yaml:"module_source,omitempty"`
+	ModuleName           *string                `yaml:"module_name,omitempty"`
+	ModuleForEach        *string                `yaml:"module_for_each,omitempty"`
+	ProvidersMap         map[string]string      `yaml:"module_providers,omitempty"`
+	Variables            []string               `yaml:"variables,omitempty"`
+	Outputs              []string               `yaml:"outputs,omitempty"`
+	Modules              []ComponentModule      `yaml:"modules,omitempty"`
+	CdktfDependencies    []JavascriptDependency `yaml:"cdktf_dependencies,omitempty"`     // Optional additional component dev dependencies, default: []
+	CdktfDevDependencies []JavascriptDependency `yaml:"cdktf_dev_dependencies,omitempty"` // Optional additional component dev dependencies, default: []
 }
 
 type ComponentModule struct {
@@ -563,6 +567,19 @@ type TfLint struct {
 	Enabled *bool `yaml:"enabled,omitempty"`
 }
 
+type TurboConfig struct {
+	Enabled  *bool   `yaml:"enabled,omitempty"`   // Enable Turbo, default: false
+	Version  *string `yaml:"version,omitempty"`   // Optional Turbo version, default: "^2.0.6"
+	RootName *string `yaml:"root_name,omitempty"` // Optional Name for the root package, default: "fogg-monorepo"
+
+	DevDependencies []JavascriptDependency `yaml:"dev_dependencies,omitempty"` // Optional additional root dev dependencies, default: []
+}
+
+type JavascriptDependency struct {
+	Name    string `yaml:"name"`    // npm package name
+	Version string `yaml:"version"` // npm package version
+}
+
 // EKSConfig is the configuration for an eks cluster
 type EKSConfig struct {
 	ClusterName string `yaml:"cluster_name"`
@@ -630,4 +647,6 @@ const (
 	DefaultComponentKind ComponentKind = "terraform"
 	// ComponentKindTerraform is a terraform component
 	ComponentKindTerraform = DefaultComponentKind
+	// ComponentKindCDKTF is a CDKTF component
+	ComponentKindCDKTF ComponentKind = "cdktf"
 )

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
+	github.com/tkrajina/typescriptify-golang-structs v0.1.12-0.20240302095452-72a3fd882f3f
 	github.com/zclconf/go-cty v1.15.0
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 	gopkg.in/go-playground/validator.v9 v9.31.0
@@ -106,6 +107,7 @@ require (
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/skeema/knownhosts v1.2.2 // indirect
+	github.com/tkrajina/go-reflector v0.5.5 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect

go.sum

@@ -802,6 +802,10 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.82+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
 github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190808065407-f07404cefc8c/go.mod h1:wk2XFUg6egk4tSDNZtXeKfe2G6690UVyt163PuUxBZk=
+github.com/tkrajina/go-reflector v0.5.5 h1:gwoQFNye30Kk7NrExj8zm3zFtrGPqOkzFMLuQZg1DtQ=
+github.com/tkrajina/go-reflector v0.5.5/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4=
+github.com/tkrajina/typescriptify-golang-structs v0.1.12-0.20240302095452-72a3fd882f3f h1:NuGUSSt6GUGiBuYxsw5yxQ+Nm62GEr+cRh7SRrDZyVc=
+github.com/tkrajina/typescriptify-golang-structs v0.1.12-0.20240302095452-72a3fd882f3f/go.mod h1:sjU00nti/PMEOZb07KljFlR+lJ+RotsC0GBQMv9EKls=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tombuildsstuff/giovanni v0.15.1/go.mod h1:0TZugJPEtqzPlMpuJHYfXY6Dq2uLPrXf98D2XQSxNbA=
 github.com/ugorji/go v0.0.0-20180813092308-00b869d2f4a5/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=

plan/ci.go

@@ -48,6 +48,14 @@ type AtlantisConfig struct {
 	RepoCfg *atlantis.RepoCfg
 }
 
+type TurboConfig struct {
+	Enabled         bool
+	Version         string
+	RootName        string
+	DevDependencies map[string]string
+	CdktfComponents map[string]Component
+}
+
 type PreCommitConfig struct {
 	Enabled            bool
 	Requirements       []string
@@ -379,6 +387,50 @@ func (p *Plan) buildGitHubActionsConfig(c *v2.Config, foggVersion string) GitHub
 	}
 }
 
+func (p *Plan) buildTurboRootConfig(c *v2.Config) *TurboConfig {
+	turboConfig := &TurboConfig{
+		Enabled:  false,
+		RootName: "fogg-monorepo",
+		DevDependencies: map[string]string{
+			"@types/node": "^20.6.0",
+			"turbo":       "^2.0.12",
+			"typescript":  "^5.4.0",
+		},
+	}
+
+	if c.Turbo != nil {
+		if c.Turbo.Enabled != nil {
+			turboConfig.Enabled = *c.Turbo.Enabled
+		}
+
+		if c.Turbo.Version != nil {
+			turboConfig.Version = *c.Turbo.Version
+		}
+
+		if c.Turbo.RootName != nil {
+			turboConfig.RootName = *c.Turbo.RootName
+		}
+
+		for _, dep := range c.Turbo.DevDependencies {
+			turboConfig.DevDependencies[dep.Name] = dep.Version
+		}
+
+		turboConfig.CdktfComponents = make(map[string]Component)
+		for env, envPlan := range p.Envs {
+			for component, componentPlan := range envPlan.Components {
+
+				kind := componentPlan.Kind.GetOrDefault()
+				if kind == v2.ComponentKindCDKTF {
+					// applyEnvs implementation detail
+					path := fmt.Sprintf("%s/envs/%s/%s", util.RootPath, env, component)
+					turboConfig.CdktfComponents[path] = componentPlan
+				}
+			}
+		}
+	}
+	return turboConfig
+}
+
 // buildAtlantisConfig must be build after Envs
 func (p *Plan) buildAtlantisConfig(c *v2.Config) AtlantisConfig {
 	enabled := false
@@ -402,7 +454,7 @@ func (p *Plan) buildAtlantisConfig(c *v2.Config) AtlantisConfig {
 						uniqueModuleSources = append(uniqueModuleSources, *m.Source)
 					}
 				}
-				whenModified := []string{"*.tf"}
+				whenModified := []string{"**/*.tf", "**/*.tf.json", "**/*.tfvars", "**/*.tfvars.json"}
 				if d.AutoplanRelativeGlobs != nil {
 					whenModified = append(whenModified, d.AutoplanRelativeGlobs...)
 				}