Merge pull request #6 from permafrost-dev/add-cache

implement cache

Author: patinthehat

README.md

@@ -57,6 +57,7 @@ The `settings` section of the configuration file is used to configure the applic
 | `defaults.tasks.platforms` | default platforms for tasks | no |
 | `defaults.tasks.silent` | default silent setting for tasks | no |
 | `dotenv`  | array of `.env` filenames to load  | no        |
+| `cache.ttl-minutes` | number of minutes to cache remote files | no |
 | `exit-on-checksum-mismatch` | `boolean` value specifying whether to exit if a checksum mismatch occurs when including a remote file | no |
 
 Example `settings` section:
@@ -68,6 +69,8 @@ version: 1.0.0
 settings:
   dotenv: ['.env', '.env.local'] # loads both `.env` and `.env.local` files, defaults to `.env`.
   exit-on-checksum-mismatch: false # do not exit if a checksum mismatch occurs, defaults to true.
+  cache:
+    ttl-minutes: 60 # cache remote files for 60 minutes, defaults to 5 minutes.
   defaults:
     tasks:
       silent: true
@@ -102,8 +105,19 @@ env:
 The `includes` section of the configuration file is used to specify a list of filenames, file urls, or s3 urls that should be merged with the configuration.  This is useful for splitting up a large configuration file into smaller, more manageable files or reusing commonly-used tasks, init scripts, or preconditions. Startup, shutdown, servers, and scheduled tasks are not merged from the included files.
 
 Included urls can be prefixed with `gh:` to indicate that the file should be fetched from GitHub.  For example, `gh:permafrost-dev/stackup/main/templates/stackup.dist.yaml` will fetch the `stackup.dist.yaml` file from the `permafrost-dev/stackup` repository on GitHub.
+Add a `headers` field to the `url` entry to specify headers to send with the request.  The `headers` field should be an array of strings, where each string is a header to send with the request.  The header value can be a javascript expression if wrapped in double braces.  For example:
 
-To use a file from an S3 bucket, prefix the url with `s3:`. For example, `s3:hostname/my-bucket-name/my-config.yaml` will fetch the `my-config.yaml` file from the `my-bucket-name` bucket on `hostname`. Amazon S3 and Minio are supported.
+```yaml
+  - url: gh:permafrost-dev/stackup/main/templates/remote-includes/node.yaml
+    headers:
+      - 'Authorization: token $GITHUB_TOKEN' 
+
+  - url: gh:permafrost-dev/stackup/main/templates/remote-includes/php.yaml
+    headers:
+      - '{{ "Authorization: token " + $myGithubTokenVar }}'
+```
+
+To import a file from an S3 bucket, prefix the url with `s3:`. For example, `s3:hostname/my-bucket-name/my-config.yaml` will fetch the `my-config.yaml` file from the `my-bucket-name` bucket on `hostname`. Amazon S3 and Minio are supported.
 
 Included files can be specified with either a relative or absolute pathname.  Relative pathnames are relative to the directory containing the configuration file.  Absolute pathnames are relative to the current working directory.
 
@@ -112,6 +126,10 @@ includes:
   - url: gh:permafrost-dev/stackup/main/templates/remote-includes/containers.yaml
     verify: false # optional, defaults to true
 
+  - url: gh:permafrost-dev/stackup/main/templates/remote-includes/node.yaml
+    headers:
+      - 'Authorization: token $GITHUB_TOKEN' # headers to send with the request, can be javascript if wrapped in double braces
+
   - file: python.yaml # includes a local file
 
   - url: s3:127.0.0.1:9000/stackup-includes/python.yaml # includes a file from a minio bucket

go.mod

@@ -42,8 +42,10 @@ require (
 require (
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/emirpasic/gods v1.18.1
+	github.com/golang-module/carbon/v2 v2.2.3
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/minio/minio-go v6.0.14+incompatible
 	github.com/minio/minio-go/v7 v7.0.61
+	go.etcd.io/bbolt v1.3.7
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )

go.sum

@@ -17,6 +17,8 @@ github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJ
 github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
+github.com/golang-module/carbon/v2 v2.2.3 h1:WvGIc5+qzq9drNzH+Gnjh1TZ0JgDY/IA+m2Dvk7Qm4Q=
+github.com/golang-module/carbon/v2 v2.2.3/go.mod h1:LdzRApgmDT/wt0eNT8MEJbHfJdSqCtT46uZhfF30dqI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -79,12 +81,19 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
+go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
@@ -134,3 +143,4 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

lib/app/workflow.go

@@ -5,9 +5,11 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 
 	lla "github.com/emirpasic/gods/lists/arraylist"
 	lls "github.com/emirpasic/gods/stacks/linkedliststack"
+	"github.com/stackup-app/stackup/lib/cache"
 	"github.com/stackup-app/stackup/lib/checksums"
 	"github.com/stackup-app/stackup/lib/downloader"
 	"github.com/stackup-app/stackup/lib/support"
@@ -31,26 +33,33 @@ type StackupWorkflow struct {
 	Scheduler     []ScheduledTask    `yaml:"scheduler"`
 	Includes      []*WorkflowInclude `yaml:"includes"`
 	State         *StackupWorkflowState
+	Cache         *cache.Cache
 }
 
 type WorkflowInclude struct {
-	Url             string `yaml:"url"`
-	File            string `yaml:"file"`
-	ChecksumUrl     string `yaml:"checksum-url"`
-	VerifyChecksum  *bool  `yaml:"verify,omitempty"`
-	AccessKey       string `yaml:"access-key"`
-	SecretKey       string `yaml:"secret-key"`
-	Secure          bool   `yaml:"secure"`
+	Url             string   `yaml:"url"`
+	Headers         []string `yaml:"headers"`
+	File            string   `yaml:"file"`
+	ChecksumUrl     string   `yaml:"checksum-url"`
+	VerifyChecksum  *bool    `yaml:"verify,omitempty"`
+	AccessKey       string   `yaml:"access-key"`
+	SecretKey       string   `yaml:"secret-key"`
+	Secure          bool     `yaml:"secure"`
 	ChecksumIsValid *bool
 	ValidationState string
 	Contents        string
+	Hash            string
+	Workflow        *StackupWorkflow
 }
 
 type WorkflowSettings struct {
 	Defaults               *WorkflowSettingsDefaults `yaml:"defaults"`
 	ExitOnChecksumMismatch bool                      `yaml:"exit-on-checksum-mismatch"`
 	DotEnvFiles            []string                  `yaml:"dotenv"`
-	Domains                struct {
+	Cache                  struct {
+		TtlMinutes int `yaml:"ttl-minutes"`
+	} `yaml:"cache"`
+	Domains struct {
 		Allowed []string `yaml:"allowed"`
 	} `yaml:"domains"`
 }
@@ -141,32 +150,39 @@ func (wi *WorkflowInclude) ValidateChecksum(contents string) (bool, error) {
 
 	algorithm := ""
 	storedChecksum := ""
+	checksumContents := ""
+	hashUrl := ""
 
 	for _, url := range checksumUrls {
+		if wi.Workflow.Cache.Has(url) && !wi.Workflow.Cache.IsExpired(url) {
+			hashUrl = url
+			checksumContents = wi.Workflow.Cache.Get(url)
+			fmt.Printf("using cached checksum file %s\n", url)
+			break
+		}
+
 		checksumContents, err := utils.GetUrlContents(url)
 		if err != nil {
-			fmt.Printf("error: %s\n", err)
 			continue
 		}
 
-		// fmt.Printf("using checksum file %s\n", url)
-
 		if checksumContents != "" {
-			storedChecksum = wi.getChecksumFromContents(checksumContents)
-
-			wi.ChecksumUrl = url
-			if strings.HasSuffix(url, ".sha256") || strings.HasSuffix(url, ".sha256.txt") {
-				algorithm = "sha256"
-			}
-			if strings.HasSuffix(url, ".sha512") || strings.HasSuffix(url, ".sha512.txt") {
-				algorithm = "sha512"
-			}
+			hashUrl = url
+			wi.Workflow.Cache.Set(url, checksumContents, wi.Workflow.Settings.Cache.TtlMinutes)
+			fmt.Printf("using non-cached checksum file %s\n", url)
 			break
 		}
 	}
 
-	if algorithm == "" {
-		// return false, fmt.Errorf("unable to find valid checksum file for %s", wi.DisplayUrl())
+	if checksumContents != "" {
+		storedChecksum = wi.getChecksumFromContents(checksumContents)
+
+		wi.ChecksumUrl = hashUrl
+		algorithm = wi.GetChecksumAlgorithm()
+	}
+
+	if algorithm == "unknown" {
+		return false, fmt.Errorf("unable to find valid checksum file for %s", wi.DisplayUrl())
 	}
 
 	var hash string
@@ -183,11 +199,6 @@ func (wi *WorkflowInclude) ValidateChecksum(contents string) (bool, error) {
 		return false, fmt.Errorf("unsupported algorithm: %s", algorithm)
 	}
 
-	// fmt.Printf("checksum url: %s\n", wi.ChecksumUrl)
-	// fmt.Printf("algorithm: %s\n", algorithm)
-	// fmt.Printf("hash: %s\n", hash)
-	// fmt.Printf("checksum: %s\n", storedChecksum)
-
 	if !strings.EqualFold(hash, storedChecksum) {
 		wi.SetChecksumIsValid(false)
 		return false, nil
@@ -252,6 +263,17 @@ func (wi *WorkflowInclude) DisplayName() string {
 	return "<unknown>"
 }
 
+func (wi *WorkflowInclude) GetChecksumAlgorithm() string {
+	if strings.HasSuffix(wi.ChecksumUrl, ".sha256") || strings.HasSuffix(wi.ChecksumUrl, ".sha256.txt") {
+		return "sha256"
+	}
+	if strings.HasSuffix(wi.ChecksumUrl, ".sha512") || strings.HasSuffix(wi.ChecksumUrl, ".sha512.txt") {
+		return "sha512"
+	}
+
+	return "unknown"
+}
+
 func (wi *WorkflowInclude) SetChecksumIsValid(value bool) {
 	wi.ChecksumIsValid = &value
 }
@@ -296,6 +318,8 @@ func (workflow *StackupWorkflow) reversePreconditions(items []*Precondition) []*
 }
 
 func (workflow *StackupWorkflow) Initialize() {
+	workflow.Cache = cache.CreateCache(utils.GetProjectName())
+
 	// generate uuids for each task as the initial step, as other code below relies on a uuid existing
 	for _, task := range workflow.Tasks {
 		task.Uuid = utils.GenerateTaskUuid()
@@ -322,6 +346,10 @@ func (workflow *StackupWorkflow) Initialize() {
 		}
 	}
 
+	if workflow.Settings.Cache.TtlMinutes <= 0 {
+		workflow.Settings.Cache.TtlMinutes = 5
+	}
+
 	if len(workflow.Settings.DotEnvFiles) == 0 {
 		workflow.Settings.DotEnvFiles = []string{".env"}
 	}
@@ -341,6 +369,11 @@ func (workflow *StackupWorkflow) Initialize() {
 		}
 	}
 
+	// initialize the includes
+	for _, inc := range workflow.Includes {
+		inc.Initialize(workflow)
+	}
+
 	workflow.ProcessIncludes()
 
 	if len(workflow.Init) > 0 {
@@ -373,20 +406,22 @@ func (workflow *StackupWorkflow) RemoveTasks(uuidsToRemove []string) {
 	workflow.Tasks = newTasks
 }
 
-func (workflow *StackupWorkflow) ProcessIncludes() {
-	// set default value for verify checksum to true
-	for _, wi := range workflow.Includes {
-		if wi.VerifyChecksum == nil {
-			boolValue := true //wi.ChecksumUrl != ""
-			wi.VerifyChecksum = &boolValue
-		}
-		wi.ValidationState = "not validated"
-		wi.ChecksumIsValid = nil
-	}
+// func (workflow *StackupWorkflow) ProcessIncludes() {
+// 	for _, include := range workflow.Includes {
+// 		workflow.ProcessInclude(include)
+// 	}
+// }
 
+func (workflow *StackupWorkflow) ProcessIncludes() {
+	var wg sync.WaitGroup
 	for _, include := range workflow.Includes {
-		workflow.ProcessInclude(include)
+		wg.Add(1)
+		go func(include *WorkflowInclude) {
+			defer wg.Done()
+			workflow.ProcessInclude(include)
+		}(include)
 	}
+	wg.Wait()
 }
 
 func (workflow *StackupWorkflow) ProcessInclude(include *WorkflowInclude) bool {
@@ -397,22 +432,33 @@ func (workflow *StackupWorkflow) ProcessInclude(include *WorkflowInclude) bool {
 	var contents string
 	var err error
 
-	if include.IsLocalFile() {
-		contents, err = utils.GetFileContents(include.Filename())
-	} else if include.IsRemoteUrl() {
-		contents, err = utils.GetUrlContents(include.FullUrl())
-	} else if include.IsS3Url() {
-		include.AccessKey = os.ExpandEnv(include.AccessKey)
-		include.SecretKey = os.ExpandEnv(include.SecretKey)
+	if workflow.Cache.Has(include.DisplayName()) && !workflow.Cache.IsExpired(include.DisplayName()) {
+		include.Contents = workflow.Cache.Get(include.DisplayName())
+		include.Hash = workflow.Cache.GetHash(include.DisplayName())
+		fmt.Println("loaded from cache")
+	}
+
+	if !workflow.Cache.Has(include.DisplayName()) || workflow.Cache.IsExpired(include.DisplayName()) {
+		fmt.Println("not loaded from cache")
+		if include.IsLocalFile() {
+			include.Contents, err = utils.GetFileContents(include.Filename())
+		} else if include.IsRemoteUrl() {
+			include.Contents, err = utils.GetUrlContentsEx(include.FullUrl(), include.Headers)
+		} else if include.IsS3Url() {
+			include.AccessKey = os.ExpandEnv(include.AccessKey)
+			include.SecretKey = os.ExpandEnv(include.SecretKey)
+			include.Contents = downloader.ReadS3FileContents(include.FullUrl(), include.AccessKey, include.SecretKey, include.Secure)
+		} else {
+			return false
+		}
 
-		contents = downloader.ReadS3FileContents(include.FullUrl(), include.AccessKey, include.SecretKey, include.Secure)
-	} else {
-		return false
+		include.Hash = checksums.CalculateSha256Hash(include.Contents)
+		workflow.Cache.Set(include.DisplayName(), include.Contents, workflow.Settings.Cache.TtlMinutes)
 	}
 
-	contents = strings.TrimSpace(contents)
-
-	include.Contents = contents
+	// fmt.Printf("value: %v\n", workflow.Cache.Get(include.DisplayName()))
+	// fmt.Printf("expires: %v\n", workflow.Cache.GetExpiresAt(include.DisplayName()))
+	// fmt.Printf("hash: %v\n", workflow.Cache.GetHash(include.DisplayName()))
 
 	// fmt.Printf("include: %s\n", include.DisplayName())
 	// fmt.Printf("include: %s\n", include.FullUrl())
@@ -428,7 +474,7 @@ func (workflow *StackupWorkflow) ProcessInclude(include *WorkflowInclude) bool {
 	if include.IsRemoteUrl() {
 		if *include.VerifyChecksum == true || include.VerifyChecksum == nil {
 			//support.StatusMessage("Validating checksum for remote include: "+include.DisplayUrl(), false)
-			validated, err := include.ValidateChecksum(contents)
+			validated, err := include.ValidateChecksum(include.Contents)
 
 			if include.ChecksumIsValid != nil && *include.ChecksumIsValid == true {
 				include.ValidationState = "checksum validated"
@@ -488,3 +534,23 @@ func (workflow *StackupWorkflow) ProcessInclude(include *WorkflowInclude) bool {
 
 	return true
 }
+
+func (wi *WorkflowInclude) Initialize(workflow *StackupWorkflow) {
+	wi.Workflow = workflow
+
+	// expand environment variables in the include headers
+	for i, v := range wi.Headers {
+		if App.JsEngine.IsEvaluatableScriptString(v) {
+			wi.Headers[i] = App.JsEngine.Evaluate(v).(string)
+		}
+		wi.Headers[i] = os.ExpandEnv(v)
+	}
+
+	// set some default values
+	if wi.VerifyChecksum == nil {
+		boolValue := true //wi.ChecksumUrl != ""
+		wi.VerifyChecksum = &boolValue
+	}
+	wi.ValidationState = "not validated"
+	wi.ChecksumIsValid = nil
+}