diff --git a/configuration/componentcfg/query.go b/configuration/componentcfg/query.go index 95cab34a..a6d8e233 100644 --- a/configuration/componentcfg/query.go +++ b/configuration/componentcfg/query.go @@ -175,7 +175,7 @@ type QueryParameters struct { func NewQueryParameters(parameters string) (p *QueryParameters, err error) { p = &QueryParameters{ - ProcessTemplates: false, + ProcessTemplates: true, VarStack: make(map[string]string), } parameters = strings.TrimSpace(parameters) diff --git a/configuration/template/dplutil.go b/configuration/template/dplutil.go index 752a8e01..efdf4ef5 100644 --- a/configuration/template/dplutil.go +++ b/configuration/template/dplutil.go @@ -28,6 +28,7 @@ import ( "crypto/sha1" "encoding/hex" "fmt" + "github.com/AliceO2Group/Control/common/logger/infologger" "os" "os/exec" "path/filepath" @@ -70,7 +71,9 @@ func extractConfigURIs(dplCommand string) (uris []string) { // jitDplGenerate takes a resolved dplCommand as an argument, // generates the corresponding tasks and workflow // and returns the resolved dplWorkflow as a string -func jitDplGenerate(confSvc ConfigurationService, varStack map[string]string, workflowRepo repos.IRepo, dplCommand string) (jitWorkflowName string, err error) { +func jitDplGenerate(confSvc ConfigurationService, varStack map[string]string, workflowRepo repos.IRepo, dplCommand string, userWfName string) (jitWorkflowName string, err error) { + envId, _ := varStack["environment_id"] + var payloads []string configURIs := extractConfigURIs(dplCommand) @@ -88,8 +91,14 @@ func jitDplGenerate(confSvc ConfigurationService, varStack map[string]string, wo if err != nil { return "", fmt.Errorf("JIT could not create a query out of path '%s'. error: %w", consulKey[1], err) } - // parse parameters if they are present + queryParams := &componentcfg.QueryParameters{ProcessTemplates: false, VarStack: nil} + // Configuration library adds "process=true" if apricot backend is used and "process" is missing, + // thus using apricot backend with no parameters always implies an expectation to process templates. + if strings.HasPrefix(uri, "apricot") { + queryParams.ProcessTemplates = true + } + // parse parameters if they are present if len(consulKeyTokens) == 2 { queryParams, err = componentcfg.NewQueryParameters(consulKeyTokens[1]) if err != nil { @@ -113,7 +122,9 @@ func jitDplGenerate(confSvc ConfigurationService, varStack map[string]string, wo o2VersionCmd := exec.Command("bash", "-c", "rpm -qa o2-O2 o2-QualityControl") o2VersionOut, err := o2VersionCmd.Output() if err != nil { - log.Warn("JIT couldn't get O2 / QualityControl version: " + err.Error()) + log.WithField("partition", envId). + WithField("level", infologger.IL_Support). + Warn("JIT couldn't get O2 / QualityControl version: " + err.Error()) } // Get the env vars necessary for JIT @@ -139,9 +150,16 @@ func jitDplGenerate(confSvc ConfigurationService, varStack map[string]string, wo // Only generate new tasks & workflows if the files don't exist // If they exist, hash comparison guarantees validity if _, err = os.Stat(filepath.Join(workflowRepo.GetCloneDir(), "workflows", jitWorkflowName+".yaml")); err == nil { - log.Tracef("Workflow '%s' already exists, skipping DPL creation", jitWorkflowName) + log.WithField("partition", envId). + WithField("level", infologger.IL_Support). + WithField("hash", jitWorkflowName). + Infof("Reusing JIT workflow templates for '%s'", userWfName) return jitWorkflowName, nil } + log.WithField("partition", envId). + WithField("level", infologger.IL_Support). + WithField("hash", jitWorkflowName). + Infof("Generating JIT workflow templates for '%s'", userWfName) // TODO: Before executing we need to check that this is a valid dpl command // If not, any command may be injected on the aliecs host @@ -157,17 +175,23 @@ func jitDplGenerate(confSvc ConfigurationService, varStack map[string]string, wo var dplOut []byte dplOut, err = dplCmd.CombinedOutput() if err != nil { - log.Errorf("failed to run DPL command due to error '%s'", err.Error()) - log.Errorf("failed DPL command output: %s", string(dplOut)) + log.WithField("partition", envId). + WithField("level", infologger.IL_Support). + Errorf("failed to run DPL command due to error '%s'", err.Error()) + log.WithField("partition", envId). + WithField("level", infologger.IL_Support). + Errorf("failed DPL command output: %s", string(dplOut)) return "", fmt.Errorf("failed to run DPL command due to error '%w'. See FLP Infologger for DPL command output", err) } else { - log.Trace("DPL command out: " + string(dplOut)) + log.WithField("partition", envId). + WithField("level", infologger.IL_Support). + Trace("DPL command out: " + string(dplOut)) } return jitWorkflowName, nil } -func generateDplSubworkflow(confSvc ConfigurationService, varStack map[string]string, workflowRepo repos.IRepo, dplCommand string) (jitWorkflowName string, err error) { +func generateDplSubworkflow(confSvc ConfigurationService, varStack map[string]string, workflowRepo repos.IRepo, dplCommand string, userWfName string) (jitWorkflowName string, err error) { if dplCommand == "none" { return "", fmt.Errorf("dplCommand is 'none'") } @@ -179,7 +203,7 @@ func generateDplSubworkflow(confSvc ConfigurationService, varStack map[string]st return "", fmt.Errorf("JIT failed in template resolution of the dpl_command: %w", err) } - return jitDplGenerate(confSvc, varStack, workflowRepo, "source /etc/profile.d/o2.sh &&"+dplCommand) + return jitDplGenerate(confSvc, varStack, workflowRepo, "source /etc/profile.d/o2.sh &&"+dplCommand, userWfName) } func generateDplSubworkflowFromUri(confSvc ConfigurationService, varStack map[string]string, workflowRepo repos.IRepo, dplCommandUri string, fallbackToTemplate bool) (jitWorkflowName string, err error) { @@ -193,7 +217,10 @@ func generateDplSubworkflowFromUri(confSvc ConfigurationService, varStack map[st // if a file in JIT is missing, it will try to fallback to a standard workflow template in 'workflows/'. // effectively, this allows us to have an intermediate switch workflow to select different JIT commands // for different nodes. - log.Debugf("JIT: There is no file 'jit/%s' with a DPL command, falling back the template at 'workflows/%s'", dplCommandUri, dplCommandUri) + envId, _ := varStack["environment_id"] + log.WithField("partition", envId). + WithField("level", infologger.IL_Support). + Debugf("JIT: There is no file 'jit/%s' with a DPL command, falling back the template at 'workflows/%s'", dplCommandUri, dplCommandUri) return dplCommandUri, nil } else { return "", fmt.Errorf("Failed to read DPL command from '%s': %w\n", dplCommandUri, err) @@ -207,7 +234,7 @@ func generateDplSubworkflowFromUri(confSvc ConfigurationService, varStack map[st return "", fmt.Errorf("JIT failed in template resolution of the dpl_command: %w", err) } - jitWorkflowName, err = jitDplGenerate(confSvc, varStack, workflowRepo, "source /etc/profile.d/o2.sh && "+dplCommand) + jitWorkflowName, err = jitDplGenerate(confSvc, varStack, workflowRepo, "source /etc/profile.d/o2.sh && "+dplCommand, dplCommandUri) if err != nil { detector := varStack["detector"] return "", fmt.Errorf("for JIT workflow '%s' and detector '%s': %w", dplCommandUri, detector, err) diff --git a/configuration/template/stack.go b/configuration/template/stack.go index 9219dd73..fbbf3a02 100644 --- a/configuration/template/stack.go +++ b/configuration/template/stack.go @@ -151,7 +151,7 @@ func MakeConfigAndRepoAccessFuncs(confSvc ConfigurationService, varStack map[str return map[string]interface{}{ "GenerateDplSubworkflow": func(dplCommand string) (string, error) { log.WithPrefix("template").Warn("GenerateDplSubworkflow is deprecated, use dpl.Generate instead") - return generateDplSubworkflow(confSvc, varStack, workflowRepo, dplCommand) + return generateDplSubworkflow(confSvc, varStack, workflowRepo, dplCommand, "unknown") }, "GenerateDplSubworkflowFromUri": func(dplCommandUri string) (string, error) { log.WithPrefix("template").Warn("GenerateDplSubworkflowFromUri is deprecated, use dpl.GenerateFromUri instead") @@ -159,7 +159,7 @@ func MakeConfigAndRepoAccessFuncs(confSvc ConfigurationService, varStack map[str }, "dpl": map[string]interface{}{ "Generate": func(dplCommand string) (string, error) { - return generateDplSubworkflow(confSvc, varStack, workflowRepo, dplCommand) + return generateDplSubworkflow(confSvc, varStack, workflowRepo, dplCommand, "unknown") }, "GenerateFromUri": func(dplCommandUri string) (string, error) { return generateDplSubworkflowFromUri(confSvc, varStack, workflowRepo, dplCommandUri, false)