feat: task config extrapolation for asset (#178)

* feat: add task config extrapolation for asset * fix: test compile asset after config compilation
goto · Nov 17, 2023 · 491bb08 · 491bb08
1 parent de95831
commit 491bb08
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 15 deletions.
diff --git a/core/scheduler/service/executor_input_compiler.go b/core/scheduler/service/executor_input_compiler.go
@@ -112,16 +112,21 @@ func (i InputCompiler) Compile(ctx context.Context, job *scheduler.JobWithDetail
 		compiler.From(systemDefinedVars).WithName(contextSystemDefined).AddToContext(),
 	)
 
-	// Compile asset files
-	fileMap, err := i.assetCompiler.CompileJobRunAssets(ctx, job.Job, systemDefinedVars, interval, taskContext)
+	confs, secretConfs, err := i.compileConfigs(job.Job.Task.Config, taskContext)
 	if err != nil {
-		i.logger.Error("error compiling job run assets: %s", err)
+		i.logger.Error("error compiling task config: %s", err)
 		return nil, err
 	}
 
-	confs, secretConfs, err := i.compileConfigs(job.Job.Task.Config, taskContext)
+	// Compile asset files
+	allTaskConfigs := compiler.PrepareContext(
+		compiler.From(confs, secretConfs).WithName(contextTask).WithKeyPrefix(taskConfigPrefix),
+	)
+
+	mergedContext := utils.MergeAnyMaps(taskContext, allTaskConfigs)
+	fileMap, err := i.assetCompiler.CompileJobRunAssets(ctx, job.Job, systemDefinedVars, interval, mergedContext)
 	if err != nil {
-		i.logger.Error("error compiling task config: %s", err)
+		i.logger.Error("error compiling job run assets: %s", err)
 		return nil, err
 	}
 
@@ -150,13 +155,6 @@ func (i InputCompiler) Compile(ctx context.Context, job *scheduler.JobWithDetail
 		}, nil
 	}
 
-	// If request for hook, add task configs to templateContext
-	hookContext := compiler.PrepareContext(
-		compiler.From(confs, secretConfs).WithName(contextTask).WithKeyPrefix(taskConfigPrefix),
-	)
-
-	mergedContext := utils.MergeAnyMaps(taskContext, hookContext)
-
 	hook, err := job.Job.GetHook(config.Executor.Name)
 	if err != nil {
 		i.logger.Error("error getting hook [%s]: %s", config.Executor.Name, err)

diff --git a/core/scheduler/service/executor_input_compiler_test.go b/core/scheduler/service/executor_input_compiler_test.go
@@ -111,6 +111,10 @@ func TestExecutorCompiler(t *testing.T) {
 				Tenant:       tnnt,
 				WindowConfig: cw,
 				Assets:       nil,
+				Task: &scheduler.Task{
+					Name:   "bq2bq",
+					Config: map[string]string{},
+				},
 			}
 			details := scheduler.JobWithDetails{
 				Job: &job,
@@ -142,11 +146,14 @@ func TestExecutorCompiler(t *testing.T) {
 			}
 			taskContext := mock.Anything
 
+			templateCompiler := new(mockTemplateCompiler)
+			templateCompiler.On("Compile", mock.Anything, taskContext).Return(map[string]string{}, nil)
+			defer templateCompiler.AssertExpectations(t)
 			assetCompiler := new(mockAssetCompiler)
 			assetCompiler.On("CompileJobRunAssets", ctx, &job, systemDefinedVars, interval, taskContext).Return(nil, fmt.Errorf("CompileJobRunAssets error"))
 			defer assetCompiler.AssertExpectations(t)
 
-			inputCompiler := service.NewJobInputCompiler(tenantService, nil, assetCompiler, logger)
+			inputCompiler := service.NewJobInputCompiler(tenantService, templateCompiler, assetCompiler, logger)
 			inputExecutor, err := inputCompiler.Compile(ctx, &details, config, executedAt)
 
 			assert.NotNil(t, err)
@@ -212,7 +219,6 @@ func TestExecutorCompiler(t *testing.T) {
 					Return(nil, fmt.Errorf("some.config compilation error"))
 				defer templateCompiler.AssertExpectations(t)
 				assetCompiler := new(mockAssetCompiler)
-				assetCompiler.On("CompileJobRunAssets", ctx, &job, systemDefinedVars, interval, taskContext).Return(compiledFile, nil)
 				defer assetCompiler.AssertExpectations(t)
 				inputCompiler := service.NewJobInputCompiler(tenantService, templateCompiler, assetCompiler, logger)
 				inputExecutor, err := inputCompiler.Compile(ctx, &details, config, executedAt)
@@ -229,7 +235,6 @@ func TestExecutorCompiler(t *testing.T) {
 					Return(nil, fmt.Errorf("secret.config compilation error"))
 				defer templateCompiler.AssertExpectations(t)
 				assetCompiler := new(mockAssetCompiler)
-				assetCompiler.On("CompileJobRunAssets", ctx, &job, systemDefinedVars, interval, taskContext).Return(compiledFile, nil)
 				defer assetCompiler.AssertExpectations(t)
 				inputCompiler := service.NewJobInputCompiler(tenantService, templateCompiler, assetCompiler, logger)
 				inputExecutor, err := inputCompiler.Compile(ctx, &details, config, executedAt)