diff --git a/client/cmd/job/validate.go b/client/cmd/job/validate.go index 3e75451022..36c10a6d72 100644 --- a/client/cmd/job/validate.go +++ b/client/cmd/job/validate.go @@ -4,12 +4,10 @@ import ( "context" "errors" "fmt" - "os" - "strings" + "io" "time" "github.com/goto/salt/log" - "github.com/olekukonko/tablewriter" "github.com/spf13/afero" "github.com/spf13/cobra" @@ -25,17 +23,13 @@ const validateTimeout = time.Minute * 15 type validateCommand struct { logger log.Logger - connection connection.Connection + connection *connection.Insecure configFilePath string - namespaceName string - jobNames []string + clientConfig *config.ClientConfig - fromServer bool - forDelete bool - verbose bool - - clientConfig *config.ClientConfig + verbose bool + namespaceName string } // NewValidateCommand initializes command for validating job specification @@ -45,319 +39,109 @@ func NewValidateCommand() *cobra.Command { } cmd := &cobra.Command{ - Use: "validate", - Short: "Execute validation on the selected jobs", - Long: "Validation is executed on the selected jobs. Each job will be validated against a sequence of criterions.", - Example: `... --namespace # upload all jobs within namespace to be validated -... --jobs -n # upload the selected jobs within namespace to be validated -... -j -n --from-server # validate existing jobs in the server, no upload will be done -... -j -n -s --for-delete # validation is for delete purpose, no actual deletion`, + Use: "validate", + Short: "Run basic checks on all jobs", + Long: "Check if specifications are valid for deployment", + Example: "optimus job validate", RunE: validate.RunE, PreRunE: validate.PreRunE, } - + // Config filepath flag cmd.Flags().StringVarP(&validate.configFilePath, "config", "c", config.EmptyPath, "File path for client configuration") - cmd.Flags().StringVarP(&validate.namespaceName, "namespace", "n", validate.namespaceName, "Namespace name in which the job resides") - cmd.Flags().StringSliceVarP(&validate.jobNames, "jobs", "j", nil, "Selected job names, comma separated without any whitespace if more than one") - - cmd.Flags().BoolVarP(&validate.fromServer, "from-server", "s", false, "Determines whether to upload jobs from local or to use existing from the server") - cmd.Flags().BoolVarP(&validate.forDelete, "for-delete", "d", false, "Determines whether the validation is for delete purpose or not, only valid with from-server flag") - cmd.Flags().BoolVarP(&validate.verbose, "verbose", "v", false, "Determines whether to show the complete message or just the summary") + cmd.Flags().BoolVarP(&validate.verbose, "verbose", "v", false, "Print details related to operation") + cmd.Flags().StringVarP(&validate.namespaceName, "namespace", "n", validate.namespaceName, "Namespace of the resource within project") cmd.MarkFlagRequired("namespace") - return cmd } -func (v *validateCommand) PreRunE(_ *cobra.Command, _ []string) error { +func (v *validateCommand) PreRunE(_ *cobra.Command, _ []string) error { // Load mandatory config conf, err := config.LoadClientConfig(v.configFilePath) if err != nil { return err } v.clientConfig = conf - v.connection = connection.New(v.logger, conf) + v.connection = connection.NewInsecure(v.logger) return nil } func (v *validateCommand) RunE(_ *cobra.Command, _ []string) error { - if v.forDelete && !v.fromServer { - return errors.New("for-delete flag is only valid with from-server flag") - } - - if v.fromServer && len(v.jobNames) == 0 { - return errors.New("from-server flag is only valid with jobs flag being set") - } - namespace, err := v.clientConfig.GetNamespaceByName(v.namespaceName) if err != nil { return err } - v.logger.Info("Validating job specifications for namespace [%s]\n", v.namespaceName) - start := time.Now() - - if err := v.executeLocalValidation(); err != nil { - v.logger.Info("Validation is finished, took %s", time.Since(start).Round(time.Second)) - return err - } - - if err := v.executeServerValidation(namespace); err != nil { - v.logger.Info("Validation is finished, took %s", time.Since(start).Round(time.Second)) - return err - } - - v.logger.Info("Validation is finished, took %s", time.Since(start).Round(time.Second)) - return nil -} - -func (v *validateCommand) executeLocalValidation() error { - jobSpecReadWriter, err := specio.NewJobSpecReadWriter(afero.NewOsFs()) - if err != nil { - return err - } - - namespaceNamesByJobName := make(map[string][]string) - for _, namespace := range v.clientConfig.Namespaces { - jobs, err := jobSpecReadWriter.ReadAll(namespace.Job.Path) - if err != nil { - return err - } - - for _, j := range jobs { - namespaceNamesByJobName[j.Name] = append(namespaceNamesByJobName[j.Name], namespace.Name) - } - } - - success := true - for jobName, namespaceNames := range namespaceNamesByJobName { - if len(namespaceNames) == 1 { - continue - } - - uniqueNames := v.deduplicate(namespaceNames) - v.logger.Error("[%s] is written [%d] times in namespace [%s]", jobName, len(namespaceNames), strings.Join(uniqueNames, ", ")) - success = false - } - - if !success { - return errors.New("local duplication is detected") - } - - return nil -} - -func (*validateCommand) deduplicate(input []string) []string { - var output []string - - tmp := make(map[string]bool) - for _, s := range input { - if !tmp[s] { - tmp[s] = true - output = append(output, s) - } - } - - return output -} - -func (v *validateCommand) executeServerValidation(namespace *config.Namespace) error { - var ( - request *pb.ValidateRequest - err error - ) - - if v.fromServer { - request = v.getRequestForJobNames() - } else { - request, err = v.getRequestForJobSpecs(namespace) - } - + jobSpecReadWriter, err := specio.NewJobSpecReadWriter(afero.NewOsFs(), specio.WithJobSpecParentReading()) if err != nil { return err } - success := true - - response, err := v.executeValidation(request) + start := time.Now() + projectName := v.clientConfig.Project.Name + v.logger.Info("Validating job specifications for project: %s, namespace: %s", projectName, namespace.Name) + jobSpecs, err := jobSpecReadWriter.ReadAll(namespace.Job.Path) if err != nil { - v.logger.Error("error returned when executing validation: %v", err) - success = false - } - - if err := v.processResponse(response); err != nil { - v.logger.Error("error returned when processing response: %v", err) - success = false - } - - if !success { - return errors.New("encountered one or more errors") - } - - return nil -} - -func (v *validateCommand) processResponse(response *pb.ValidateResponse) error { - if response == nil { - return nil - } - - if v.verbose { - return v.printCompleteResponse(response) + return fmt.Errorf("directory '%s': %w", namespace.Job.Path, err) } - - return v.printBriefResponse(response) -} - -func (v *validateCommand) printBriefResponse(response *pb.ValidateResponse) error { - resultsByJobName := response.GetResultsByJobName() - - var successJobsCount int - for jobName, rawResult := range resultsByJobName { - var successResultCount int - - v.logger.Info("[%s]", jobName) - result := rawResult.GetResults() - for _, r := range result { - if r.Success { - successResultCount++ - } else { - v.logger.Info(" %s", r.GetName()) - for _, m := range r.GetMessages() { - v.logger.Error(" %s", m) - } - } - } - - if successResultCount == len(result) { - successJobsCount++ - } - - v.logger.Info("[%s] pass %d of %d validations", jobName, successResultCount, len(result)) - } - - if successJobsCount != len(resultsByJobName) { - return errors.New("validation encountered errors") - } - - return nil -} - -func (v *validateCommand) printCompleteResponse(response *pb.ValidateResponse) error { - resultsByJobName := response.GetResultsByJobName() - - var successJobsCount int - for jobName, rawResult := range resultsByJobName { - v.logger.Info("[%s]", jobName) - - table := tablewriter.NewWriter(os.Stdout) - table.SetAutoMergeCellsByColumnIndex([]int{0, 1}) - table.SetRowLine(true) - - table.SetHeader([]string{"NAME", "SUCCESS", "MESSAGE"}) - - var successResultCount int - - result := rawResult.GetResults() - for _, r := range result { - if r.Success { - successResultCount++ - } - - for i := 0; i < len(r.Messages); i++ { - table.Append([]string{r.Name, strings.ToUpper(fmt.Sprintf("%t", r.Success)), r.Messages[i]}) - } - } - - if successResultCount == len(result) { - successJobsCount++ - } - - table.SetFooter([]string{"", "", fmt.Sprintf("passed %d of %d validations", successResultCount, len(result))}) - table.SetFooterAlignment(tablewriter.ALIGN_RIGHT) - - table.Render() - - v.logger.Info("\n") - } - - if successJobsCount != len(resultsByJobName) { - return errors.New("validation encountered errors") + if err := v.validateJobSpecificationRequest(jobSpecs); err != nil { + return err } - + v.logger.Info("Jobs validated successfully, took %s", time.Since(start).Round(time.Second)) return nil } -func (v *validateCommand) executeValidation(request *pb.ValidateRequest) (*pb.ValidateResponse, error) { +func (v *validateCommand) validateJobSpecificationRequest(jobSpecs []*model.JobSpec) error { conn, err := v.connection.Create(v.clientConfig.Host) if err != nil { - return nil, err + return err } defer conn.Close() - client := pb.NewJobSpecificationServiceClient(conn) - - ctx, dialCancel := context.WithTimeout(context.Background(), validateTimeout) - defer dialCancel() - - return client.Validate(ctx, request) -} + jobSpecsProto := []*pb.JobSpecification{} + for _, jobSpec := range jobSpecs { + jobSpecsProto = append(jobSpecsProto, jobSpec.ToProto()) + } -func (v *validateCommand) getRequestForJobNames() *pb.ValidateRequest { - return &pb.ValidateRequest{ + checkJobSpecRequest := &pb.CheckJobSpecificationsRequest{ ProjectName: v.clientConfig.Project.Name, + Jobs: jobSpecsProto, NamespaceName: v.namespaceName, - Payload: &pb.ValidateRequest_FromServer_{ - FromServer: &pb.ValidateRequest_FromServer{ - JobNames: v.jobNames, - DeletionMode: v.forDelete, - }, - }, } -} + job := pb.NewJobSpecificationServiceClient(conn) -func (v *validateCommand) getRequestForJobSpecs(namespace *config.Namespace) (*pb.ValidateRequest, error) { - jobSpecReadWriter, err := specio.NewJobSpecReadWriter(afero.NewOsFs(), specio.WithJobSpecParentReading()) - if err != nil { - return nil, err - } + ctx, dialCancel := context.WithTimeout(context.Background(), validateTimeout) + defer dialCancel() - jobSpecs, err := jobSpecReadWriter.ReadAll(namespace.Job.Path) + respStream, err := job.CheckJobSpecifications(ctx, checkJobSpecRequest) if err != nil { - return nil, err + if errors.Is(err, context.DeadlineExceeded) { + v.logger.Error("Validate process took too long, timing out") + } + return fmt.Errorf("validate request failed: %w", err) } + return v.getCheckJobSpecificationsResponse(respStream) +} - if len(v.jobNames) > 0 { - specByName := make(map[string]*model.JobSpec) - for _, spec := range jobSpecs { - specByName[spec.Name] = spec +func (v *validateCommand) getCheckJobSpecificationsResponse(stream pb.JobSpecificationService_CheckJobSpecificationsClient) error { + for { + resp, err := stream.Recv() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return err } - var specsToValidate []*model.JobSpec - for _, name := range v.jobNames { - spec, ok := specByName[name] - if !ok { - return nil, fmt.Errorf("spec for job [%s] is not found in local namespace [%s]", name, v.namespaceName) + if logStatus := resp.GetLogStatus(); logStatus != nil { + if v.verbose { + logger.PrintLogStatusVerbose(v.logger, logStatus) + } else { + logger.PrintLogStatus(v.logger, logStatus) } - - specsToValidate = append(specsToValidate, spec) + continue } - - jobSpecs = specsToValidate } - jobSpecsProto := make([]*pb.JobSpecification, len(jobSpecs)) - for i, jobSpec := range jobSpecs { - jobSpecsProto[i] = jobSpec.ToProto() - } - - return &pb.ValidateRequest{ - ProjectName: v.clientConfig.Project.Name, - NamespaceName: namespace.Name, - Payload: &pb.ValidateRequest_FromOutside_{ - FromOutside: &pb.ValidateRequest_FromOutside{ - Jobs: jobSpecsProto, - }, - }, - }, nil + return nil } diff --git a/core/job/dto/stage.go b/core/job/dto/stage.go deleted file mode 100644 index f2535f140e..0000000000 --- a/core/job/dto/stage.go +++ /dev/null @@ -1,19 +0,0 @@ -package dto - -const ( - StageCyclicValidation ValidateStage = "cyclic validation" - StageDeletionValidation ValidateStage = "validation for deletion" - StageDestinationValidation ValidateStage = "destination validation" - StagePreparation ValidateStage = "validation preparation" - StageRunCompileValidation ValidateStage = "compile validation for run" - StageSourceValidation ValidateStage = "source validation" - StageTenantValidation ValidateStage = "tenant validation" - StageUpstreamValidation ValidateStage = "upstream validation" - StageWindowValidation ValidateStage = "window validation" -) - -type ValidateStage string - -func (v ValidateStage) String() string { - return string(v) -} diff --git a/core/job/dto/validation.go b/core/job/dto/validation.go deleted file mode 100644 index b5d7127af4..0000000000 --- a/core/job/dto/validation.go +++ /dev/null @@ -1,19 +0,0 @@ -package dto - -import ( - "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/tenant" -) - -type ValidateRequest struct { - Tenant tenant.Tenant - JobSpecs []*job.Spec - JobNames []string - DeletionMode bool -} - -type ValidateResult struct { - Stage ValidateStage - Messages []string - Success bool -} diff --git a/core/job/handler/v1beta1/job.go b/core/job/handler/v1beta1/job.go index 8320b5baf3..6c4f5a7e91 100644 --- a/core/job/handler/v1beta1/job.go +++ b/core/job/handler/v1beta1/job.go @@ -11,7 +11,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/job/dto" "github.com/goto/optimus/core/job/service/filter" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/errors" @@ -25,6 +24,7 @@ import ( const ( metricReplaceAllDuration = "job_replace_all_duration_seconds" metricRefreshDuration = "job_refresh_duration_seconds" + metricValidationDuration = "job_validation_duration_seconds" ) type JobHandler struct { @@ -53,7 +53,7 @@ type JobService interface { GetByFilter(ctx context.Context, filters ...filter.FilterOpt) (jobSpecs []*job.Job, err error) ReplaceAll(ctx context.Context, jobTenant tenant.Tenant, jobs []*job.Spec, jobNamesWithInvalidSpec []job.Name, logWriter writer.LogWriter) error Refresh(ctx context.Context, projectName tenant.ProjectName, namespaceNames, jobNames []string, logWriter writer.LogWriter) error - Validate(context.Context, dto.ValidateRequest) (map[job.Name][]dto.ValidateResult, error) + Validate(ctx context.Context, jobTenant tenant.Tenant, jobSpecs []*job.Spec, jobNamesWithInvalidSpec []job.Name, logWriter writer.LogWriter) error GetJobBasicInfo(ctx context.Context, jobTenant tenant.Tenant, jobName job.Name, spec *job.Spec) (*job.Job, writer.BufferedLogger) GetUpstreamsToInspect(ctx context.Context, subjectJob *job.Job, localJob bool) ([]*job.Upstream, error) @@ -419,42 +419,35 @@ func (jh *JobHandler) RefreshJobs(request *pb.RefreshJobsRequest, stream pb.JobS return nil } -// Deprecated: Do not use. -func (*JobHandler) CheckJobSpecifications(_ *pb.CheckJobSpecificationsRequest, _ pb.JobSpecificationService_CheckJobSpecificationsServer) error { - panic("deprecated, use validate endpoint instead") -} +func (jh *JobHandler) CheckJobSpecifications(req *pb.CheckJobSpecificationsRequest, stream pb.JobSpecificationService_CheckJobSpecificationsServer) error { + startTime := time.Now() -func (jh *JobHandler) Validate(ctx context.Context, request *pb.ValidateRequest) (*pb.ValidateResponse, error) { - tnnt, err := tenant.NewTenant(request.GetProjectName(), request.GetNamespaceName()) + responseWriter := writer.NewCheckJobSpecificationResponseWriter(stream) + jobTenant, err := tenant.NewTenant(req.ProjectName, req.NamespaceName) if err != nil { - return nil, err + jh.l.Error("invalid tenant information request project [%s] namespace [%s]: %s", req.GetProjectName(), req.GetNamespaceName(), err) + return err } - jobSpecs := make([]*job.Spec, len(request.GetFromOutside().GetJobs())) - for i, job := range request.GetFromOutside().GetJobs() { - spec, err := fromJobProto(job) - if err != nil { - return nil, err - } - - jobSpecs[i] = spec + me := errors.NewMultiError("check / validate job spec errors") + jobSpecs, jobNamesWithInvalidSpec, err := fromJobProtos(req.Jobs) + if err != nil { + jh.l.Error("error when adapting job specifications: %s", err) + me.Append(err) } - validateRequest := dto.ValidateRequest{ - Tenant: tnnt, - JobSpecs: jobSpecs, - JobNames: request.GetFromServer().GetJobNames(), - DeletionMode: request.GetFromServer().GetDeletionMode(), + if err := jh.jobService.Validate(stream.Context(), jobTenant, jobSpecs, jobNamesWithInvalidSpec, responseWriter); err != nil { + jh.l.Error("error validating job: %s", err) + me.Append(err) } - result, err := jh.jobService.Validate(ctx, validateRequest) - if err != nil { - return nil, err - } + processDuration := time.Since(startTime) + telemetry.NewGauge(metricValidationDuration, map[string]string{ + "project": jobTenant.ProjectName().String(), + "namespace": jobTenant.NamespaceName().String(), + }).Add(processDuration.Seconds()) - return &pb.ValidateResponse{ - ResultsByJobName: toValidateResultProto(result), - }, nil + return me.ToErr() } func (jh *JobHandler) GetJobTask(ctx context.Context, req *pb.GetJobTaskRequest) (*pb.GetJobTaskResponse, error) { @@ -645,23 +638,3 @@ func raiseJobEventMetric(jobTenant tenant.Tenant, state string, metricValue int) "status": state, }).Add(float64(metricValue)) } - -func toValidateResultProto(result map[job.Name][]dto.ValidateResult) map[string]*pb.ValidateResponse_ResultList { - output := make(map[string]*pb.ValidateResponse_ResultList) - for jobName, validateResults := range result { - resultsProto := make([]*pb.ValidateResponse_Result, len(validateResults)) - for i, rst := range validateResults { - resultsProto[i] = &pb.ValidateResponse_Result{ - Name: rst.Stage.String(), - Messages: rst.Messages, - Success: rst.Success, - } - } - - output[jobName.String()] = &pb.ValidateResponse_ResultList{ - Results: resultsProto, - } - } - - return output -} diff --git a/core/job/handler/v1beta1/job_adapter.go b/core/job/handler/v1beta1/job_adapter.go index 24969068f5..9ac5c2f33f 100644 --- a/core/job/handler/v1beta1/job_adapter.go +++ b/core/job/handler/v1beta1/job_adapter.go @@ -6,7 +6,6 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/internal/errors" "github.com/goto/optimus/internal/lib/labels" "github.com/goto/optimus/internal/lib/window" @@ -200,7 +199,7 @@ func fromJobProto(js *pb.JobSpecification) (*job.Spec, error) { return jobSpecBuilder.Build() } -func fromResourceURNs(resourceURNs []resource.URN) []string { +func fromResourceURNs(resourceURNs []job.ResourceURN) []string { var resources []string for _, resourceURN := range resourceURNs { resources = append(resources, resourceURN.String()) diff --git a/core/job/handler/v1beta1/job_test.go b/core/job/handler/v1beta1/job_test.go index 21a6cc60db..feeae31aab 100644 --- a/core/job/handler/v1beta1/job_test.go +++ b/core/job/handler/v1beta1/job_test.go @@ -14,10 +14,8 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/job/dto" "github.com/goto/optimus/core/job/handler/v1beta1" "github.com/goto/optimus/core/job/service/filter" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/lib/window" "github.com/goto/optimus/internal/models" @@ -78,15 +76,6 @@ func TestNewJobHandler(t *testing.T) { log := log.NewNoop() sampleOwner := "sample-owner" - resourceURNA, err := resource.ParseURN("store://table-A") - assert.NoError(t, err) - resourceURNB, err := resource.ParseURN("store://table-B") - assert.NoError(t, err) - resourceURNC, err := resource.ParseURN("store://table-C") - assert.NoError(t, err) - resourceURND, err := resource.ParseURN("store://table-D") - assert.NoError(t, err) - t.Run("AddJobSpecifications", func(t *testing.T) { t.Run("adds job", func(t *testing.T) { jobService := new(JobService) @@ -1271,7 +1260,7 @@ func TestNewJobHandler(t *testing.T) { defer jobService.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"table-B"}, false) request := pb.GetJobSpecificationRequest{ ProjectName: sampleTenant.ProjectName().String(), @@ -1310,9 +1299,9 @@ func TestNewJobHandler(t *testing.T) { request := pb.GetJobSpecificationsRequest{} specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithAsset(asset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"table-B"}, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", sampleOwner, jobSchedule, jobWindow, jobTask).WithAsset(asset).Build() - jobB := job.NewJob(sampleTenant, specB, resourceURNB, []resource.URN{resourceURNC}, false) + jobB := job.NewJob(sampleTenant, specB, "table-B", []job.ResourceURN{"table-C"}, false) jobService.On("GetByFilter", ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]*job.Job{jobA, jobB}, nil) jobHandler := v1beta1.NewJobHandler(jobService, log) @@ -1333,9 +1322,9 @@ func TestNewJobHandler(t *testing.T) { } specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithAsset(asset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"table-B"}, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", sampleOwner, jobSchedule, jobWindow, jobTask).WithAsset(asset).Build() - jobB := job.NewJob(sampleTenant, specB, resourceURNB, []resource.URN{resourceURNC}, false) + jobB := job.NewJob(sampleTenant, specB, "table-B", []job.ResourceURN{"table-C"}, false) jobService.On("GetByFilter", ctx, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]*job.Job{jobA, jobB}, nil) jobHandler := v1beta1.NewJobHandler(jobService, log) @@ -1375,9 +1364,9 @@ func TestNewJobHandler(t *testing.T) { } specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithAsset(asset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"table-B"}, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", sampleOwner, jobSchedule, jobWindow, jobTask).WithAsset(asset).Build() - jobB := job.NewJob(sampleTenant, specB, resourceURNB, []resource.URN{resourceURNC}, false) + jobB := job.NewJob(sampleTenant, specB, "table-B", []job.ResourceURN{"table-C"}, false) jobService.On("GetByFilter", ctx, mock.Anything, mock.Anything).Return([]*job.Job{jobA, jobB}, nil) jobHandler := v1beta1.NewJobHandler(jobService, log) @@ -1400,9 +1389,9 @@ func TestNewJobHandler(t *testing.T) { } specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithAsset(asset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"table-B"}, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", sampleOwner, jobSchedule, jobWindow, jobTask).WithAsset(asset).Build() - jobB := job.NewJob(sampleTenant, specB, resourceURNB, []resource.URN{resourceURNC}, false) + jobB := job.NewJob(sampleTenant, specB, "table-B", []job.ResourceURN{"table-C"}, false) jobService.On("GetByFilter", ctx, mock.Anything, mock.Anything).Return([]*job.Job{jobA, jobB}, nil) jobHandler := v1beta1.NewJobHandler(jobService, log) @@ -1415,6 +1404,126 @@ func TestNewJobHandler(t *testing.T) { assert.Empty(t, resp.Jobs[1].Assets) }) }) + t.Run("CheckJobSpecifications", func(t *testing.T) { + t.Run("return error when creating tenant failed", func(t *testing.T) { + jobService := new(JobService) + defer jobService.AssertExpectations(t) + + stream := new(CheckJobSpecificationsServer) + defer stream.AssertExpectations(t) + + request := &pb.CheckJobSpecificationsRequest{} + + jobHandler := v1beta1.NewJobHandler(jobService, log) + err := jobHandler.CheckJobSpecifications(request, stream) + assert.Error(t, err) + assert.Equal(t, "invalid argument for entity project: project name is empty", err.Error()) + }) + t.Run("return error when job proto conversion failed", func(t *testing.T) { + jobService := new(JobService) + defer jobService.AssertExpectations(t) + + stream := new(CheckJobSpecificationsServer) + defer stream.AssertExpectations(t) + + jobSpecProto := &pb.JobSpecification{ + Version: int32(jobVersion), + Name: "", + Owner: sampleOwner, + StartDate: jobSchedule.StartDate().String(), + EndDate: jobSchedule.EndDate().String(), + Interval: jobSchedule.Interval(), + TaskName: jobTask.Name().String(), + WindowSize: jobWindow.GetSize(), + WindowOffset: jobWindow.GetOffset(), + WindowTruncateTo: jobWindow.GetTruncateTo(), + } + jobProtos := []*pb.JobSpecification{jobSpecProto} + + stream.On("Context").Return(ctx) + jobService.On("Validate", ctx, sampleTenant, mock.Anything, mock.Anything).Return(nil) + + request := &pb.CheckJobSpecificationsRequest{ + ProjectName: sampleTenant.ProjectName().String(), + NamespaceName: sampleTenant.NamespaceName().String(), + Jobs: jobProtos, + } + + jobHandler := v1beta1.NewJobHandler(jobService, log) + err := jobHandler.CheckJobSpecifications(request, stream) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid argument for entity job: name is empty") + }) + t.Run("return error when service validate job is failed", func(t *testing.T) { + jobService := new(JobService) + defer jobService.AssertExpectations(t) + + stream := new(CheckJobSpecificationsServer) + defer stream.AssertExpectations(t) + + jobSpecProto := &pb.JobSpecification{ + Version: int32(jobVersion), + Name: "job-A", + Owner: sampleOwner, + StartDate: jobSchedule.StartDate().String(), + EndDate: jobSchedule.EndDate().String(), + Interval: jobSchedule.Interval(), + TaskName: jobTask.Name().String(), + WindowSize: jobWindow.GetSize(), + WindowOffset: jobWindow.GetOffset(), + WindowTruncateTo: jobWindow.GetTruncateTo(), + } + jobProtos := []*pb.JobSpecification{jobSpecProto} + + stream.On("Context").Return(ctx) + jobService.On("Validate", ctx, sampleTenant, mock.Anything, mock.Anything).Return(errors.New("error encountered")) + + request := &pb.CheckJobSpecificationsRequest{ + ProjectName: sampleTenant.ProjectName().String(), + NamespaceName: sampleTenant.NamespaceName().String(), + Jobs: jobProtos, + } + + jobHandler := v1beta1.NewJobHandler(jobService, log) + err := jobHandler.CheckJobSpecifications(request, stream) + assert.Error(t, err) + assert.Contains(t, err.Error(), "error encountered") + }) + t.Run("return success", func(t *testing.T) { + jobService := new(JobService) + defer jobService.AssertExpectations(t) + + stream := new(CheckJobSpecificationsServer) + defer stream.AssertExpectations(t) + + jobSpecProto := &pb.JobSpecification{ + Version: int32(jobVersion), + Name: "job-A", + Owner: sampleOwner, + StartDate: jobSchedule.StartDate().String(), + EndDate: jobSchedule.EndDate().String(), + Interval: jobSchedule.Interval(), + TaskName: jobTask.Name().String(), + WindowSize: jobWindow.GetSize(), + WindowOffset: jobWindow.GetOffset(), + WindowTruncateTo: jobWindow.GetTruncateTo(), + } + jobProtos := []*pb.JobSpecification{jobSpecProto} + + stream.On("Context").Return(ctx) + jobService.On("Validate", ctx, sampleTenant, mock.Anything, mock.Anything).Return(nil) + + request := &pb.CheckJobSpecificationsRequest{ + ProjectName: sampleTenant.ProjectName().String(), + NamespaceName: sampleTenant.NamespaceName().String(), + Jobs: jobProtos, + } + + jobHandler := v1beta1.NewJobHandler(jobService, log) + err := jobHandler.CheckJobSpecifications(request, stream) + assert.NoError(t, err) + }) + }) t.Run("JobInspect", func(t *testing.T) { configs := []*pb.JobConfigItem{ { @@ -1428,10 +1537,10 @@ func TestNewJobHandler(t *testing.T) { httpUpstream, _ := job.NewSpecHTTPUpstreamBuilder("sample-upstream", "sample-url").Build() upstreamSpec, _ := job.NewSpecUpstreamBuilder().WithSpecHTTPUpstream([]*job.SpecHTTPUpstream{httpUpstream}).Build() specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).WithMetadata(metadataSpec).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, nil, false) + jobA := job.NewJob(sampleTenant, specA, "resource-A", nil, false) - upstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", "bq2bq", false) - upstreamC := job.NewUpstreamResolved("job-C", "other-host", resourceURNC, sampleTenant, "inferred", "bq2bq", true) + upstreamB := job.NewUpstreamResolved("job-B", "", "resource-b", sampleTenant, "static", "bq2bq", false) + upstreamC := job.NewUpstreamResolved("job-C", "other-host", "resource-c", sampleTenant, "inferred", "bq2bq", true) jobAUpstream := []*job.Upstream{ upstreamB, @@ -1467,7 +1576,7 @@ func TestNewJobHandler(t *testing.T) { WindowSize: specA.WindowConfig().GetSize(), WindowOffset: specA.WindowConfig().GetOffset(), WindowTruncateTo: specA.WindowConfig().GetTruncateTo(), - Destination: "store://table-A", + Destination: "resource-A", Config: []*pb.JobConfigItem{{ Name: "sample_key", Value: "sample_value", @@ -1482,7 +1591,7 @@ func TestNewJobHandler(t *testing.T) { }, Metadata: jobMetadata, }, - Destination: "store://table-A", + Destination: "resource-A", }, Upstreams: &pb.JobInspectResponse_UpstreamSection{ ExternalDependency: []*pb.JobInspectResponse_JobDependency{ @@ -1540,11 +1649,11 @@ func TestNewJobHandler(t *testing.T) { specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask). WithSpecUpstream(upstreamSpec). WithHooks([]*job.Hook{hook1}).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, nil, false) + jobA := job.NewJob(sampleTenant, specA, "resource-A", nil, false) - upstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", "bq2bq", false) - upstreamC := job.NewUpstreamResolved("job-C", "other-host", resourceURNC, sampleTenant, "inferred", "bq2bq", true) - upstreamD := job.NewUpstreamUnresolvedInferred(resourceURND) + upstreamB := job.NewUpstreamResolved("job-B", "", "resource-b", sampleTenant, "static", "bq2bq", false) + upstreamC := job.NewUpstreamResolved("job-C", "other-host", "resource-c", sampleTenant, "inferred", "bq2bq", true) + upstreamD := job.NewUpstreamUnresolvedInferred("resource-d") upstreamE := job.NewUpstreamUnresolvedStatic("job-e", project.Name()) jobAUpstream := []*job.Upstream{ @@ -1607,12 +1716,12 @@ func TestNewJobHandler(t *testing.T) { WindowSize: specA.WindowConfig().GetSize(), WindowOffset: specA.WindowConfig().GetOffset(), WindowTruncateTo: specA.WindowConfig().GetTruncateTo(), - Destination: "store://table-A", + Destination: "resource-A", Config: configs, Dependencies: jobDependenciesWithHTTPProto, Hooks: jobHooksProto, }, - Destination: "store://table-A", + Destination: "resource-A", }, Upstreams: &pb.JobInspectResponse_UpstreamSection{ ExternalDependency: []*pb.JobInspectResponse_JobDependency{ @@ -1710,10 +1819,10 @@ func TestNewJobHandler(t *testing.T) { jobService := new(JobService) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, nil, false) + jobA := job.NewJob(sampleTenant, specA, "resource-A", nil, false) - upstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", "bq2bq", false) - upstreamC := job.NewUpstreamResolved("job-C", "other-host", resourceURNC, sampleTenant, "inferred", "bq2bq", true) + upstreamB := job.NewUpstreamResolved("job-B", "", "resource-b", sampleTenant, "static", "bq2bq", false) + upstreamC := job.NewUpstreamResolved("job-C", "other-host", "resource-c", sampleTenant, "inferred", "bq2bq", true) jobAUpstream := []*job.Upstream{ upstreamB, @@ -1753,13 +1862,13 @@ func TestNewJobHandler(t *testing.T) { WindowSize: specA.WindowConfig().GetSize(), WindowOffset: specA.WindowConfig().GetOffset(), WindowTruncateTo: specA.WindowConfig().GetTruncateTo(), - Destination: "store://table-A", + Destination: "resource-A", Config: []*pb.JobConfigItem{{ Name: "sample_key", Value: "sample_value", }}, }, - Destination: "store://table-A", + Destination: "resource-A", }, Upstreams: &pb.JobInspectResponse_UpstreamSection{ ExternalDependency: []*pb.JobInspectResponse_JobDependency{ @@ -1873,7 +1982,7 @@ func TestNewJobHandler(t *testing.T) { defer jobService.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"table-B"}, false) req := &pb.GetJobTaskRequest{ ProjectName: sampleTenant.ProjectName().String(), @@ -1893,7 +2002,7 @@ func TestNewJobHandler(t *testing.T) { defer jobService.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"table-B"}, false) req := &pb.GetJobTaskRequest{ ProjectName: sampleTenant.ProjectName().String(), @@ -2164,25 +2273,18 @@ func (_m *JobService) Update(ctx context.Context, jobTenant tenant.Tenant, jobs return r0 } -// Validate provides a mock function with given fields: ctx, request -func (_m *JobService) Validate(ctx context.Context, request dto.ValidateRequest) (map[job.Name][]dto.ValidateResult, error) { - ret := _m.Called(ctx, request) +// Validate provides a mock function with given fields: ctx, jobTenant, jobSpecs, logWriter +func (_m *JobService) Validate(ctx context.Context, jobTenant tenant.Tenant, jobSpecs []*job.Spec, jobNamesWithInvalidSpec []job.Name, logWriter writer.LogWriter) error { + ret := _m.Called(ctx, jobTenant, jobSpecs, logWriter) - var r0 map[job.Name][]dto.ValidateResult - if rf, ok := ret.Get(0).(func(context.Context, dto.ValidateRequest) map[job.Name][]dto.ValidateResult); ok { - r0 = rf(ctx, request) - } else { - r0 = ret.Get(0).(map[job.Name][]dto.ValidateResult) - } - - var r1 error - if rf, ok := ret.Get(1).(func(context.Context, dto.ValidateRequest) error); ok { - r1 = rf(ctx, request) + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, tenant.Tenant, []*job.Spec, []job.Name, writer.LogWriter) error); ok { + r0 = rf(ctx, jobTenant, jobSpecs, jobNamesWithInvalidSpec, logWriter) } else { - r1 = ret.Error(1) + r0 = ret.Error(0) } - return r0, r1 + return r0 } // ReplaceAllJobSpecificationsServer is an autogenerated mock type for the ReplaceAllJobSpecificationsServer type diff --git a/core/job/job.go b/core/job/job.go index 9dde620e3f..70b1a73165 100644 --- a/core/job/job.go +++ b/core/job/job.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/errors" ) @@ -29,8 +28,6 @@ const ( MetricJobEventDisabled = "disabled" MetricJobEventFoundDirty = "found_dirty" - MetricJobValidation = "job_validation" - MetricJobRefreshResourceDownstream = "refresh_resource_downstream_total" UnspecifiedImpactChange UpdateImpact = "unspecified_impact" @@ -43,8 +40,8 @@ type Job struct { spec *Spec - destination resource.URN - sources []resource.URN + destination ResourceURN + sources []ResourceURN isDirty bool } @@ -133,7 +130,7 @@ func (n ResourceURN) String() string { } type ResourceURNWithUpstreams struct { - URN resource.URN + URN ResourceURN Upstreams []*ResourceURNWithUpstreams } @@ -154,11 +151,11 @@ func (rs ResourceURNWithUpstreamsList) Flatten() []*ResourceURNWithUpstreams { return output } -func (j *Job) Destination() resource.URN { +func (j *Job) Destination() ResourceURN { return j.destination } -func (j *Job) Sources() []resource.URN { +func (j *Job) Sources() []ResourceURN { return j.sources } @@ -173,7 +170,7 @@ func (j *Job) ProjectName() tenant.ProjectName { return j.Tenant().ProjectName() } -func NewJob(tenant tenant.Tenant, spec *Spec, destination resource.URN, sources []resource.URN, isDirty bool) *Job { +func NewJob(tenant tenant.Tenant, spec *Spec, destination ResourceURN, sources []ResourceURN, isDirty bool) *Job { return &Job{tenant: tenant, spec: spec, destination: destination, sources: sources, isDirty: isDirty} } @@ -195,6 +192,14 @@ func (j Jobs) GetNameMap() map[Name]*Job { return jobNameMap } +func (j Jobs) GetNameAndJobMap() map[Name]*Job { + nameAndJobMap := make(map[Name]*Job, len(j)) + for _, job := range j { + nameAndJobMap[job.spec.Name()] = job + } + return nameAndJobMap +} + func (j Jobs) GetNamespaceNameAndJobsMap() map[tenant.NamespaceName][]*Job { jobsPerNamespaceName := make(map[tenant.NamespaceName][]*Job, len(j)) for _, job := range j { @@ -211,16 +216,6 @@ func (j Jobs) GetSpecs() []*Spec { return specs } -func (j Jobs) GetFullNameToSpecMap() map[FullName]*Spec { - fullNameToSpecMap := make(map[FullName]*Spec, len(j)) - for _, subjectJob := range j { - fullName := FullNameFrom(subjectJob.ProjectName(), subjectJob.Spec().Name()) - fullNameToSpecMap[fullName] = subjectJob.Spec() - } - - return fullNameToSpecMap -} - func (j Jobs) GetJobsWithUnresolvedUpstreams() ([]*WithUpstream, error) { me := errors.NewMultiError("get unresolved upstreams errors") @@ -330,7 +325,7 @@ func (w WithUpstreams) MergeWithResolvedUpstreams(resolvedUpstreamsBySubjectJobM type Upstream struct { name Name host string - resource resource.URN + resource ResourceURN taskName TaskName projectName tenant.ProjectName @@ -342,7 +337,7 @@ type Upstream struct { external bool } -func NewUpstreamResolved(name Name, host string, resource resource.URN, jobTenant tenant.Tenant, upstreamType UpstreamType, taskName TaskName, external bool) *Upstream { +func NewUpstreamResolved(name Name, host string, resource ResourceURN, jobTenant tenant.Tenant, upstreamType UpstreamType, taskName TaskName, external bool) *Upstream { return &Upstream{ name: name, host: host, @@ -356,7 +351,7 @@ func NewUpstreamResolved(name Name, host string, resource resource.URN, jobTenan } } -func NewUpstreamUnresolvedInferred(resource resource.URN) *Upstream { +func NewUpstreamUnresolvedInferred(resource ResourceURN) *Upstream { return &Upstream{resource: resource, _type: UpstreamTypeInferred, state: UpstreamStateUnresolved} } @@ -372,7 +367,7 @@ func (u *Upstream) Host() string { return u.host } -func (u *Upstream) Resource() resource.URN { +func (u *Upstream) Resource() ResourceURN { return u.resource } @@ -441,7 +436,7 @@ func (u Upstreams) ToFullNameAndUpstreamMap() map[string]*Upstream { func (u Upstreams) ToResourceDestinationAndUpstreamMap() map[string]*Upstream { resourceDestinationUpstreamMap := make(map[string]*Upstream) for _, upstream := range u { - if upstream.resource.IsZero() { + if upstream.resource == "" { continue } resourceDestinationUpstreamMap[upstream.resource.String()] = upstream diff --git a/core/job/job_test.go b/core/job/job_test.go index e04823c775..42232dfb87 100644 --- a/core/job/job_test.go +++ b/core/job/job_test.go @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/lib/window" "github.com/goto/optimus/internal/models" @@ -33,15 +32,13 @@ func TestEntityJob(t *testing.T) { jobTask := job.NewTask("bq2bq", jobTaskConfig) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).Build() - jobADestination, _ := resource.ParseURN("store://project.dataset.sample-a") - jobASource, _ := resource.ParseURN("store://project.dataset.sample-b") - jobASources := []resource.URN{jobASource} + jobADestination := job.ResourceURN("project.dataset.sample-a") + jobASources := []job.ResourceURN{"project.dataset.sample-b"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobASources, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).Build() - jobBDestination, _ := resource.ParseURN("store://project.dataset.sample-b") - jobBSource, _ := resource.ParseURN("store://project.dataset.sample-c") - jobBSources := []resource.URN{jobBSource} + jobBDestination := job.ResourceURN("project.dataset.sample-b") + jobBSources := []job.ResourceURN{"project.dataset.sample-c"} jobB := job.NewJob(sampleTenant, specB, jobBDestination, jobBSources, false) t.Run("GetJobNames", func(t *testing.T) { @@ -67,25 +64,19 @@ func TestEntityJob(t *testing.T) { assert.EqualValues(t, expectedMap, resultMap) }) }) - - t.Run("GetFullNameToSpecMap", func(t *testing.T) { - t.Run("should return map with key full name and value spec", func(t *testing.T) { - jobs := []*job.Job{jobA, jobB} - - fullNameA := job.FullNameFrom(jobA.ProjectName(), job.Name(jobA.GetName())) - fullNameB := job.FullNameFrom(jobB.ProjectName(), job.Name(jobB.GetName())) - expectedMap := map[job.FullName]*job.Spec{ - fullNameA: specA, - fullNameB: specB, + t.Run("GetNameAndJobMap", func(t *testing.T) { + t.Run("should return map with name as key and job as value", func(t *testing.T) { + expectedMap := map[job.Name]*job.Job{ + specA.Name(): jobA, + specB.Name(): jobB, } - actualMap := job.Jobs(jobs).GetFullNameToSpecMap() + jobs := job.Jobs([]*job.Job{jobA, jobB}) + resultMap := jobs.GetNameAndJobMap() - assert.EqualValues(t, expectedMap[fullNameA], actualMap[fullNameA]) - assert.EqualValues(t, expectedMap[fullNameB], actualMap[fullNameB]) + assert.EqualValues(t, expectedMap, resultMap) }) }) - t.Run("GetNamespaceNameAndJobsMap", func(t *testing.T) { t.Run("should return map with namespace name as key and jobs as value", func(t *testing.T) { expectedMap := map[tenant.NamespaceName][]*job.Job{ @@ -114,10 +105,8 @@ func TestEntityJob(t *testing.T) { t.Run("GetUnresolvedUpstreams", func(t *testing.T) { t.Run("should return upstreams with state unresolved", func(t *testing.T) { upstreamUnresolved1 := job.NewUpstreamUnresolvedStatic("job-B", project.Name()) - resourceURNC, _ := resource.ParseURN("store://project.dataset.sample-c") - upstreamUnresolved2 := job.NewUpstreamUnresolvedInferred(resourceURNC) - resourceURND, _ := resource.ParseURN("store://project.dataset.sample-d") - upstreamResolved := job.NewUpstreamResolved("job-d", "host-sample", resourceURND, sampleTenant, job.UpstreamTypeStatic, "", false) + upstreamUnresolved2 := job.NewUpstreamUnresolvedInferred("project.dataset.sample-c") + upstreamResolved := job.NewUpstreamResolved("job-d", "host-sample", "project.dataset.sample-d", sampleTenant, job.UpstreamTypeStatic, "", false) expected := []*job.Upstream{upstreamUnresolved1, upstreamUnresolved2} @@ -130,12 +119,9 @@ func TestEntityJob(t *testing.T) { t.Run("GetResolvedUpstreams", func(t *testing.T) { t.Run("should return upstreams with state resolved", func(t *testing.T) { upstreamUnresolved1 := job.NewUpstreamUnresolvedStatic("job-B", project.Name()) - resourceURNC, _ := resource.ParseURN("store://project.dataset.sample-c") - upstreamUnresolved2 := job.NewUpstreamUnresolvedInferred(resourceURNC) - resourceURND, _ := resource.ParseURN("store://project.dataset.sample-d") - resourceURNE, _ := resource.ParseURN("store://project.dataset.sample-e") - upstreamResolved1 := job.NewUpstreamResolved("job-d", "host-sample", resourceURND, sampleTenant, job.UpstreamTypeStatic, "", false) - upstreamResolved2 := job.NewUpstreamResolved("job-e", "host-sample", resourceURNE, sampleTenant, job.UpstreamTypeInferred, "", true) + upstreamUnresolved2 := job.NewUpstreamUnresolvedInferred("project.dataset.sample-c") + upstreamResolved1 := job.NewUpstreamResolved("job-d", "host-sample", "project.dataset.sample-d", sampleTenant, job.UpstreamTypeStatic, "", false) + upstreamResolved2 := job.NewUpstreamResolved("job-e", "host-sample", "project.dataset.sample-e", sampleTenant, job.UpstreamTypeInferred, "", true) expected := []*job.Upstream{upstreamResolved1, upstreamResolved2} @@ -165,10 +151,8 @@ func TestEntityJob(t *testing.T) { t.Run("ToFullNameAndUpstreamMap", func(t *testing.T) { t.Run("should return a map with full name as key and boolean as value", func(t *testing.T) { - resourceURNA, _ := resource.ParseURN("store://project.dataset.sample-a") - resourceURNB, _ := resource.ParseURN("store://project.dataset.sample-b") - upstreamResolved1 := job.NewUpstreamResolved("job-a", "host-sample", resourceURNA, sampleTenant, job.UpstreamTypeStatic, "", false) - upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", resourceURNB, sampleTenant, job.UpstreamTypeInferred, "", false) + upstreamResolved1 := job.NewUpstreamResolved("job-a", "host-sample", "project.dataset.sample-a", sampleTenant, job.UpstreamTypeStatic, "", false) + upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", "project.dataset.sample-b", sampleTenant, job.UpstreamTypeInferred, "", false) expectedMap := map[string]*job.Upstream{ "test-proj/job-a": upstreamResolved1, @@ -184,14 +168,12 @@ func TestEntityJob(t *testing.T) { t.Run("ToResourceDestinationAndUpstreamMap", func(t *testing.T) { t.Run("should return a map with destination resource urn as key and boolean as value", func(t *testing.T) { - resourceURNA, _ := resource.ParseURN("store://project.dataset.sample-a") - resourceURNB, _ := resource.ParseURN("store://project.dataset.sample-b") - upstreamResolved1 := job.NewUpstreamResolved("job-a", "host-sample", resourceURNA, sampleTenant, job.UpstreamTypeStatic, "", false) - upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", resourceURNB, sampleTenant, job.UpstreamTypeInferred, "", false) + upstreamResolved1 := job.NewUpstreamResolved("job-a", "host-sample", "project.dataset.sample-a", sampleTenant, job.UpstreamTypeStatic, "", false) + upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", "project.dataset.sample-b", sampleTenant, job.UpstreamTypeInferred, "", false) expectedMap := map[string]*job.Upstream{ - "store://project.dataset.sample-a": upstreamResolved1, - "store://project.dataset.sample-b": upstreamResolved2, + "project.dataset.sample-a": upstreamResolved1, + "project.dataset.sample-b": upstreamResolved2, } upstreams := job.Upstreams([]*job.Upstream{upstreamResolved1, upstreamResolved2}) @@ -200,14 +182,11 @@ func TestEntityJob(t *testing.T) { assert.EqualValues(t, expectedMap, resultMap) }) t.Run("should skip a job if resource destination is not found and should not return error", func(t *testing.T) { - var zeroURN resource.URN - resourceURNB, _ := resource.ParseURN("store://project.dataset.sample-b") - - upstreamResolved1 := job.NewUpstreamResolved("job-a", "host-sample", zeroURN, sampleTenant, job.UpstreamTypeStatic, "", false) - upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", resourceURNB, sampleTenant, job.UpstreamTypeInferred, "", false) + upstreamResolved1 := job.NewUpstreamResolved("job-a", "host-sample", "", sampleTenant, job.UpstreamTypeStatic, "", false) + upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", "project.dataset.sample-b", sampleTenant, job.UpstreamTypeInferred, "", false) expectedMap := map[string]*job.Upstream{ - "store://project.dataset.sample-b": upstreamResolved2, + "project.dataset.sample-b": upstreamResolved2, } upstreams := job.Upstreams([]*job.Upstream{upstreamResolved1, upstreamResolved2}) @@ -219,16 +198,13 @@ func TestEntityJob(t *testing.T) { t.Run("Deduplicate", func(t *testing.T) { t.Run("should return upstreams with static being prioritized if duplication is found", func(t *testing.T) { - resourceURNA, _ := resource.ParseURN("store://project.dataset.sample-a") - resourceURNB, _ := resource.ParseURN("store://project.dataset.sample-b") - resourceURND, _ := resource.ParseURN("store://project.dataset.sample-d") - upstreamResolved1Inferred := job.NewUpstreamResolved("job-a", "host-sample", resourceURNA, sampleTenant, job.UpstreamTypeInferred, "", false) - upstreamResolved1Static := job.NewUpstreamResolved("job-a", "host-sample", resourceURNA, sampleTenant, job.UpstreamTypeStatic, "", false) - upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", resourceURNB, sampleTenant, job.UpstreamTypeInferred, "", false) + upstreamResolved1Inferred := job.NewUpstreamResolved("job-a", "host-sample", "project.dataset.sample-a", sampleTenant, job.UpstreamTypeInferred, "", false) + upstreamResolved1Static := job.NewUpstreamResolved("job-a", "host-sample", "project.dataset.sample-a", sampleTenant, job.UpstreamTypeStatic, "", false) + upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", "project.dataset.sample-b", sampleTenant, job.UpstreamTypeInferred, "", false) upstreamUnresolved1 := job.NewUpstreamUnresolvedStatic("job-c", sampleTenant.ProjectName()) - upstreamUnresolved2 := job.NewUpstreamUnresolvedInferred(resourceURND) + upstreamUnresolved2 := job.NewUpstreamUnresolvedInferred("project.dataset.sample-d") upstreamUnresolved3 := job.NewUpstreamUnresolvedStatic("job-c", sampleTenant.ProjectName()) - upstreamUnresolved4 := job.NewUpstreamUnresolvedInferred(resourceURND) + upstreamUnresolved4 := job.NewUpstreamUnresolvedInferred("project.dataset.sample-d") expected := []*job.Upstream{ upstreamResolved1Static, @@ -251,11 +227,9 @@ func TestEntityJob(t *testing.T) { assert.ElementsMatch(t, expected, result) }) t.Run("should successfully return distinct upstreams when only resolved upstream is present", func(t *testing.T) { - resourceURNA, _ := resource.ParseURN("store://project.dataset.sample-a") - resourceURNB, _ := resource.ParseURN("store://project.dataset.sample-b") - upstreamResolved1Inferred := job.NewUpstreamResolved("job-a", "host-sample", resourceURNA, sampleTenant, job.UpstreamTypeInferred, "", false) - upstreamResolved1Static := job.NewUpstreamResolved("job-a", "host-sample", resourceURNA, sampleTenant, job.UpstreamTypeStatic, "", false) - upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", resourceURNB, sampleTenant, job.UpstreamTypeInferred, "", false) + upstreamResolved1Inferred := job.NewUpstreamResolved("job-a", "host-sample", "project.dataset.sample-a", sampleTenant, job.UpstreamTypeInferred, "", false) + upstreamResolved1Static := job.NewUpstreamResolved("job-a", "host-sample", "project.dataset.sample-a", sampleTenant, job.UpstreamTypeStatic, "", false) + upstreamResolved2 := job.NewUpstreamResolved("job-b", "host-sample", "project.dataset.sample-b", sampleTenant, job.UpstreamTypeInferred, "", false) expected := []*job.Upstream{ upstreamResolved1Static, @@ -272,11 +246,10 @@ func TestEntityJob(t *testing.T) { assert.ElementsMatch(t, expected, result) }) t.Run("should successfully return distinct upstreams when only unresolved upstream is present", func(t *testing.T) { - resourceURND, _ := resource.ParseURN("store://project.dataset.sample-d") upstreamUnresolved1 := job.NewUpstreamUnresolvedStatic("job-c", sampleTenant.ProjectName()) - upstreamUnresolved2 := job.NewUpstreamUnresolvedInferred(resourceURND) + upstreamUnresolved2 := job.NewUpstreamUnresolvedInferred("project.dataset.sample-d") upstreamUnresolved3 := job.NewUpstreamUnresolvedStatic("job-c", sampleTenant.ProjectName()) - upstreamUnresolved4 := job.NewUpstreamUnresolvedInferred(resourceURND) + upstreamUnresolved4 := job.NewUpstreamUnresolvedInferred("project.dataset.sample-d") expected := []*job.Upstream{ upstreamUnresolved1, @@ -317,9 +290,8 @@ func TestEntityJob(t *testing.T) { t.Run("should return values as inserted", func(t *testing.T) { specUpstream, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"job-E"}).Build() specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).WithSpecUpstream(specUpstream).Build() - jobCDestination, _ := resource.ParseURN("store://project.dataset.sample-c") - jobCSource, _ := resource.ParseURN("store://project.dataset.sample-c") - jobCSources := []resource.URN{jobCSource} + jobCDestination := job.ResourceURN("project.dataset.sample-c") + jobCSources := []job.ResourceURN{"project.dataset.sample-d"} jobC := job.NewJob(sampleTenant, specC, jobCDestination, jobCSources, false) assert.Equal(t, sampleTenant, jobC.Tenant()) @@ -341,12 +313,10 @@ func TestEntityJob(t *testing.T) { t.Run("WithUpstream", func(t *testing.T) { t.Run("should return values as constructed", func(t *testing.T) { - resourceURNC, _ := resource.ParseURN("store://project.dataset.sample-c") - resourceURND, _ := resource.ParseURN("store://project.dataset.sample-d") - upstreamResolved := job.NewUpstreamResolved("job-d", "host-sample", resourceURND, sampleTenant, job.UpstreamTypeStatic, "bq2bq", false) + upstreamResolved := job.NewUpstreamResolved("job-d", "host-sample", "project.dataset.sample-d", sampleTenant, job.UpstreamTypeStatic, "bq2bq", false) assert.Equal(t, job.Name("job-d"), upstreamResolved.Name()) assert.Equal(t, "host-sample", upstreamResolved.Host()) - assert.Equal(t, resourceURND, upstreamResolved.Resource()) + assert.Equal(t, job.ResourceURN("project.dataset.sample-d"), upstreamResolved.Resource()) assert.Equal(t, job.UpstreamTypeStatic, upstreamResolved.Type()) assert.Equal(t, job.UpstreamStateResolved, upstreamResolved.State()) assert.Equal(t, project.Name(), upstreamResolved.ProjectName()) @@ -355,7 +325,7 @@ func TestEntityJob(t *testing.T) { assert.Equal(t, job.TaskName("bq2bq"), upstreamResolved.TaskName()) assert.Equal(t, "test-proj/job-d", upstreamResolved.FullName()) - upstreamUnresolved := job.NewUpstreamUnresolvedInferred(resourceURNC) + upstreamUnresolved := job.NewUpstreamUnresolvedInferred("project.dataset.sample-c") jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamResolved, upstreamUnresolved}) assert.Equal(t, jobA, jobAWithUpstream.Job()) @@ -367,8 +337,7 @@ func TestEntityJob(t *testing.T) { t.Run("WithUpstreams", func(t *testing.T) { t.Run("GetSubjectJobNames", func(t *testing.T) { t.Run("should return job names of WithUpstream list", func(t *testing.T) { - resourceURND, _ := resource.ParseURN("store://project.dataset.sample-d") - upstreamResolved := job.NewUpstreamResolved("job-d", "host-sample", resourceURND, sampleTenant, job.UpstreamTypeStatic, "bq2bq", false) + upstreamResolved := job.NewUpstreamResolved("job-d", "host-sample", "project.dataset.sample-d", sampleTenant, job.UpstreamTypeStatic, "bq2bq", false) jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamResolved}) jobBWithUpstream := job.NewWithUpstream(jobB, []*job.Upstream{upstreamResolved}) jobsWithUpstream := []*job.WithUpstream{jobAWithUpstream, jobBWithUpstream} @@ -378,18 +347,15 @@ func TestEntityJob(t *testing.T) { }) }) t.Run("MergeWithResolvedUpstreams", func(t *testing.T) { - resourceURNC, _ := resource.ParseURN("store://project.dataset.sample-c") - resourceURND, _ := resource.ParseURN("store://project.dataset.sample-d") - resourceURNF, _ := resource.ParseURN("store://project.dataset.sample-f") - upstreamCUnresolved := job.NewUpstreamUnresolvedInferred(resourceURNC) - upstreamDUnresolvedInferred := job.NewUpstreamUnresolvedInferred(resourceURND) + upstreamCUnresolved := job.NewUpstreamUnresolvedInferred("project.dataset.sample-c") + upstreamDUnresolvedInferred := job.NewUpstreamUnresolvedInferred("project.dataset.sample-d") upstreamDUnresolvedStatic := job.NewUpstreamUnresolvedStatic("job-D", project.Name()) upstreamEUnresolved := job.NewUpstreamUnresolvedStatic("job-E", project.Name()) - upstreamFUnresolved := job.NewUpstreamUnresolvedInferred(resourceURNF) + upstreamFUnresolved := job.NewUpstreamUnresolvedInferred("project.dataset.sample-f") - upstreamCResolved := job.NewUpstreamResolved("job-C", "host-sample", resourceURNC, sampleTenant, job.UpstreamTypeInferred, "bq2bq", false) - upstreamDResolvedStatic := job.NewUpstreamResolved("job-D", "host-sample", resourceURND, sampleTenant, job.UpstreamTypeStatic, "bq2bq", false) - upstreamDResolvedInferred := job.NewUpstreamResolved("job-D", "host-sample", resourceURND, sampleTenant, job.UpstreamTypeInferred, "bq2bq", false) + upstreamCResolved := job.NewUpstreamResolved("job-C", "host-sample", "project.dataset.sample-c", sampleTenant, job.UpstreamTypeInferred, "bq2bq", false) + upstreamDResolvedStatic := job.NewUpstreamResolved("job-D", "host-sample", "project.dataset.sample-d", sampleTenant, job.UpstreamTypeStatic, "bq2bq", false) + upstreamDResolvedInferred := job.NewUpstreamResolved("job-D", "host-sample", "project.dataset.sample-d", sampleTenant, job.UpstreamTypeInferred, "bq2bq", false) resolvedUpstreamMap := map[job.Name][]*job.Upstream{ "job-A": {upstreamCResolved, upstreamDResolvedInferred}, diff --git a/core/job/resolver/external_upstream_resolver_test.go b/core/job/resolver/external_upstream_resolver_test.go index 94f6fef42f..6bedd608f1 100644 --- a/core/job/resolver/external_upstream_resolver_test.go +++ b/core/job/resolver/external_upstream_resolver_test.go @@ -12,7 +12,6 @@ import ( "github.com/goto/optimus/config" "github.com/goto/optimus/core/job" "github.com/goto/optimus/core/job/resolver" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/ext/resourcemanager" "github.com/goto/optimus/internal/lib/window" @@ -36,15 +35,7 @@ func TestExternalUpstreamResolver(t *testing.T) { jobTask := job.NewTask(taskName, jobTaskConfig) upstreamSpec, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"external-project/job-B"}).Build() specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).Build() - - resourceURNB, err := resource.ParseURN("store://resource-B") - assert.NoError(t, err) - resourceURNC, err := resource.ParseURN("store://resource-C") - assert.NoError(t, err) - resourceURND, err := resource.ParseURN("store://resource-D") - assert.NoError(t, err) - - jobA := job.NewJob(sampleTenant, specA, resource.ZeroURN(), []resource.URN{resourceURNC}, false) + jobA := job.NewJob(sampleTenant, specA, "", []job.ResourceURN{"resource-C"}, false) t.Run("BulkResolve", func(t *testing.T) { t.Run("resolves upstream externally", func(t *testing.T) { @@ -52,11 +43,11 @@ func TestExternalUpstreamResolver(t *testing.T) { defer logWriter.AssertExpectations(t) unresolvedUpstreamB := job.NewUpstreamUnresolvedStatic("job-B", externalTenant.ProjectName()) - unresolvedUpstreamC := job.NewUpstreamUnresolvedInferred(resourceURNC) + unresolvedUpstreamC := job.NewUpstreamUnresolvedInferred("resource-C") jobWithUnresolvedUpstream := job.NewWithUpstream(jobA, []*job.Upstream{unresolvedUpstreamB, unresolvedUpstreamC}) - upstreamB := job.NewUpstreamResolved("job-B", "external-host", resourceURNB, externalTenant, "static", taskName, true) - upstreamC := job.NewUpstreamResolved("job-C", "external-host", resourceURNC, externalTenant, "inferred", taskName, true) + upstreamB := job.NewUpstreamResolved("job-B", "external-host", "resource-B", externalTenant, "static", taskName, true) + upstreamC := job.NewUpstreamResolved("job-C", "external-host", "resource-C", externalTenant, "inferred", taskName, true) resourceManager.On("GetOptimusUpstreams", ctx, unresolvedUpstreamB).Return([]*job.Upstream{upstreamB}, nil).Once() resourceManager.On("GetOptimusUpstreams", ctx, unresolvedUpstreamC).Return([]*job.Upstream{upstreamC}, nil).Once() @@ -76,12 +67,12 @@ func TestExternalUpstreamResolver(t *testing.T) { defer logWriter.AssertExpectations(t) unresolvedUpstreamB := job.NewUpstreamUnresolvedStatic("job-B", externalTenant.ProjectName()) - unresolvedUpstreamC := job.NewUpstreamUnresolvedInferred(resourceURNC) - upstreamD := job.NewUpstreamResolved("job-D", "internal-host", resourceURND, sampleTenant, "inferred", taskName, false) + unresolvedUpstreamC := job.NewUpstreamUnresolvedInferred("resource-C") + upstreamD := job.NewUpstreamResolved("job-D", "internal-host", "resource-D", sampleTenant, "inferred", taskName, false) jobWithUnresolvedUpstream := job.NewWithUpstream(jobA, []*job.Upstream{unresolvedUpstreamB, unresolvedUpstreamC, upstreamD}) - upstreamB := job.NewUpstreamResolved("job-B", "external-host", resourceURNB, externalTenant, "static", taskName, true) - upstreamC := job.NewUpstreamResolved("job-C", "external-host", resourceURNC, externalTenant, "inferred", taskName, true) + upstreamB := job.NewUpstreamResolved("job-B", "external-host", "resource-B", externalTenant, "static", taskName, true) + upstreamC := job.NewUpstreamResolved("job-C", "external-host", "resource-C", externalTenant, "inferred", taskName, true) resourceManager.On("GetOptimusUpstreams", ctx, unresolvedUpstreamB).Return([]*job.Upstream{upstreamB}, nil).Once() resourceManager.On("GetOptimusUpstreams", ctx, unresolvedUpstreamC).Return([]*job.Upstream{upstreamC}, nil).Once() @@ -101,7 +92,7 @@ func TestExternalUpstreamResolver(t *testing.T) { defer logWriter.AssertExpectations(t) unresolvedUpstreamB := job.NewUpstreamUnresolvedStatic("job-B", externalTenant.ProjectName()) - unresolvedUpstreamC := job.NewUpstreamUnresolvedInferred(resourceURNC) + unresolvedUpstreamC := job.NewUpstreamUnresolvedInferred("resource-C") jobWithUnresolvedUpstream := job.NewWithUpstream(jobA, []*job.Upstream{unresolvedUpstreamB, unresolvedUpstreamC}) resourceManager.On("GetOptimusUpstreams", ctx, unresolvedUpstreamB).Return([]*job.Upstream{}, errors.New("connection error")).Once() @@ -122,7 +113,7 @@ func TestExternalUpstreamResolver(t *testing.T) { defer logWriter.AssertExpectations(t) unresolvedUpstreamB := job.NewUpstreamUnresolvedStatic("job-B", externalTenant.ProjectName()) - unresolvedUpstreamC := job.NewUpstreamUnresolvedInferred(resourceURNC) + unresolvedUpstreamC := job.NewUpstreamUnresolvedInferred("resource-C") jobWithUnresolvedUpstream := job.NewWithUpstream(jobA, []*job.Upstream{unresolvedUpstreamB, unresolvedUpstreamC}) extUpstreamResolver := resolver.NewTestExternalUpstreamResolver(nil) diff --git a/core/job/resolver/internal_upstream_resolver.go b/core/job/resolver/internal_upstream_resolver.go index 5769822ac7..97f3540486 100644 --- a/core/job/resolver/internal_upstream_resolver.go +++ b/core/job/resolver/internal_upstream_resolver.go @@ -6,7 +6,6 @@ import ( "golang.org/x/net/context" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/errors" ) @@ -67,7 +66,7 @@ func (i internalUpstreamResolver) BulkResolve(ctx context.Context, projectName t return jobsWithMergedUpstream, nil } -func (i internalUpstreamResolver) resolveInferredUpstream(ctx context.Context, sources []resource.URN) ([]*job.Upstream, error) { +func (i internalUpstreamResolver) resolveInferredUpstream(ctx context.Context, sources []job.ResourceURN) ([]*job.Upstream, error) { var internalUpstream []*job.Upstream me := errors.NewMultiError("resolve internal inferred upstream errors") for _, source := range sources { diff --git a/core/job/resolver/internal_upstream_resolver_test.go b/core/job/resolver/internal_upstream_resolver_test.go index cf1016f26d..53ed292531 100644 --- a/core/job/resolver/internal_upstream_resolver_test.go +++ b/core/job/resolver/internal_upstream_resolver_test.go @@ -9,7 +9,6 @@ import ( "github.com/goto/optimus/core/job" "github.com/goto/optimus/core/job/resolver" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/lib/window" "github.com/goto/optimus/internal/models" @@ -19,19 +18,6 @@ func TestInternalUpstreamResolver(t *testing.T) { ctx := context.Background() sampleTenant, _ := tenant.NewTenant("project", "namespace") - resourceURNA, err := resource.ParseURN("store://resource-A") - assert.NoError(t, err) - resourceURNB, err := resource.ParseURN("store://resource-B") - assert.NoError(t, err) - resourceURNC, err := resource.ParseURN("store://resource-C") - assert.NoError(t, err) - resourceURND, err := resource.ParseURN("store://resource-D") - assert.NoError(t, err) - resourceURNE, err := resource.ParseURN("store://resource-E") - assert.NoError(t, err) - resourceURNX, err := resource.ParseURN("store://resource-X") - assert.NoError(t, err) - jobVersion := 1 startDate, _ := job.ScheduleDateFrom("2022-10-01") jobSchedule, _ := job.NewScheduleBuilder(startDate).Build() @@ -42,24 +28,24 @@ func TestInternalUpstreamResolver(t *testing.T) { jobTask := job.NewTask(taskName, jobTaskConfig) upstreamSpec, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"job-C"}).Build() specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).Build() - jobADestination := resourceURNA - jobASources := []resource.URN{resourceURNB, resourceURND} + jobADestination := job.ResourceURN("resource-A") + jobASources := []job.ResourceURN{"resource-B", "resource-D"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobASources, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).Build() - jobBDestination := resourceURNB + jobBDestination := job.ResourceURN("resource-B") jobB := job.NewJob(sampleTenant, specB, jobBDestination, nil, false) specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).Build() - jobCDestination := resourceURNC + jobCDestination := job.ResourceURN("resource-C") jobC := job.NewJob(sampleTenant, specC, jobCDestination, nil, false) - internalUpstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "inferred", taskName, false) - internalUpstreamC := job.NewUpstreamResolved("job-C", "", resourceURNC, sampleTenant, "static", taskName, false) + internalUpstreamB := job.NewUpstreamResolved("job-B", "", "resource-B", sampleTenant, "inferred", taskName, false) + internalUpstreamC := job.NewUpstreamResolved("job-C", "", "resource-C", sampleTenant, "static", taskName, false) - unresolvedUpstreamB := job.NewUpstreamUnresolvedInferred(resourceURNB) + unresolvedUpstreamB := job.NewUpstreamUnresolvedInferred("resource-B") unresolvedUpstreamC := job.NewUpstreamUnresolvedStatic("job-C", "project") - unresolvedUpstreamD := job.NewUpstreamUnresolvedInferred(resourceURND) + unresolvedUpstreamD := job.NewUpstreamUnresolvedInferred("resource-D") t.Run("Resolve", func(t *testing.T) { t.Run("resolves inferred and static upstream internally", func(t *testing.T) { @@ -86,13 +72,13 @@ func TestInternalUpstreamResolver(t *testing.T) { defer logWriter.AssertExpectations(t) specD, _ := job.NewSpecBuilder(jobVersion, "job-D", "sample-owner", jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).Build() - jobDDestination := resourceURND - jobDSources := []resource.URN{resourceURNC} + jobDDestination := job.ResourceURN("resource-D") + jobDSources := []job.ResourceURN{"resource-C"} jobD := job.NewJob(sampleTenant, specD, jobDDestination, jobDSources, false) - unresolvedUpstreamCInferred := job.NewUpstreamUnresolvedInferred(resourceURNC) + unresolvedUpstreamCInferred := job.NewUpstreamUnresolvedInferred("resource-C") unresolvedUpstreamCStatic := job.NewUpstreamUnresolvedStatic("job-C", sampleTenant.ProjectName()) - internalUpstreamCStatic := job.NewUpstreamResolved("job-C", "", resourceURNC, sampleTenant, "static", taskName, false) + internalUpstreamCStatic := job.NewUpstreamResolved("job-C", "", "resource-C", sampleTenant, "static", taskName, false) jobRepo.On("GetAllByResourceDestination", ctx, jobDSources[0]).Return([]*job.Job{jobC}, nil) jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specC.Name()).Return(jobC, nil) @@ -111,8 +97,8 @@ func TestInternalUpstreamResolver(t *testing.T) { defer logWriter.AssertExpectations(t) specX, _ := job.NewSpecBuilder(jobVersion, "job-X", "sample-owner", jobSchedule, jobWindow, jobTask).Build() - jobXDestination := resourceURNX - jobX := job.NewJob(sampleTenant, specX, jobXDestination, []resource.URN{resourceURNB}, false) + jobXDestination := job.ResourceURN("resource-X") + jobX := job.NewJob(sampleTenant, specX, jobXDestination, []job.ResourceURN{"resource-B"}, false) jobRepo.On("GetAllByResourceDestination", ctx, jobX.Sources()[0]).Return([]*job.Job{jobB}, nil) @@ -130,7 +116,7 @@ func TestInternalUpstreamResolver(t *testing.T) { defer logWriter.AssertExpectations(t) specX, _ := job.NewSpecBuilder(jobVersion, "job-X", "sample-owner", jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).Build() - jobXDestination := resourceURNX + jobXDestination := job.ResourceURN("resource-X") jobX := job.NewJob(sampleTenant, specX, jobXDestination, nil, false) jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specC.Name()).Return(jobC, nil) @@ -172,7 +158,7 @@ func TestInternalUpstreamResolver(t *testing.T) { specEUpstreamSpec, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"job-unknown", "job-C"}).Build() specE, err := job.NewSpecBuilder(jobVersion, "job-E", "sample-owner", jobSchedule, jobWindow, jobTask).WithSpecUpstream(specEUpstreamSpec).Build() assert.NoError(t, err) - jobEDestination := resourceURNE + jobEDestination := job.ResourceURN("resource-E") jobE := job.NewJob(sampleTenant, specE, jobEDestination, nil, false) jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job-unknown")).Return(nil, errors.New("not found")) @@ -200,7 +186,7 @@ func TestInternalUpstreamResolver(t *testing.T) { specE, err := job.NewSpecBuilder(jobVersion, "job-E", "sample-owner", jobSchedule, jobWindow, jobTask).WithSpecUpstream(specEUpstreamSpec).Build() assert.NoError(t, err) - jobEDestination := resourceURNE + jobEDestination := job.ResourceURN("resource-E") jobE := job.NewJob(sampleTenant, specE, jobEDestination, nil, false) jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specC.Name()).Return(jobC, nil) @@ -220,8 +206,8 @@ func TestInternalUpstreamResolver(t *testing.T) { specX, err := job.NewSpecBuilder(jobVersion, "job-X", "sample-owner", jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).Build() assert.NoError(t, err) - jobXDestination := resourceURNX - jobX := job.NewJob(sampleTenant, specX, jobXDestination, []resource.URN{resourceURNB}, false) + jobXDestination := job.ResourceURN("resource-X") + jobX := job.NewJob(sampleTenant, specX, jobXDestination, []job.ResourceURN{"resource-B"}, false) t.Run("resolves upstream internally in bulk", func(t *testing.T) { jobRepo := new(JobRepository) diff --git a/core/job/resolver/upstream_resolver.go b/core/job/resolver/upstream_resolver.go index 472ed5cb79..2b60d43452 100644 --- a/core/job/resolver/upstream_resolver.go +++ b/core/job/resolver/upstream_resolver.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/errors" "github.com/goto/optimus/internal/writer" @@ -39,7 +38,7 @@ type InternalUpstreamResolver interface { type JobRepository interface { ResolveUpstreams(ctx context.Context, projectName tenant.ProjectName, jobNames []job.Name) (map[job.Name][]*job.Upstream, error) - GetAllByResourceDestination(ctx context.Context, resourceDestination resource.URN) ([]*job.Job, error) + GetAllByResourceDestination(ctx context.Context, resourceDestination job.ResourceURN) ([]*job.Job, error) GetByJobName(ctx context.Context, projectName tenant.ProjectName, jobName job.Name) (*job.Job, error) } diff --git a/core/job/resolver/upstream_resolver_test.go b/core/job/resolver/upstream_resolver_test.go index 10e506c7f7..f2d6b29a70 100644 --- a/core/job/resolver/upstream_resolver_test.go +++ b/core/job/resolver/upstream_resolver_test.go @@ -10,7 +10,6 @@ import ( "github.com/goto/optimus/core/job" "github.com/goto/optimus/core/job/resolver" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/lib/window" "github.com/goto/optimus/internal/models" @@ -45,19 +44,6 @@ func TestUpstreamResolver(t *testing.T) { jobTask := job.NewTask(taskName, jobTaskConfig) sampleOwner := "sample-owner" - resourceURNA, err := resource.ParseURN("store://resource-A") - assert.NoError(t, err) - resourceURNB, err := resource.ParseURN("store://resource-B") - assert.NoError(t, err) - resourceURNC, err := resource.ParseURN("store://resource-C") - assert.NoError(t, err) - resourceURND, err := resource.ParseURN("store://resource-D") - assert.NoError(t, err) - resourceURNF, err := resource.ParseURN("store://resource-F") - assert.NoError(t, err) - resourceURNG, err := resource.ParseURN("store://resource-G") - assert.NoError(t, err) - t.Run("CheckStaticResolvable", func(t *testing.T) { specA, err := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).Build() assert.NoError(t, err) @@ -75,8 +61,8 @@ func TestUpstreamResolver(t *testing.T) { logWriter := new(mockWriter) defer logWriter.AssertExpectations(t) - jobBDestination := resourceURNB - jobBUpstreams := []resource.URN{resourceURNG} // unresolved inferred dependency + jobBDestination := job.ResourceURN("resource-B") + jobBUpstreams := []job.ResourceURN{"resource-G"} // unresolved inferred dependency jobB := job.NewJob(sampleTenant, specB, jobBDestination, jobBUpstreams, true) jobs := []*job.Job{jobB} @@ -100,8 +86,8 @@ func TestUpstreamResolver(t *testing.T) { logWriter := new(mockWriter) defer logWriter.AssertExpectations(t) - jobBDestination := resourceURNB - jobBUpstreams := []resource.URN{resourceURNG} // unresolved inferred dependency + jobBDestination := job.ResourceURN("resource-B") + jobBUpstreams := []job.ResourceURN{"resource-G"} // unresolved inferred dependency jobB := job.NewJob(sampleTenant, specB, jobBDestination, jobBUpstreams, true) jobs := []*job.Job{jobB} @@ -126,13 +112,13 @@ func TestUpstreamResolver(t *testing.T) { logWriter := new(mockWriter) defer logWriter.AssertExpectations(t) - jobBDestination := resourceURNB - jobBUpstreams := []resource.URN{resourceURNG} // unresolved inferred dependency + jobBDestination := job.ResourceURN("resource-B") + jobBUpstreams := []job.ResourceURN{"resource-G"} // unresolved inferred dependency jobB := job.NewJob(sampleTenant, specB, jobBDestination, jobBUpstreams, true) jobs := []*job.Job{jobB} - upstreamB0 := job.NewUpstreamResolved(specA.Name(), "", resourceURNB, sampleTenant, job.UpstreamTypeStatic, taskName, false) + upstreamB0 := job.NewUpstreamResolved(specA.Name(), "", "resource-B", sampleTenant, job.UpstreamTypeStatic, taskName, false) upstreamsB := []*job.Upstream{upstreamB0} jobBWithUpstream := job.NewWithUpstream(jobB, upstreamsB) @@ -151,12 +137,12 @@ func TestUpstreamResolver(t *testing.T) { logWriter := new(mockWriter) defer logWriter.AssertExpectations(t) - jobADestination := resourceURNA - jobAUpstreams := []resource.URN{resourceURNF} // unresolved inferred dependency + jobADestination := job.ResourceURN("resource-A") + jobAUpstreams := []job.ResourceURN{"resource-F"} // unresolved inferred dependency jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreams, false) - jobBDestination := resourceURNB - jobBUpstreams := []resource.URN{resourceURNG} // unresolved inferred dependency + jobBDestination := job.ResourceURN("resource-B") + jobBUpstreams := []job.ResourceURN{"resource-G"} // unresolved inferred dependency jobB := job.NewJob(sampleTenant, specB, jobBDestination, jobBUpstreams, true) jobs := []*job.Job{jobA, jobB} @@ -187,12 +173,12 @@ func TestUpstreamResolver(t *testing.T) { logWriter := new(mockWriter) defer logWriter.AssertExpectations(t) - jobADestination := resourceURNA - jobAUpstreams := []resource.URN{resourceURNF} // unresolved inferred dependency + jobADestination := job.ResourceURN("resource-A") + jobAUpstreams := []job.ResourceURN{"resource-F"} // unresolved inferred dependency jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreams, false) - jobBDestination := resourceURNB - jobBUpstreams := []resource.URN{resourceURNG} // unresolved inferred dependency + jobBDestination := job.ResourceURN("resource-B") + jobBUpstreams := []job.ResourceURN{"resource-G"} // unresolved inferred dependency jobB := job.NewJob(sampleTenant, specB, jobBDestination, jobBUpstreams, true) jobs := []*job.Job{jobA, jobB} @@ -235,13 +221,13 @@ func TestUpstreamResolver(t *testing.T) { specA, err := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).Build() assert.NoError(t, err) - jobADestination := resourceURNA - jobAUpstreams := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("resource-A") + jobAUpstreams := []job.ResourceURN{"resource-B"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreams, false) jobs := []*job.Job{jobA} - upstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "inferred", taskName, false) - upstreamC := job.NewUpstreamResolved("job-C", "", resourceURNC, sampleTenant, "static", taskName, false) + upstreamB := job.NewUpstreamResolved("job-B", "", "resource-B", sampleTenant, "inferred", taskName, false) + upstreamC := job.NewUpstreamResolved("job-C", "", "resource-C", sampleTenant, "static", taskName, false) upstreams := []*job.Upstream{upstreamB, upstreamC} jobAWithUpstream := job.NewWithUpstream(jobA, upstreams) @@ -271,23 +257,23 @@ func TestUpstreamResolver(t *testing.T) { specA, err := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).Build() assert.NoError(t, err) - jobADestination := resourceURNA - jobAUpstreams := []resource.URN{resourceURNB, resourceURND} + jobADestination := job.ResourceURN("resource-A") + jobAUpstreams := []job.ResourceURN{"resource-B", "resource-D"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreams, false) jobs := []*job.Job{jobA} unresolvedUpstreams := []*job.Upstream{ - job.NewUpstreamUnresolvedInferred(resourceURNB), + job.NewUpstreamUnresolvedInferred("resource-B"), job.NewUpstreamUnresolvedStatic("job-c", project.Name()), - job.NewUpstreamUnresolvedInferred(resourceURND), + job.NewUpstreamUnresolvedInferred("resource-D"), } - internalUpstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + internalUpstream := job.NewUpstreamResolved("job-B", "", "resource-B", sampleTenant, "static", taskName, false) jobWithInternalUpstreams := job.NewWithUpstream(jobA, []*job.Upstream{internalUpstream, unresolvedUpstreams[1], unresolvedUpstreams[2]}) internalUpstreamResolver.On("BulkResolve", ctx, project.Name(), mock.Anything).Return([]*job.WithUpstream{jobWithInternalUpstreams}, nil) - externalUpstreamC := job.NewUpstreamResolved("job-C", "external-host", resourceURNC, externalTenant, "static", taskName, true) - externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", resourceURND, externalTenant, "inferred", taskName, true) + externalUpstreamC := job.NewUpstreamResolved("job-C", "external-host", "resource-C", externalTenant, "static", taskName, true) + externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", "resource-D", externalTenant, "inferred", taskName, true) jobWithExternalUpstreams := job.NewWithUpstream(jobA, []*job.Upstream{internalUpstream, externalUpstreamC, externalUpstreamD}) externalUpstreamResolver.On("BulkResolve", ctx, mock.Anything, mock.Anything).Return([]*job.WithUpstream{jobWithExternalUpstreams}, nil) @@ -311,8 +297,8 @@ func TestUpstreamResolver(t *testing.T) { specA, err := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).Build() assert.NoError(t, err) - jobADestination := resourceURNA - jobAUpstreams := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("resource-A") + jobAUpstreams := []job.ResourceURN{"resource-B"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreams, false) jobs := []*job.Job{jobA} @@ -340,23 +326,23 @@ func TestUpstreamResolver(t *testing.T) { specA, err := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).Build() assert.NoError(t, err) - jobADestination := resourceURNA - jobAUpstreams := []resource.URN{resourceURNB, resourceURND} + jobADestination := job.ResourceURN("resource-A") + jobAUpstreams := []job.ResourceURN{"resource-B", "resource-D"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreams, false) jobs := []*job.Job{jobA} unresolvedUpstreams := []*job.Upstream{ - job.NewUpstreamUnresolvedInferred(resourceURNB), + job.NewUpstreamUnresolvedInferred("resource-B"), job.NewUpstreamUnresolvedStatic("job-c", project.Name()), - job.NewUpstreamUnresolvedInferred(resourceURND), + job.NewUpstreamUnresolvedInferred("resource-D"), } - internalUpstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + internalUpstream := job.NewUpstreamResolved("job-B", "", "resource-B", sampleTenant, "static", taskName, false) jobWithInternalUpstreams := job.NewWithUpstream(jobA, []*job.Upstream{internalUpstream, unresolvedUpstreams[1], unresolvedUpstreams[2]}) internalUpstreamResolver.On("BulkResolve", ctx, project.Name(), mock.Anything).Return([]*job.WithUpstream{jobWithInternalUpstreams}, nil) - externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", resourceURND, externalTenant, "inferred", taskName, true) + externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", "resource-D", externalTenant, "inferred", taskName, true) jobWithExternalUpstreams := job.NewWithUpstream(jobA, []*job.Upstream{internalUpstream, unresolvedUpstreams[1], externalUpstreamD}) externalUpstreamResolver.On("BulkResolve", ctx, mock.Anything, mock.Anything).Return([]*job.WithUpstream{jobWithExternalUpstreams}, nil) @@ -382,23 +368,23 @@ func TestUpstreamResolver(t *testing.T) { specA, err := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithSpecUpstream(upstreamSpec).Build() assert.NoError(t, err) - jobADestination := resourceURNA - jobAUpstreams := []resource.URN{resourceURNB, resourceURND} + jobADestination := job.ResourceURN("resource-A") + jobAUpstreams := []job.ResourceURN{"resource-B", "resource-D"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreams, false) jobs := []*job.Job{jobA} unresolvedUpstreams := []*job.Upstream{ - job.NewUpstreamUnresolvedInferred(resourceURNB), + job.NewUpstreamUnresolvedInferred("resource-B"), job.NewUpstreamUnresolvedStatic("job-c", project.Name()), - job.NewUpstreamUnresolvedInferred(resourceURND), + job.NewUpstreamUnresolvedInferred("resource-D"), } - internalUpstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + internalUpstream := job.NewUpstreamResolved("job-B", "", "resource-B", sampleTenant, "static", taskName, false) jobWithInternalUpstreams := job.NewWithUpstream(jobA, []*job.Upstream{internalUpstream, unresolvedUpstreams[1], unresolvedUpstreams[2]}) internalUpstreamResolver.On("BulkResolve", ctx, project.Name(), mock.Anything).Return([]*job.WithUpstream{jobWithInternalUpstreams}, nil) - externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", resourceURND, externalTenant, "inferred", taskName, true) + externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", "resource-D", externalTenant, "inferred", taskName, true) jobWithExternalUpstreams := job.NewWithUpstream(jobA, []*job.Upstream{internalUpstream, unresolvedUpstreams[1], externalUpstreamD}) externalUpstreamResolver.On("BulkResolve", ctx, mock.Anything, mock.Anything).Return([]*job.WithUpstream{jobWithExternalUpstreams}, errors.New("internal error")) @@ -426,19 +412,19 @@ func TestUpstreamResolver(t *testing.T) { specA, err := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithSpecUpstream(jobAUpstreamSpec).Build() assert.NoError(t, err) - jobADestination := resourceURNA - jobASources := []resource.URN{resourceURNB, resourceURND} + jobADestination := job.ResourceURN("resource-A") + jobASources := []job.ResourceURN{"resource-B", "resource-D"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobASources, false) unresolvedUpstreams := []*job.Upstream{ job.NewUpstreamUnresolvedStatic("job-C", project.Name()), - job.NewUpstreamUnresolvedInferred(resourceURNB), - job.NewUpstreamUnresolvedInferred(resourceURND), + job.NewUpstreamUnresolvedInferred("resource-B"), + job.NewUpstreamUnresolvedInferred("resource-D"), } - internalUpstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "inferred", taskName, false) - internalUpstreamC := job.NewUpstreamResolved("job-C", "", resourceURNC, sampleTenant, "static", taskName, false) - externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", resourceURND, externalTenant, "inferred", taskName, true) + internalUpstreamB := job.NewUpstreamResolved("job-B", "", "resource-B", sampleTenant, "inferred", taskName, false) + internalUpstreamC := job.NewUpstreamResolved("job-C", "", "resource-C", sampleTenant, "static", taskName, false) + externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", "resource-D", externalTenant, "inferred", taskName, true) jobWithInternalUpstream := job.NewWithUpstream(jobA, []*job.Upstream{internalUpstreamC, internalUpstreamB, unresolvedUpstreams[2]}) jobWithUnresolvedUpstream := job.NewWithUpstream(jobA, unresolvedUpstreams) @@ -467,17 +453,17 @@ func TestUpstreamResolver(t *testing.T) { specA, err := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithSpecUpstream(jobAUpstreamSpec).Build() assert.NoError(t, err) - jobADestination := resourceURNA - jobASources := []resource.URN{resourceURNB, resourceURND} + jobADestination := job.ResourceURN("resource-A") + jobASources := []job.ResourceURN{"resource-B", "resource-D"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobASources, false) unresolvedUpstreams := []*job.Upstream{ - job.NewUpstreamUnresolvedInferred(resourceURNB), - job.NewUpstreamUnresolvedInferred(resourceURND), + job.NewUpstreamUnresolvedInferred("resource-B"), + job.NewUpstreamUnresolvedInferred("resource-D"), } - internalUpstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "inferred", taskName, false) - externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", resourceURND, externalTenant, "inferred", taskName, true) + internalUpstreamB := job.NewUpstreamResolved("job-B", "", "resource-B", sampleTenant, "inferred", taskName, false) + externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", "resource-D", externalTenant, "inferred", taskName, true) jobWithInternalUpstream := job.NewWithUpstream(jobA, []*job.Upstream{internalUpstreamB, unresolvedUpstreams[1]}) jobWithUnresolvedUpstream := job.NewWithUpstream(jobA, []*job.Upstream{unresolvedUpstreams[0], unresolvedUpstreams[1]}) @@ -508,18 +494,18 @@ func TestUpstreamResolver(t *testing.T) { specA, err := job.NewSpecBuilder(jobVersion, "job-A", sampleOwner, jobSchedule, jobWindow, jobTask).WithSpecUpstream(jobAUpstreamSpec).Build() assert.NoError(t, err) - jobADestination := resourceURNA - jobASources := []resource.URN{resourceURNB, resourceURND} + jobADestination := job.ResourceURN("resource-A") + jobASources := []job.ResourceURN{"resource-B", "resource-D"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobASources, false) unresolvedUpstreams := []*job.Upstream{ job.NewUpstreamUnresolvedStatic("job-C", project.Name()), - job.NewUpstreamUnresolvedInferred(resourceURNB), - job.NewUpstreamUnresolvedInferred(resourceURND), + job.NewUpstreamUnresolvedInferred("resource-B"), + job.NewUpstreamUnresolvedInferred("resource-D"), } - internalUpstreamC := job.NewUpstreamResolved("job-C", "", resourceURNC, sampleTenant, "static", taskName, false) - externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", resourceURND, externalTenant, "inferred", taskName, true) + internalUpstreamC := job.NewUpstreamResolved("job-C", "", "resource-C", sampleTenant, "static", taskName, false) + externalUpstreamD := job.NewUpstreamResolved("job-D", "external-host", "resource-D", externalTenant, "inferred", taskName, true) jobWithUnresolvedUpstream := job.NewWithUpstream(jobA, unresolvedUpstreams) jobWithInternalUpstream := job.NewWithUpstream(jobA, unresolvedUpstreams) @@ -647,11 +633,11 @@ type JobRepository struct { } // GetAllByResourceDestination provides a mock function with given fields: ctx, resourceDestination -func (_m *JobRepository) GetAllByResourceDestination(ctx context.Context, resourceDestination resource.URN) ([]*job.Job, error) { +func (_m *JobRepository) GetAllByResourceDestination(ctx context.Context, resourceDestination job.ResourceURN) ([]*job.Job, error) { ret := _m.Called(ctx, resourceDestination) var r0 []*job.Job - if rf, ok := ret.Get(0).(func(context.Context, resource.URN) []*job.Job); ok { + if rf, ok := ret.Get(0).(func(context.Context, job.ResourceURN) []*job.Job); ok { r0 = rf(ctx, resourceDestination) } else { if ret.Get(0) != nil { @@ -660,7 +646,7 @@ func (_m *JobRepository) GetAllByResourceDestination(ctx context.Context, resour } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, resource.URN) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, job.ResourceURN) error); ok { r1 = rf(ctx, resourceDestination) } else { r1 = ret.Error(1) diff --git a/core/job/service/job_service.go b/core/job/service/job_service.go index 8bb854e5af..e7046c7816 100644 --- a/core/job/service/job_service.go +++ b/core/job/service/job_service.go @@ -5,7 +5,6 @@ import ( "fmt" "reflect" "strings" - "time" "github.com/goto/salt/log" "github.com/kushsharma/parallel" @@ -13,10 +12,7 @@ import ( "github.com/goto/optimus/core/event" "github.com/goto/optimus/core/event/moderator" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/job/dto" "github.com/goto/optimus/core/job/service/filter" - "github.com/goto/optimus/core/resource" - "github.com/goto/optimus/core/scheduler" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/compiler" "github.com/goto/optimus/internal/errors" @@ -50,9 +46,6 @@ type JobService struct { jobDeploymentService JobDeploymentService engine Engine - jobRunInputCompiler JobRunInputCompiler - resourceChecker ResourceExistenceChecker - logger log.Logger } @@ -61,7 +54,6 @@ func NewJobService( pluginService PluginService, upstreamResolver UpstreamResolver, tenantDetailsGetter TenantDetailsGetter, eventHandler EventHandler, logger log.Logger, jobDeploymentService JobDeploymentService, engine Engine, - jobInputCompiler JobRunInputCompiler, resourceChecker ResourceExistenceChecker, ) *JobService { return &JobService{ jobRepo: jobRepo, @@ -74,8 +66,6 @@ func NewJobService( logger: logger, jobDeploymentService: jobDeploymentService, engine: engine, - jobRunInputCompiler: jobInputCompiler, - resourceChecker: resourceChecker, } } @@ -86,8 +76,8 @@ type Engine interface { type PluginService interface { Info(ctx context.Context, taskName string) (*plugin.Info, error) - IdentifyUpstreams(ctx context.Context, taskName string, compiledConfig, assets map[string]string) (resourceURNs []resource.URN, err error) - ConstructDestinationURN(ctx context.Context, taskName string, compiledConfig map[string]string) (destinationURN resource.URN, err error) + IdentifyUpstreams(ctx context.Context, taskName string, compiledConfig, assets map[string]string) (resourceURNs []string, err error) + ConstructDestinationURN(ctx context.Context, taskName string, compiledConfig map[string]string) (destinationURN string, err error) } type TenantDetailsGetter interface { @@ -108,7 +98,7 @@ type JobRepository interface { ChangeJobNamespace(ctx context.Context, jobName job.Name, tenant, newTenant tenant.Tenant) error GetByJobName(ctx context.Context, projectName tenant.ProjectName, jobName job.Name) (*job.Job, error) - GetAllByResourceDestination(ctx context.Context, resourceDestination resource.URN) ([]*job.Job, error) + GetAllByResourceDestination(ctx context.Context, resourceDestination job.ResourceURN) ([]*job.Job, error) GetAllByTenant(ctx context.Context, jobTenant tenant.Tenant) ([]*job.Job, error) GetAllByProjectName(ctx context.Context, projectName tenant.ProjectName) ([]*job.Job, error) SyncState(ctx context.Context, jobTenant tenant.Tenant, disabledJobNames, enabledJobNames []job.Name) error @@ -122,9 +112,9 @@ type UpstreamRepository interface { } type DownstreamRepository interface { - GetDownstreamByDestination(ctx context.Context, projectName tenant.ProjectName, destination resource.URN) ([]*job.Downstream, error) + GetDownstreamByDestination(ctx context.Context, projectName tenant.ProjectName, destination job.ResourceURN) ([]*job.Downstream, error) GetDownstreamByJobName(ctx context.Context, projectName tenant.ProjectName, jobName job.Name) ([]*job.Downstream, error) - GetDownstreamBySources(ctx context.Context, sources []resource.URN) ([]*job.Downstream, error) + GetDownstreamBySources(ctx context.Context, sources []job.ResourceURN) ([]*job.Downstream, error) } type EventHandler interface { @@ -137,15 +127,6 @@ type UpstreamResolver interface { Resolve(ctx context.Context, subjectJob *job.Job, logWriter writer.LogWriter) ([]*job.Upstream, error) } -type JobRunInputCompiler interface { - Compile(ctx context.Context, job *scheduler.JobWithDetails, config scheduler.RunConfig, executedAt time.Time) (*scheduler.ExecutorInput, error) -} - -type ResourceExistenceChecker interface { - GetByURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (*resource.Resource, error) - ExistInStore(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) -} - func (j *JobService) Add(ctx context.Context, jobTenant tenant.Tenant, specs []*job.Spec) error { logWriter := writer.NewLogWriter(j.logger) me := errors.NewMultiError("add specs errors") @@ -357,12 +338,8 @@ func (j *JobService) GetByFilter(ctx context.Context, filters ...filter.FilterOp if f.Contains(filter.ResourceDestination) { j.logger.Debug("getting all jobs by resource destination [%s]", f.GetStringValue(filter.ResourceDestination)) - resourceDestinationURN, err := resource.ParseURN(f.GetStringValue(filter.ResourceDestination)) - if err != nil { - return nil, err - } - - return j.jobRepo.GetAllByResourceDestination(ctx, resourceDestinationURN) + resourceDestination := job.ResourceURN(f.GetStringValue(filter.ResourceDestination)) + return j.jobRepo.GetAllByResourceDestination(ctx, resourceDestination) } // when project name and job names exist, filter by project and job names @@ -648,7 +625,7 @@ func (j *JobService) Refresh(ctx context.Context, projectName tenant.ProjectName return me.ToErr() } -func (j *JobService) RefreshResourceDownstream(ctx context.Context, resourceURNs []resource.URN, logWriter writer.LogWriter) error { +func (j *JobService) RefreshResourceDownstream(ctx context.Context, resourceURNs []job.ResourceURN, logWriter writer.LogWriter) error { downstreams, err := j.downstreamRepo.GetDownstreamBySources(ctx, resourceURNs) if err != nil { j.logger.Error("error identifying job downstream for given resources: %s", err) @@ -682,6 +659,76 @@ func (j *JobService) RefreshResourceDownstream(ctx context.Context, resourceURNs return me.ToErr() } +func (j *JobService) Validate(ctx context.Context, jobTenant tenant.Tenant, jobSpecs []*job.Spec, jobNamesWithInvalidSpec []job.Name, logWriter writer.LogWriter) error { + tenantWithDetails, err := j.tenantDetailsGetter.GetDetails(ctx, jobTenant) + if err != nil { + j.logger.Error("error getting tenant details: %s", err) + return err + } + + if err := job.Specs(jobSpecs).Validate(); err != nil { + return err + } + + validatedJobSpecs := job.Specs(jobSpecs).GetValid() + existingJobs, err := j.jobRepo.GetAllByTenant(ctx, jobTenant) + if err != nil { + return err + } + + toAdd, toUpdate, toDelete, unmodifiedSpecs, unmodifiedDirtySpecs := j.differentiateSpecs(jobTenant, existingJobs, validatedJobSpecs, jobNamesWithInvalidSpec) + logWriter.Write(writer.LogLevelInfo, fmt.Sprintf("[%s] found %d new, %d modified, %d deleted specs", jobTenant.NamespaceName().String(), len(toAdd), len(toUpdate), len(toDelete))) + if len(unmodifiedDirtySpecs) > 0 { + j.logFoundDirtySpecs(tenantWithDetails.ToTenant(), unmodifiedDirtySpecs, logWriter) + } + if len(toAdd)+len(toUpdate)+len(toDelete)+len(unmodifiedDirtySpecs) == 0 { + return nil + } + + me := errors.NewMultiError("validate specs errors") + // TODO: add dry_run check because, generate jobs does not go a Dry Run , and Invalid SQL schemas could not be detected without that. + incomingJobs, err := j.generateJobs(ctx, tenantWithDetails, append(toAdd, append(unmodifiedDirtySpecs, toUpdate...)...), logWriter) + me.Append(err) + + err = j.upstreamResolver.CheckStaticResolvable(ctx, jobTenant, incomingJobs, logWriter) + me.Append(err) + + // TODO: resolve job dependencies before check cyclic + err = j.validateDeleteJobs(ctx, jobTenant, toDelete, logWriter) + me.Append(err) + + // NOTE: only check cyclic deps across internal upstreams (sources), need further discussion to check cyclic deps for external upstream + // assumption, all job specs from input are also the job within same project + jobsToValidateMap := getAllJobsToValidateMap(incomingJobs, existingJobs, append(unmodifiedSpecs, unmodifiedDirtySpecs...)) + identifierToJobsMap := getIdentifierToJobsMap(jobsToValidateMap) + for _, jobEntity := range incomingJobs { + if _, err := j.validateCyclic(jobEntity.Spec().Name(), jobsToValidateMap, identifierToJobsMap); err != nil { + j.logger.Error("error when executing cyclic validation on [%s]: %s", jobEntity.Spec().Name(), err) + me.Append(err) + break + } + } + + return me.ToErr() +} + +func (j *JobService) validateDeleteJobs(ctx context.Context, jobTenant tenant.Tenant, toDelete []*job.Spec, logWriter writer.LogWriter) error { + me := errors.NewMultiError("delete job specs check errors") + toDeleteMap := job.Specs(toDelete).ToFullNameAndSpecMap(jobTenant.ProjectName()) + + for _, jobToDelete := range toDelete { + downstreams, err := j.getAllDownstreams(ctx, jobTenant.ProjectName(), jobToDelete.Name(), map[job.FullName]bool{}) + if err != nil { + j.logger.Error("error getting all downstreams for job [%s]: %s", jobToDelete.Name().String(), err) + logWriter.Write(writer.LogLevelError, fmt.Sprintf("[%s] pre-delete check for job %s failed: %s", jobTenant.NamespaceName().String(), jobToDelete.Name().String(), err.Error())) + me.Append(err) + continue + } + validateDeleteJob(jobTenant, downstreams, toDeleteMap, jobToDelete, logWriter, me) + } + return me.ToErr() +} + func validateDeleteJob(jobTenant tenant.Tenant, downstreams []*job.Downstream, toDeleteMap map[job.FullName]*job.Spec, jobToDelete *job.Spec, logWriter writer.LogWriter, me *errors.MultiError) bool { notDeleted, safeToDelete := isJobSafeToDelete(toDeleteMap, job.DownstreamList(downstreams).GetDownstreamFullNames()) @@ -731,7 +778,34 @@ func (j *JobService) getAllDownstreams(ctx context.Context, projectName tenant.P return downstreams, nil } -func (*JobService) getIdentifierToJobsMap(jobsToValidateMap map[job.Name]*job.WithUpstream) map[string][]*job.WithUpstream { +func getAllJobsToValidateMap(incomingJobs, existingJobs []*job.Job, unmodifiedSpecs []*job.Spec) map[job.Name]*job.WithUpstream { + // TODO: check whether we need to accumulate encountered errors + me := errors.NewMultiError("validate specs errors") + + existingJobMap := job.Jobs(existingJobs).GetNameAndJobMap() + var unmodifiedJobs []*job.Job + for _, unmodifiedSpec := range unmodifiedSpecs { + if unmodifiedJob, ok := existingJobMap[unmodifiedSpec.Name()]; ok { + unmodifiedJobs = append(unmodifiedJobs, unmodifiedJob) + continue + } + errorsMsg := fmt.Sprintf("unable to validate existing job %s", unmodifiedSpec.Name().String()) + me.Append(errors.NewError(errors.ErrInternalError, job.EntityJob, errorsMsg)) + } + + jobsToValidateMap := make(map[job.Name]*job.WithUpstream) + for _, jobToValidate := range append(incomingJobs, unmodifiedJobs...) { + jobWithUpstream, err := jobToValidate.GetJobWithUnresolvedUpstream() + if err != nil { + me.Append(err) + continue + } + jobsToValidateMap[jobToValidate.Spec().Name()] = jobWithUpstream + } + return jobsToValidateMap +} + +func getIdentifierToJobsMap(jobsToValidateMap map[job.Name]*job.WithUpstream) map[string][]*job.WithUpstream { identifierToJobsMap := make(map[string][]*job.WithUpstream) for _, jobEntity := range jobsToValidateMap { jobIdentifiers := []string{jobEntity.Job().FullName()} @@ -1002,6 +1076,11 @@ func (j *JobService) generateJob(ctx context.Context, tenantWithDetails *tenant. return job.NewJob(tenantWithDetails.ToTenant(), spec, destination, sources, false), nil } +func (j *JobService) validateCyclic(rootName job.Name, jobMap map[job.Name]*job.WithUpstream, identifierToJobMap map[string][]*job.WithUpstream) ([]string, error) { + dagTree := j.buildDAGTree(rootName, jobMap, identifierToJobMap) + return dagTree.ValidateCyclic() +} + func (*JobService) buildDAGTree(rootName job.Name, jobMap map[job.Name]*job.WithUpstream, identifierToJobMap map[string][]*job.WithUpstream) *tree.MultiRootTree { rootJob := jobMap[rootName] @@ -1174,621 +1253,42 @@ func raiseJobEventMetric(jobTenant tenant.Tenant, state string, metricValue int) }).Add(float64(metricValue)) } -func (j *JobService) identifyUpstreamURNs(ctx context.Context, tenantWithDetails *tenant.WithDetails, spec *job.Spec) ([]resource.URN, error) { +func (j *JobService) identifyUpstreamURNs(ctx context.Context, tenantWithDetails *tenant.WithDetails, spec *job.Spec) ([]job.ResourceURN, error) { taskName := spec.Task().Name().String() taskConfig := spec.Task().Config() compileConfigs := j.compileConfigs(taskConfig, tenantWithDetails) assets := spec.Asset() - return j.pluginService.IdentifyUpstreams(ctx, taskName, compileConfigs, assets) -} - -func (j *JobService) generateDestinationURN(ctx context.Context, tenantWithDetails *tenant.WithDetails, spec *job.Spec) (resource.URN, error) { - taskName := spec.Task().Name().String() - taskConfig := spec.Task().Config() - compileConfigs := j.compileConfigs(taskConfig, tenantWithDetails) - - return j.pluginService.ConstructDestinationURN(ctx, taskName, compileConfigs) -} - -func (j *JobService) Validate(ctx context.Context, request dto.ValidateRequest) (map[job.Name][]dto.ValidateResult, error) { - if err := j.validateRequest(request); err != nil { - registerJobValidationMetric(request.Tenant, dto.StagePreparation, false) - return nil, err - } - - if err := j.validateDuplication(request); err != nil { - registerJobValidationMetric(request.Tenant, dto.StagePreparation, false) - return nil, err - } - - tenantDetails, err := j.tenantDetailsGetter.GetDetails(ctx, request.Tenant) - if err != nil { - registerJobValidationMetric(request.Tenant, dto.StagePreparation, false) - return nil, err - } - - jobsToValidate, err := j.getJobsToValidate(ctx, tenantDetails, request) + resourceURNs, err := j.pluginService.IdentifyUpstreams(ctx, taskName, compileConfigs, assets) if err != nil { - registerJobValidationMetric(request.Tenant, dto.StagePreparation, false) return nil, err } - registerJobValidationMetric(request.Tenant, dto.StagePreparation, true) - - if output := j.validateTenantOnEachJob(request.Tenant, jobsToValidate); len(output) > 0 { - return output, nil - } - - if request.DeletionMode { - return j.validateJobsForDeletion(ctx, request.Tenant, jobsToValidate), nil - } - - if result, err := j.validateCyclic(ctx, request.Tenant, jobsToValidate); err != nil { - return nil, err - } else if len(result) > 0 { - return result, nil - } - - return j.validateJobs(ctx, tenantDetails, jobsToValidate) -} - -func (*JobService) validateRequest(request dto.ValidateRequest) error { - if len(request.JobNames) > 0 && len(request.JobSpecs) > 0 { - return errors.NewError(errors.ErrInvalidArgument, job.EntityJob, "job names and specs can not be specified together") - } - - if len(request.JobNames) == 0 && len(request.JobSpecs) == 0 { - return errors.NewError(errors.ErrInvalidArgument, job.EntityJob, "job names and job specs are both empty") - } - - if request.DeletionMode && len(request.JobNames) == 0 { - return errors.NewError(errors.ErrInvalidArgument, job.EntityJob, "deletion job only accepts job names") - } - - return nil -} - -func (*JobService) validateDuplication(request dto.ValidateRequest) error { - jobNameCountMap := make(map[string]int) - if len(request.JobNames) > 0 { - for _, name := range request.JobNames { - jobNameCountMap[name]++ - } - } else { - for _, spec := range request.JobSpecs { - jobNameCountMap[spec.Name().String()]++ - } - } - - var duplicated []string - for name, count := range jobNameCountMap { - if count > 1 { - duplicated = append(duplicated, name) - } - } - - if len(duplicated) > 0 { - message := fmt.Sprintf("the following jobs are duplicated: [%s]", strings.Join(duplicated, ", ")) - return errors.NewError(errors.ErrInvalidArgument, job.EntityJob, message) - } - - return nil -} - -func (j *JobService) getJobsToValidate(ctx context.Context, tenantDetails *tenant.WithDetails, request dto.ValidateRequest) ([]*job.Job, error) { - if len(request.JobNames) > 0 { - return j.getJobByNames(ctx, request.Tenant, request.JobNames) - } - - return j.generateJobs(ctx, tenantDetails, request.JobSpecs, writer.NewSafeBufferedLogger()) -} - -func (j *JobService) getJobByNames(ctx context.Context, tnnt tenant.Tenant, jobNames []string) ([]*job.Job, error) { - me := errors.NewMultiError("getting job by name") - - var retrievedJobs []*job.Job - for _, name := range jobNames { - jobName, err := job.NameFrom(name) - if err != nil { - me.Append(err) - continue - } - - subjectJob, err := j.jobRepo.GetByJobName(ctx, tnnt.ProjectName(), jobName) - if err != nil { - me.Append(err) - continue - } - - retrievedJobs = append(retrievedJobs, subjectJob) - } - - return retrievedJobs, me.ToErr() -} - -func (j *JobService) validateJobsForDeletion(ctx context.Context, jobTenant tenant.Tenant, jobsToDelete []*job.Job) map[job.Name][]dto.ValidateResult { - specByFullName := job.Jobs(jobsToDelete).GetFullNameToSpecMap() - - output := make(map[job.Name][]dto.ValidateResult) - for _, job := range jobsToDelete { - output[job.Spec().Name()] = j.validateOneJobForDeletion(ctx, jobTenant, job.Spec(), specByFullName) + jobResourceURNs := make([]job.ResourceURN, len(resourceURNs)) + for i, resourceURN := range resourceURNs { + jobResourceURNs[i] = job.ResourceURN(resourceURN) } - return output + return jobResourceURNs, nil } -func (j *JobService) validateOneJobForDeletion( - ctx context.Context, - jobTenant tenant.Tenant, spec *job.Spec, - specByFullName map[job.FullName]*job.Spec, -) []dto.ValidateResult { - downstreams, err := j.getAllDownstreams(ctx, jobTenant.ProjectName(), spec.Name(), make(map[job.FullName]bool)) - if err != nil { - result := dto.ValidateResult{ - Stage: dto.StageDeletionValidation, - Messages: []string{ - "downstreams can not be fetched", - err.Error(), - }, - Success: false, - } - - registerJobValidationMetric(jobTenant, dto.StageDeletionValidation, false) - - return []dto.ValidateResult{result} - } - - me := errors.NewMultiError("validating job for deletion errors") - safeToDelete := validateDeleteJob(jobTenant, downstreams, specByFullName, spec, writer.NewSafeBufferedLogger(), me) - - var messages []string - success := true - if !safeToDelete { - messages = []string{"job is not safe for deletion"} - if err := me.ToErr(); err != nil { - messages = append(messages, err.Error()) - } - - success = false - } else { - messages = []string{"job is safe for deletion"} - } - - registerJobValidationMetric(jobTenant, dto.StageDeletionValidation, success) - - return []dto.ValidateResult{ - { - Stage: dto.StageDeletionValidation, - Messages: messages, - Success: success, - }, - } -} - -func (*JobService) validateTenantOnEachJob(rootTnnt tenant.Tenant, jobsToValidate []*job.Job) map[job.Name][]dto.ValidateResult { - output := make(map[job.Name][]dto.ValidateResult) - for _, subjectJob := range jobsToValidate { - tnnt := subjectJob.Tenant() - if tnnt.ProjectName() != rootTnnt.ProjectName() || tnnt.NamespaceName() != rootTnnt.NamespaceName() { - result := dto.ValidateResult{ - Stage: dto.StageTenantValidation, - Messages: []string{ - fmt.Sprintf("current tenant is [%s.%s]", tnnt.ProjectName(), tnnt.NamespaceName()), - fmt.Sprintf("expected tenant is [%s.%s]", rootTnnt.ProjectName(), rootTnnt.NamespaceName()), - }, - Success: false, - } - - output[subjectJob.Spec().Name()] = []dto.ValidateResult{result} - - registerJobValidationMetric(rootTnnt, dto.StageTenantValidation, false) - } else { - registerJobValidationMetric(rootTnnt, dto.StageTenantValidation, true) - } - } - - return output -} - -func (j *JobService) validateJobs(ctx context.Context, tenantDetails *tenant.WithDetails, jobsToValidate []*job.Job) (map[job.Name][]dto.ValidateResult, error) { - output := make(map[job.Name][]dto.ValidateResult) - for _, subjectJob := range jobsToValidate { - output[subjectJob.Spec().Name()] = j.validateOneJob(ctx, tenantDetails, subjectJob) - } - - return output, nil -} - -func (j *JobService) validateOneJob(ctx context.Context, tenantDetails *tenant.WithDetails, subjectJob *job.Job) []dto.ValidateResult { - destination, err := j.generateDestinationURN(ctx, tenantDetails, subjectJob.Spec()) - if err != nil { - result := dto.ValidateResult{ - Stage: dto.StageDestinationValidation, - Messages: []string{ - "can not generate destination resource", - err.Error(), - }, - Success: false, - } - - registerJobValidationMetric(tenantDetails.ToTenant(), dto.StageDestinationValidation, false) - - return []dto.ValidateResult{result} - } - - var output []dto.ValidateResult - - result := j.validateDestination(ctx, tenantDetails.ToTenant(), destination) - output = append(output, result) - - result = j.validateSource(ctx, tenantDetails, subjectJob.Spec()) - output = append(output, result) - - result = j.validateWindow(tenantDetails, subjectJob.Spec().WindowConfig()) - output = append(output, result) - - result = j.validateRun(ctx, subjectJob, destination) - output = append(output, result) - - result = j.validateUpstream(ctx, subjectJob) - output = append(output, result) - - return output -} - -func (j *JobService) validateCyclic(ctx context.Context, tnnt tenant.Tenant, incomingJobs []*job.Job) (map[job.Name][]dto.ValidateResult, error) { - existingJobs, err := j.jobRepo.GetAllByTenant(ctx, tnnt) - if err != nil { - return nil, err - } - - if err := j.upstreamResolver.CheckStaticResolvable(ctx, tnnt, incomingJobs, writer.NewSafeBufferedLogger()); err != nil { - return nil, err - } - - jobsToValidate := j.getCombinedJobsToValidate(incomingJobs, existingJobs) - - jobsWithUpstream, err := j.upstreamResolver.BulkResolve(ctx, tnnt.ProjectName(), jobsToValidate, writer.NewSafeBufferedLogger()) - if err != nil { - return nil, err - } - - // NOTE: only check cyclic deps across internal upstreams (sources), need further discussion to check cyclic deps for external upstream - // assumption, all job specs from input are also the job within same project - jobWithUpstreamPerJobName := j.getJobWithUpstreamPerJobName(jobsWithUpstream) - identifierToJobsMap := j.getIdentifierToJobsMap(jobWithUpstreamPerJobName) - - output := make(map[job.Name][]dto.ValidateResult) - - isJobNameCyclic := make(map[string]bool) - for _, subjectJob := range incomingJobs { - dagTree := j.buildDAGTree(subjectJob.Spec().Name(), jobWithUpstreamPerJobName, identifierToJobsMap) - cyclicNames, err := dagTree.ValidateCyclic() - - for _, name := range cyclicNames { - isJobNameCyclic[name] = true - } - - if err != nil && isJobNameCyclic[subjectJob.GetName()] { - output[subjectJob.Spec().Name()] = []dto.ValidateResult{ - { - Stage: dto.StageCyclicValidation, - Messages: append([]string{ - "cyclic dependency is detected", - }, cyclicNames...), - Success: false, - }, - } - - registerJobValidationMetric(tnnt, dto.StageCyclicValidation, false) - } else { - registerJobValidationMetric(tnnt, dto.StageCyclicValidation, true) - } - } - - return output, nil -} - -func (*JobService) getCombinedJobsToValidate(incoming, existing []*job.Job) []*job.Job { - uniqueJobs := make(map[job.Name]*job.Job) - for _, j := range existing { - uniqueJobs[j.Spec().Name()] = j - } - - for _, j := range incoming { - uniqueJobs[j.Spec().Name()] = j - } - - var output []*job.Job - for _, j := range uniqueJobs { - output = append(output, j) - } - - return output -} - -func (*JobService) getJobWithUpstreamPerJobName(jobsWithUpstream []*job.WithUpstream) map[job.Name]*job.WithUpstream { - jobsToValidateMap := make(map[job.Name]*job.WithUpstream) - for _, jobWithUpstream := range jobsWithUpstream { - jobsToValidateMap[jobWithUpstream.Name()] = jobWithUpstream - } - - return jobsToValidateMap -} - -func (j *JobService) validateUpstream(ctx context.Context, subjectJob *job.Job) dto.ValidateResult { - if _, err := j.upstreamResolver.Resolve(ctx, subjectJob, writer.NewSafeBufferedLogger()); err != nil { - registerJobValidationMetric(subjectJob.Tenant(), dto.StageUpstreamValidation, false) - - return dto.ValidateResult{ - Stage: dto.StageUpstreamValidation, - Messages: []string{ - "can not resolve upstream", - err.Error(), - }, - Success: false, - } - } - - registerJobValidationMetric(subjectJob.Tenant(), dto.StageUpstreamValidation, true) - - return dto.ValidateResult{ - Stage: dto.StageUpstreamValidation, - Messages: []string{"no issue"}, - Success: true, - } -} - -func (j *JobService) validateDestination(ctx context.Context, tnnt tenant.Tenant, destination resource.URN) dto.ValidateResult { - if destination.IsZero() { - registerJobValidationMetric(tnnt, dto.StageDestinationValidation, true) - - return dto.ValidateResult{ - Stage: dto.StageDestinationValidation, - Messages: []string{"no issue"}, - Success: true, - } - } - - message, success := j.validateResourceURN(ctx, tnnt, destination) - registerJobValidationMetric(tnnt, dto.StageDestinationValidation, success) - - return dto.ValidateResult{ - Stage: dto.StageDestinationValidation, - Messages: []string{fmt.Sprintf("%s: %s", destination.String(), message)}, - Success: success, - } -} - -func (j *JobService) validateSource(ctx context.Context, tenantWithDetails *tenant.WithDetails, spec *job.Spec) dto.ValidateResult { - sourceURNs, err := j.identifyUpstreamURNs(ctx, tenantWithDetails, spec) - if err != nil { - registerJobValidationMetric(tenantWithDetails.ToTenant(), dto.StageSourceValidation, false) - - return dto.ValidateResult{ - Stage: dto.StageSourceValidation, - Messages: []string{ - "can not identify the resource sources of the job", - err.Error(), - }, - Success: false, - } - } - - if len(sourceURNs) == 0 { - return dto.ValidateResult{ - Stage: dto.StageSourceValidation, - Messages: []string{"no issue"}, - Success: true, - } - } - - messages := make([]string, len(sourceURNs)) - success := true - - for i, urn := range sourceURNs { - currentMessage, currentSuccess := j.validateResourceURN(ctx, tenantWithDetails.ToTenant(), urn) - if !currentSuccess { - success = false - } - - messages[i] = fmt.Sprintf("%s: %s", urn.String(), currentMessage) - } - - registerJobValidationMetric(tenantWithDetails.ToTenant(), dto.StageSourceValidation, success) - - return dto.ValidateResult{ - Stage: dto.StageSourceValidation, - Messages: messages, - Success: success, - } -} - -func (j *JobService) validateResourceURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (string, bool) { - activeInDB := true - if rsc, err := j.resourceChecker.GetByURN(ctx, tnnt, urn); err != nil { - j.logger.Warn("suppress error is encountered when reading resource from db: %v", err) - activeInDB = false - } else { - switch rsc.Status() { - case resource.StatusToDelete, resource.StatusDeleted: - activeInDB = false - } - } - - existInStore, err := j.resourceChecker.ExistInStore(ctx, tnnt, urn) - if err != nil { - return err.Error(), false - } - - if activeInDB && existInStore { - return "no issue", true - } - - if existInStore { - return "resource exists in store but not in db", true - } - - if activeInDB { - return "resource exists in db but not in store", false - } - - return "resource does not exist in both db and store", false -} - -func (j *JobService) validateRun(ctx context.Context, subjectJob *job.Job, destination resource.URN) dto.ValidateResult { - referenceTime := time.Now() - runConfigs, err := j.getRunConfigs(referenceTime, subjectJob.Spec()) - if err != nil { - registerJobValidationMetric(subjectJob.Tenant(), dto.StageRunCompileValidation, false) - - return dto.ValidateResult{ - Stage: dto.StageRunCompileValidation, - Messages: []string{ - "can not get run config", - err.Error(), - }, - Success: false, - } - } - - var messages []string - success := true - - jobWithDetails := j.getSchedulerJobWithDetail(subjectJob, destination) - for _, config := range runConfigs { - var msg string - if _, err := j.jobRunInputCompiler.Compile(ctx, jobWithDetails, config, referenceTime); err != nil { - success = false - - msg = fmt.Sprintf("compiling [%s] with type [%s] failed with error: %v", config.Executor.Name, config.Executor.Type.String(), err) - } else { - msg = fmt.Sprintf("compiling [%s] with type [%s] contains no issue", config.Executor.Name, config.Executor.Type.String()) - } - - messages = append(messages, msg) - } - - registerJobValidationMetric(subjectJob.Tenant(), dto.StageRunCompileValidation, success) - - return dto.ValidateResult{ - Stage: dto.StageRunCompileValidation, - Messages: messages, - Success: success, - } -} - -func (*JobService) getSchedulerJobWithDetail(subjectJob *job.Job, destination resource.URN) *scheduler.JobWithDetails { - hooks := make([]*scheduler.Hook, len(subjectJob.Spec().Hooks())) - for i, hook := range subjectJob.Spec().Hooks() { - hooks[i] = &scheduler.Hook{ - Name: hook.Name(), - Config: hook.Config(), - } - } - - return &scheduler.JobWithDetails{ - Name: scheduler.JobName(subjectJob.GetName()), - Job: &scheduler.Job{ - Name: scheduler.JobName(subjectJob.GetName()), - Tenant: subjectJob.Tenant(), - Destination: destination, - Task: &scheduler.Task{ - Name: string(subjectJob.Spec().Task().Name()), - Config: subjectJob.Spec().Task().Config(), - }, - Hooks: hooks, - WindowConfig: subjectJob.Spec().WindowConfig(), - Assets: subjectJob.Spec().Asset(), - }, - JobMetadata: &scheduler.JobMetadata{ - Version: subjectJob.Spec().Version(), - Owner: subjectJob.Spec().Owner(), - Description: subjectJob.Spec().Description(), - Labels: subjectJob.Spec().Labels(), - }, - Schedule: &scheduler.Schedule{ - DependsOnPast: subjectJob.Spec().Schedule().DependsOnPast(), - CatchUp: subjectJob.Spec().Schedule().CatchUp(), - Interval: subjectJob.Spec().Schedule().Interval(), - }, - } -} - -func (*JobService) getRunConfigs(referenceTime time.Time, spec *job.Spec) ([]scheduler.RunConfig, error) { - var runConfigs []scheduler.RunConfig - - executor, err := scheduler.ExecutorFromEnum(spec.Task().Name().String(), scheduler.ExecutorTask.String()) - if err != nil { - return nil, err - } +func (j *JobService) generateDestinationURN(ctx context.Context, tenantWithDetails *tenant.WithDetails, spec *job.Spec) (job.ResourceURN, error) { + taskName := spec.Task().Name().String() + taskConfig := spec.Task().Config() + compileConfigs := j.compileConfigs(taskConfig, tenantWithDetails) - runConfig, err := scheduler.RunConfigFrom(executor, referenceTime, "") + destinationURN, err := j.pluginService.ConstructDestinationURN(ctx, taskName, compileConfigs) if err != nil { - return nil, err - } - - runConfigs = append(runConfigs, runConfig) - - for _, hook := range spec.Hooks() { - executor, err := scheduler.ExecutorFromEnum(hook.Name(), scheduler.ExecutorHook.String()) - if err != nil { - return nil, err - } - - runConfig, err := scheduler.RunConfigFrom(executor, referenceTime, "") - if err != nil { - return nil, err - } - - runConfigs = append(runConfigs, runConfig) - } - - return runConfigs, nil -} - -func (*JobService) validateWindow(tenantDetails *tenant.WithDetails, windowConfig window.Config) dto.ValidateResult { - if windowType := windowConfig.Type(); windowType == window.Preset { - preset := windowConfig.Preset - if _, err := tenantDetails.Project().GetPreset(preset); err != nil { - registerJobValidationMetric(tenantDetails.ToTenant(), dto.StageWindowValidation, false) - - return dto.ValidateResult{ - Stage: dto.StageWindowValidation, - Messages: []string{ - fmt.Sprintf("window preset [%s] is not found within project", preset), - err.Error(), - }, - Success: false, - } - } - } - - registerJobValidationMetric(tenantDetails.ToTenant(), dto.StageWindowValidation, true) - - return dto.ValidateResult{ - Stage: dto.StageWindowValidation, - Messages: []string{"no issue"}, - Success: true, + return "", err } -} - -func registerJobValidationMetric(tnnt tenant.Tenant, stage dto.ValidateStage, success bool) { - counter := telemetry.NewCounter(job.MetricJobValidation, map[string]string{ - "project": tnnt.ProjectName().String(), - "namespace": tnnt.NamespaceName().String(), - "stage": stage.String(), - "success": fmt.Sprintf("%t", success), - }) - counter.Add(1) + return job.ResourceURN(destinationURN), nil } -func (j *JobService) GetDownstreamByResourceURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (job.DownstreamList, error) { +func (j *JobService) GetDownstreamByResourceURN(ctx context.Context, tnnt tenant.Tenant, urn job.ResourceURN) (job.DownstreamList, error) { var dependentJobs []*job.Downstream - jobs, err := j.downstreamRepo.GetDownstreamBySources(ctx, []resource.URN{urn}) + jobs, err := j.downstreamRepo.GetDownstreamBySources(ctx, []job.ResourceURN{urn}) if err != nil { return nil, err } diff --git a/core/job/service/job_service_test.go b/core/job/service/job_service_test.go index c5c4f70466..843b1c5e0a 100644 --- a/core/job/service/job_service_test.go +++ b/core/job/service/job_service_test.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "testing" - time "time" "github.com/goto/salt/log" "github.com/stretchr/testify/assert" @@ -13,11 +12,8 @@ import ( "github.com/goto/optimus/core/event/moderator" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/job/dto" "github.com/goto/optimus/core/job/service" "github.com/goto/optimus/core/job/service/filter" - resource "github.com/goto/optimus/core/resource" - scheduler "github.com/goto/optimus/core/scheduler" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/compiler" optErrors "github.com/goto/optimus/internal/errors" @@ -77,14 +73,10 @@ func TestJobService(t *testing.T) { var jobNamesWithInvalidSpec []job.Name var emptyJobNames []string - resourceURNA, err := resource.ParseURN("bigquery://project:dataset.tableA") - assert.NoError(t, err) - resourceURNB, err := resource.ParseURN("bigquery://project:dataset.tableB") - assert.NoError(t, err) - resourceURNC, err := resource.ParseURN("bigquery://project:dataset.tableC") - assert.NoError(t, err) - resourceURND, err := resource.ParseURN("bigquery://project:dataset.tableD") - assert.NoError(t, err) + resourceURNA := job.ResourceURN("bigquery://project:dataset.tableA") + resourceURNB := job.ResourceURN("bigquery://project:dataset.tableB") + resourceURNC := job.ResourceURN("bigquery://project:dataset.tableC") + resourceURND := job.ResourceURN("bigquery://project:dataset.tableD") t.Run("Add", func(t *testing.T) { t.Run("add jobs", func(t *testing.T) { @@ -97,7 +89,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -115,17 +108,17 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - jobADestination := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil) + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil) - jobAUpstreamName := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) jobs := []*job.Job{jobA} jobRepo.On("Add", ctx, mock.Anything).Return(jobs, nil, nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), jobs, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -136,7 +129,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Add(ctx, sampleTenant, specs) assert.NoError(t, err) }) @@ -150,7 +143,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -163,7 +157,7 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(&tenant.WithDetails{}, errors.New("internal error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) err := jobService.Add(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "internal error") }) @@ -177,7 +171,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -197,22 +192,22 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - jobADestination := resourceURNA - jobBDestination := resourceURNB - var jobDestination resource.URN - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specC.Task().Name().String(), mock.Anything).Return(jobDestination, errors.New("generate destination error")).Once() + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobBDestination := job.ResourceURN("bigquery://project:dataset.tableB") + var jobDestination job.ResourceURN + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specC.Task().Name().String(), mock.Anything).Return(jobDestination.String(), errors.New("generate destination error")).Once() - jobAUpstreamName := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return([]resource.URN{}, errors.New("generate upstream error")) - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return([]string{}, errors.New("generate upstream error")) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) jobs := []*job.Job{jobA} jobRepo.On("Add", ctx, mock.Anything).Return(jobs, nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), jobs, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -223,7 +218,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Add(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "generate upstream error") }) @@ -237,7 +232,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -257,14 +253,14 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - var jobADestination resource.URN - jobBDestination := resourceURNB - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, errors.New("generate destination error")).Once() + var jobADestination job.ResourceURN + jobBDestination := job.ResourceURN("bigquery://project:dataset.tableB") + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), errors.New("generate destination error")).Once() pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(nil, errors.New("generate upstream error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) err := jobService.Add(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "generate upstream error") }) @@ -278,7 +274,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -297,13 +294,13 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - jobA := job.NewJob(sampleTenant, specA, resource.ZeroURN(), nil, false) + jobA := job.NewJob(sampleTenant, specA, "", nil, false) jobs := []*job.Job{jobA} jobRepo.On("Add", ctx, mock.Anything).Return(jobs, nil) - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resource.ZeroURN(), nil) - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return([]resource.URN{}, nil) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return("", nil) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return([]string{}, nil) jobWithUpstream := job.NewWithUpstream(jobA, nil) upstreamResolver.On("BulkResolve", ctx, project.Name(), jobs, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -314,7 +311,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Add(ctx, sampleTenant, specs) assert.NoError(t, err) }) @@ -328,7 +325,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -347,16 +345,16 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - resourceA := resourceURNA - var resourceB resource.URN - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(resourceB, errors.New("some error")).Once() + resourceA := job.ResourceURN("bigquery://project:dataset.tableA") + var resourceB job.ResourceURN + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(resourceB.String(), errors.New("some error")).Once() - jobSourcesA := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobSourcesA, nil) + jobSourcesA := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobSourcesA), nil) pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(nil, errors.New("another error")) - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) savedJobs := []*job.Job{jobB} jobRepo.On("Add", ctx, mock.Anything).Return(savedJobs, errors.New("unable to save job A")) @@ -370,7 +368,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Add(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "unable to save job A") }) @@ -384,7 +382,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -401,15 +400,15 @@ func TestJobService(t *testing.T) { upstreamRepo.On("ReplaceUpstreams", ctx, mock.Anything).Return(nil) - resourceA := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA, nil).Once() + resourceA := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA.String(), nil).Once() - jobSourcesA := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobSourcesA, nil) + jobSourcesA := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobSourcesA), nil) jobRepo.On("Add", ctx, mock.Anything).Return([]*job.Job{}, errors.New("unable to save job A"), errors.New("all jobs failed")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) err := jobService.Add(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "unable to save job A") }) @@ -423,7 +422,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -441,11 +441,11 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - resourceA := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA, nil).Once() + resourceA := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA.String(), nil).Once() - jobSourcesA := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobSourcesA, nil) + jobSourcesA := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobSourcesA), nil) jobA := job.NewJob(sampleTenant, specA, resourceA, jobSourcesA, false) jobs := []*job.Job{jobA} @@ -462,7 +462,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Add(ctx, sampleTenant, specs) assert.Error(t, err) }) @@ -476,7 +476,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -494,17 +495,17 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - jobADestination := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil) + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil) - jobAUpstreamName := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) jobs := []*job.Job{jobA} jobRepo.On("Add", ctx, mock.Anything).Return(jobs, nil, nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), jobs, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -515,7 +516,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Add(ctx, sampleTenant, specs) assert.ErrorContains(t, err, errorMsg) }) @@ -531,7 +532,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -549,18 +551,18 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - jobADestination := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil) + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil) - jobAUpstreamName := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) jobs := []*job.Job{jobA} jobRepo.On("Update", ctx, mock.Anything).Return(jobs, nil, nil) jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(jobA, nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), jobs, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -571,7 +573,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Update(ctx, sampleTenant, specs) assert.NoError(t, err) }) @@ -585,7 +587,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -598,7 +601,7 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(&tenant.WithDetails{}, errors.New("internal error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) err := jobService.Update(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "internal error") }) @@ -612,7 +615,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -635,15 +639,15 @@ func TestJobService(t *testing.T) { jobADestination := resourceURNA jobBDestination := resourceURNB jobCDestination := resourceURNC - var jobDestination resource.URN - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specC.Task().Name().String(), mock.Anything).Return(jobDestination, errors.New("generate destination error")).Once() - - jobAUpstreamName := []resource.URN{resourceURNB} - jobBUpstreamName := []resource.URN{resourceURNC} - jobCUpstreamName := []resource.URN{resourceURND} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return([]resource.URN{}, errors.New("generate upstream error")) + var jobDestination job.ResourceURN + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specC.Task().Name().String(), mock.Anything).Return(jobDestination.String(), errors.New("generate destination error")).Once() + + jobAUpstreamName := []job.ResourceURN{resourceURNB} + jobBUpstreamName := []job.ResourceURN{resourceURNC} + jobCUpstreamName := []job.ResourceURN{resourceURND} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return([]string{}, errors.New("generate upstream error")) pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) @@ -655,7 +659,7 @@ func TestJobService(t *testing.T) { jobRepo.On("GetByJobName", ctx, project.Name(), specB.Name()).Return(jobB, nil) jobRepo.On("GetByJobName", ctx, project.Name(), specC.Name()).Return(jobC, nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), jobs, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -666,7 +670,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Update(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "generate upstream error") }) @@ -680,7 +684,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -690,8 +695,8 @@ func TestJobService(t *testing.T) { specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) - jobB := job.NewJob(sampleTenant, specA, resourceURNB, []resource.URN{resourceURNA}, false) + jobA := job.NewJob(sampleTenant, specA, resourceURNA, []job.ResourceURN{resourceURNB}, false) + jobB := job.NewJob(sampleTenant, specA, resourceURNB, []job.ResourceURN{resourceURNA}, false) specs := []*job.Spec{specB, specA} jobRepo.On("Update", ctx, mock.Anything).Return(nil, nil) @@ -704,14 +709,14 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - var jobADestination resource.URN - jobBDestination := resourceURNB - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, errors.New("generate destination error")).Once() + var jobADestination job.ResourceURN + jobBDestination := job.ResourceURN("bigquery://project:dataset.tableB") + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), errors.New("generate destination error")).Once() pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(nil, errors.New("generate upstream error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) err := jobService.Update(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "generate upstream error") }) @@ -725,7 +730,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -744,14 +750,14 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - jobA := job.NewJob(sampleTenant, specA, resource.ZeroURN(), nil, false) + jobA := job.NewJob(sampleTenant, specA, "", nil, false) jobs := []*job.Job{jobA} jobRepo.On("Update", ctx, mock.Anything).Return(jobs, nil, nil) jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(jobA, nil) - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resource.ZeroURN(), nil) - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return([]resource.URN{}, nil) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return("", nil) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return([]string{}, nil) jobWithUpstream := job.NewWithUpstream(jobA, nil) upstreamResolver.On("BulkResolve", ctx, project.Name(), jobs, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -762,7 +768,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Update(ctx, sampleTenant, specs) assert.NoError(t, err) }) @@ -776,7 +782,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -795,18 +802,18 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - resourceA := resourceURNA - var jobDestination resource.URN - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobDestination, errors.New("something error")).Once() + resourceA := job.ResourceURN("bigquery://project:dataset.tableA") + var jobDestination job.ResourceURN + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobDestination.String(), errors.New("something error")).Once() - jobSourcesA := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobSourcesA, nil) + jobSourcesA := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobSourcesA), nil) jobADestination := resourceURNA - jobAUpstreamName := []resource.URN{resourceURNB} + jobAUpstreamName := []job.ResourceURN{resourceURNB} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) savedJobs := []*job.Job{jobB} jobRepo.On("Update", ctx, mock.Anything).Return(savedJobs, errors.New("unable to save job A"), nil) jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(jobA, nil) @@ -822,7 +829,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Update(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "unable to save job A") }) @@ -836,7 +843,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -853,19 +861,19 @@ func TestJobService(t *testing.T) { upstreamRepo.On("ReplaceUpstreams", ctx, mock.Anything).Return(nil) - resourceA := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA, nil).Once() + resourceA := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA.String(), nil).Once() - jobSourcesA := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobSourcesA, nil) + jobSourcesA := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobSourcesA), nil) jobADestination := resourceURNA - jobAUpstreamName := []resource.URN{resourceURNB} + jobAUpstreamName := []job.ResourceURN{resourceURNB} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) jobRepo.On("Update", ctx, mock.Anything).Return([]*job.Job{}, errors.New("unable to update job A"), errors.New("all jobs failed")) jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(jobA, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) err := jobService.Update(ctx, sampleTenant, specs) assert.ErrorContains(t, err, "unable to update job A") }) @@ -879,7 +887,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -897,11 +906,11 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - resourceA := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA, nil).Once() + resourceA := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(resourceA.String(), nil).Once() - jobSourcesA := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobSourcesA, nil) + jobSourcesA := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobSourcesA), nil) jobA := job.NewJob(sampleTenant, specA, resourceA, jobSourcesA, false) jobs := []*job.Job{jobA} @@ -918,7 +927,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Update(ctx, sampleTenant, specs) assert.Error(t, err) }) @@ -932,7 +941,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -950,18 +960,18 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - jobADestination := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil) + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil) - jobAUpstreamName := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) jobs := []*job.Job{jobA} jobRepo.On("Update", ctx, mock.Anything).Return(jobs, nil, nil) jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(jobA, nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), jobs, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -972,7 +982,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Update(ctx, sampleTenant, specs) assert.ErrorContains(t, err, errorMsg) }) @@ -982,14 +992,13 @@ func TestJobService(t *testing.T) { newNamespaceName := "newNamespace" newTenant, _ := tenant.NewTenant(project.Name().String(), newNamespaceName) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(newTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) - + jobA := job.NewJob(newTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) t.Run("should fail if error in repo", func(t *testing.T) { jobRepo := new(JobRepository) jobRepo.On("ChangeJobNamespace", ctx, specA.Name(), sampleTenant, newTenant).Return(errors.New("error in transaction")) defer jobRepo.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, nil, nil) err := jobService.ChangeNamespace(ctx, sampleTenant, newTenant, specA.Name()) assert.ErrorContains(t, err, "error in transaction") }) @@ -1000,7 +1009,7 @@ func TestJobService(t *testing.T) { jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(nil, errors.New("error in fetching job from DB")) defer jobRepo.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, nil, nil) err := jobService.ChangeNamespace(ctx, sampleTenant, newTenant, specA.Name()) assert.ErrorContains(t, err, "error in fetching job from DB") }) @@ -1016,7 +1025,7 @@ func TestJobService(t *testing.T) { jobDeploymentService.On("UploadJobs", ctx, sampleTenant, emptyJobNames, jobModified).Return(errors.New("error in upload jobs")) defer jobDeploymentService.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, jobDeploymentService, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, jobDeploymentService, nil) err := jobService.ChangeNamespace(ctx, sampleTenant, newTenant, specA.Name()) assert.ErrorContains(t, err, "error in upload jobs") }) @@ -1033,7 +1042,7 @@ func TestJobService(t *testing.T) { jobDeploymentService.On("UploadJobs", ctx, newTenant, jobModified, emptyJobNames).Return(errors.New("error in upload new job")) defer jobDeploymentService.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, jobDeploymentService, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, jobDeploymentService, nil) err := jobService.ChangeNamespace(ctx, sampleTenant, newTenant, specA.Name()) assert.ErrorContains(t, err, "error in upload new job") @@ -1055,7 +1064,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) defer eventHandler.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, eventHandler, nil, jobDeploymentService, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, eventHandler, nil, jobDeploymentService, nil) err := jobService.ChangeNamespace(ctx, sampleTenant, newTenant, specA.Name()) assert.NoError(t, err) }) @@ -1087,7 +1096,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, eventHandler, log, jobDeploymentService, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, eventHandler, log, jobDeploymentService, nil) affectedDownstream, err := jobService.Delete(ctx, sampleTenant, specA.Name(), false, false) assert.NoError(t, err) assert.Empty(t, affectedDownstream) @@ -1123,7 +1132,7 @@ func TestJobService(t *testing.T) { jobDeploymentService.On("UploadJobs", ctx, sampleTenant, emptyJobNames, jobNamesToRemove).Return(nil) eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, eventHandler, log, jobDeploymentService, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, eventHandler, log, jobDeploymentService, nil) affectedDownstream, err := jobService.Delete(ctx, sampleTenant, specA.Name(), false, true) assert.NoError(t, err) @@ -1147,7 +1156,7 @@ func TestJobService(t *testing.T) { } downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specA.Name()).Return(downstreamList, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil) affectedDownstream, err := jobService.Delete(ctx, sampleTenant, specA.Name(), false, false) assert.Error(t, err) assert.Empty(t, affectedDownstream) @@ -1166,7 +1175,7 @@ func TestJobService(t *testing.T) { downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specA.Name()).Return(nil, errors.New("internal error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil) affectedDownstream, err := jobService.Delete(ctx, sampleTenant, specA.Name(), false, false) assert.Error(t, err) @@ -1187,7 +1196,7 @@ func TestJobService(t *testing.T) { downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specA.Name()).Return(nil, nil) jobRepo.On("Delete", ctx, project.Name(), specA.Name(), false).Return(errors.New("internal error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil) affectedDownstream, err := jobService.Delete(ctx, sampleTenant, specA.Name(), false, false) assert.Error(t, err) assert.Empty(t, affectedDownstream) @@ -1215,7 +1224,7 @@ func TestJobService(t *testing.T) { errorMsg := "internal error" jobDeploymentService.On("UploadJobs", ctx, sampleTenant, emptyJobNames, mock.Anything).Return(errors.New(errorMsg)) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, eventHandler, log, jobDeploymentService, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, eventHandler, log, jobDeploymentService, nil) affectedDownstream, err := jobService.Delete(ctx, sampleTenant, specA.Name(), false, false) assert.ErrorContains(t, err, errorMsg) assert.Empty(t, affectedDownstream) @@ -1232,7 +1241,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1251,12 +1261,12 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobAUpstreamName := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) incomingSpecs := []*job.Spec{specA, specB} @@ -1264,14 +1274,14 @@ func TestJobService(t *testing.T) { jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(existingJobs, nil) - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) jobRepo.On("Add", ctx, mock.Anything).Return([]*job.Job{jobA}, nil) jobRepo.On("SetDirty", ctx, sampleTenant, []job.Name{jobA.Spec().Name()}, true).Return(nil) jobRepo.On("SetDirty", ctx, sampleTenant, []job.Name{jobA.Spec().Name()}, false).Return(nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), []*job.Job{jobA}, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -1285,7 +1295,7 @@ func TestJobService(t *testing.T) { var jobNamesToRemove []string jobDeploymentService.On("UploadJobs", ctx, sampleTenant, jobNamesToUpload, jobNamesToRemove).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.NoError(t, err) }) @@ -1299,7 +1309,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1318,8 +1329,8 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobAUpstreamName := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) incomingSpecs := []*job.Spec{specA} @@ -1330,8 +1341,8 @@ func TestJobService(t *testing.T) { existingJobA := job.NewJob(sampleTenant, existingSpecA, jobADestination, jobAUpstreamName, false) existingSpecs := []*job.Job{existingJobA} - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(existingSpecs, nil) jobRepo.On("Update", ctx, mock.Anything).Return([]*job.Job{jobA}, nil) @@ -1340,7 +1351,7 @@ func TestJobService(t *testing.T) { eventHandler.On("HandleEvent", mock.Anything).Times(1) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), []*job.Job{jobA}, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -1353,7 +1364,7 @@ func TestJobService(t *testing.T) { var jobNamesToRemove []string jobDeploymentService.On("UploadJobs", ctx, sampleTenant, jobNamesToUpload, jobNamesToRemove).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.NoError(t, err) }) @@ -1367,7 +1378,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1386,10 +1398,10 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resource.ZeroURN(), nil, false) + jobA := job.NewJob(sampleTenant, specA, "", nil, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) incomingSpecs := []*job.Spec{specA} @@ -1408,7 +1420,7 @@ func TestJobService(t *testing.T) { jobNamesToRemove := []string{specB.Name().String()} jobDeploymentService.On("UploadJobs", ctx, sampleTenant, jobNamesToUpload, jobNamesToRemove).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.NoError(t, err) }) @@ -1422,7 +1434,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1444,10 +1457,10 @@ func TestJobService(t *testing.T) { specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() specD, _ := job.NewSpecBuilder(jobVersion, "job-D", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resource.ZeroURN(), nil, false) - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) - jobC := job.NewJob(sampleTenant, specC, resource.ZeroURN(), nil, false) - jobD := job.NewJob(sampleTenant, specD, resource.ZeroURN(), nil, false) + jobA := job.NewJob(sampleTenant, specA, "", nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) + jobC := job.NewJob(sampleTenant, specC, "", nil, false) + jobD := job.NewJob(sampleTenant, specD, "", nil, false) incomingSpecs := []*job.Spec{} @@ -1473,7 +1486,7 @@ func TestJobService(t *testing.T) { jobDeploymentService.On("UploadJobs", ctx, sampleTenant, mock.Anything, mock.Anything).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.NoError(t, err) }) @@ -1487,7 +1500,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1512,28 +1526,28 @@ func TestJobService(t *testing.T) { w2, _ := models.NewWindow(jobVersion, "d", "0h", "24h") existingJobWindow := window.NewCustomConfig(w2) existingSpecB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, existingJobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobB := job.NewJob(sampleTenant, existingSpecB, resource.ZeroURN(), nil, false) + existingJobB := job.NewJob(sampleTenant, existingSpecB, "", nil, false) existingSpecC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobC := job.NewJob(sampleTenant, existingSpecC, resource.ZeroURN(), nil, false) + existingJobC := job.NewJob(sampleTenant, existingSpecC, "", nil, false) existingSpecs := []*job.Job{existingJobB, existingJobC} jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(existingSpecs, nil) - jobADestination := resourceURNA - jobBDestination := resourceURNB - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobBDestination := job.ResourceURN("bigquery://project:dataset.tableB") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() - jobAUpstreamNames := []resource.URN{resourceURNB} - var jobBUpstreamNames []resource.URN - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamNames, nil) - pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobBUpstreamNames, nil) + jobAUpstreamNames := []job.ResourceURN{"bigquery://project:dataset.tableB"} + var jobBUpstreamNames []job.ResourceURN + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamNames), nil) + pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobBUpstreamNames), nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamNames, false) jobRepo.On("Add", ctx, mock.Anything).Return([]*job.Job{jobA}, nil) - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) jobRepo.On("Update", ctx, mock.Anything).Return([]*job.Job{jobB}, nil) jobRepo.On("SetDirty", ctx, sampleTenant, []job.Name{jobA.Spec().Name(), jobB.Spec().Name()}, true).Return(nil) @@ -1542,7 +1556,7 @@ func TestJobService(t *testing.T) { downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), existingSpecC.Name()).Return(nil, nil) jobRepo.On("Delete", ctx, project.Name(), existingSpecC.Name(), false).Return(nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), []*job.Job{jobA, jobB}, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -1557,7 +1571,7 @@ func TestJobService(t *testing.T) { jobDeploymentService.On("UploadJobs", ctx, sampleTenant, emptyStringArr, jobNamesToRemove).Return(nil) jobDeploymentService.On("UploadJobs", ctx, sampleTenant, jobNamesToUpload, emptyStringArr).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.NoError(t, err) }) @@ -1571,7 +1585,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1596,29 +1611,29 @@ func TestJobService(t *testing.T) { w2, _ := models.NewWindow(jobVersion, "d", "0h", "24h") existingJobWindow := window.NewCustomConfig(w2) existingSpecB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, existingJobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobB := job.NewJob(sampleTenant, existingSpecB, resource.ZeroURN(), nil, false) + existingJobB := job.NewJob(sampleTenant, existingSpecB, "", nil, false) existingSpecC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobC := job.NewJob(sampleTenant, existingSpecC, resource.ZeroURN(), nil, false) + existingJobC := job.NewJob(sampleTenant, existingSpecC, "", nil, false) existingSpecD, _ := job.NewSpecBuilder(jobVersion, "job-D", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobD := job.NewJob(sampleTenant, existingSpecD, resource.ZeroURN(), nil, false) + existingJobD := job.NewJob(sampleTenant, existingSpecD, "", nil, false) existingSpecs := []*job.Job{existingJobB, existingJobC, existingJobD} jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(existingSpecs, nil) - jobADestination := resourceURNA - jobBDestination := resourceURNB - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobBDestination := job.ResourceURN("bigquery://project:dataset.tableB") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() - jobAUpstreamNames := []resource.URN{resourceURNB} - var jobBUpstreamNames []resource.URN - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamNames, nil) - pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobBUpstreamNames, nil) + jobAUpstreamNames := []job.ResourceURN{"bigquery://project:dataset.tableB"} + var jobBUpstreamNames []job.ResourceURN + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamNames), nil) + pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobBUpstreamNames), nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamNames, false) jobRepo.On("Add", ctx, mock.Anything).Return([]*job.Job{jobA}, nil) - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) jobRepo.On("Update", ctx, mock.Anything).Return([]*job.Job{jobB}, nil) jobRepo.On("SetDirty", ctx, sampleTenant, []job.Name{jobA.Spec().Name(), jobB.Spec().Name()}, true).Return(nil) jobRepo.On("SetDirty", ctx, sampleTenant, []job.Name{jobA.Spec().Name(), jobB.Spec().Name()}, false).Return(nil) @@ -1626,7 +1641,7 @@ func TestJobService(t *testing.T) { downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), existingSpecC.Name()).Return(nil, nil) jobRepo.On("Delete", ctx, project.Name(), existingSpecC.Name(), false).Return(nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), []*job.Job{jobA, jobB}, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -1641,7 +1656,7 @@ func TestJobService(t *testing.T) { jobDeploymentService.On("UploadJobs", ctx, sampleTenant, jobNamesToUpload, emptyStringArr).Return(nil) jobDeploymentService.On("UploadJobs", ctx, sampleTenant, emptyStringArr, jobNamesToRemove).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, []job.Name{"job-D"}, logWriter) assert.NoError(t, err) }) @@ -1655,7 +1670,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1677,17 +1693,17 @@ func TestJobService(t *testing.T) { incomingSpecs := []*job.Spec{specA} existingSpecC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobC := job.NewJob(sampleTenant, existingSpecC, resource.ZeroURN(), nil, false) + existingJobC := job.NewJob(sampleTenant, existingSpecC, "", nil, false) existingSpecs := []*job.Job{existingJobC} jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(existingSpecs, nil) - var specADestination resource.URN - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(specADestination, errors.New("internal error")).Once() + var specADestination job.ResourceURN + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(specADestination.String(), errors.New("internal error")).Once() logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.ErrorContains(t, err, "internal error") }) @@ -1702,7 +1718,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1726,18 +1743,18 @@ func TestJobService(t *testing.T) { w2, _ := models.NewWindow(jobVersion, "d", "0h", "24h") existingJobWindow := window.NewCustomConfig(w2) existingSpecB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, existingJobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobB := job.NewJob(sampleTenant, existingSpecB, resource.ZeroURN(), nil, false) + existingJobB := job.NewJob(sampleTenant, existingSpecB, "", nil, false) existingSpecC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobC := job.NewJob(sampleTenant, existingSpecC, resource.ZeroURN(), nil, false) + existingJobC := job.NewJob(sampleTenant, existingSpecC, "", nil, false) existingSpecs := []*job.Job{existingJobB, existingJobC} jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(existingSpecs, nil) - var jobBDestination resource.URN - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, errors.New("internal error")).Once() + var jobBDestination job.ResourceURN + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), errors.New("internal error")).Once() logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.ErrorContains(t, err, "internal error") }) @@ -1767,7 +1784,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1790,20 +1808,20 @@ func TestJobService(t *testing.T) { incomingSpecs := []*job.Spec{specA, specE} existingSpecC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobC := job.NewJob(sampleTenant, existingSpecC, resource.ZeroURN(), nil, false) + existingJobC := job.NewJob(sampleTenant, existingSpecC, "", nil, false) existingSpecD, _ := job.NewSpecBuilder(jobVersion, "job-D", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobD := job.NewJob(sampleTenant, existingSpecD, resource.ZeroURN(), nil, false) + existingJobD := job.NewJob(sampleTenant, existingSpecD, "", nil, false) existingSpecE, _ := job.NewSpecBuilder(jobVersion, "job-E", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - existingJobE := job.NewJob(sampleTenant, existingSpecE, resource.ZeroURN(), nil, false) + existingJobE := job.NewJob(sampleTenant, existingSpecE, "", nil, false) existingSpecs := []*job.Job{existingJobC, existingJobD, existingJobE} jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(existingSpecs, nil) - jobADestination := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() - jobAUpstreamNames := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamNames, nil) + jobAUpstreamNames := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamNames), nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamNames, false) jobRepo.On("Add", ctx, mock.Anything).Return([]*job.Job{jobA}, nil) @@ -1826,7 +1844,7 @@ func TestJobService(t *testing.T) { jobDeploymentService.On("UploadJobs", ctx, sampleTenant, emptyStringArr, jobNamesToRemove).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.ErrorContains(t, err, "job is being used by") }) @@ -1848,7 +1866,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1867,23 +1886,23 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobAUpstreamName := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobBDestination := resourceURNB - jobBUpstreamName := []resource.URN{resourceURNC} + jobBDestination := job.ResourceURN("bigquery://project:dataset.tableB") + jobBUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableC"} incomingSpecs := []*job.Spec{specA, specB} jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{}, nil) - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobBUpstreamName, nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobBUpstreamName), nil).Once() jobRepo.On("Add", ctx, mock.Anything).Return([]*job.Job{jobA}, errors.New("internal error")) jobRepo.On("SetDirty", ctx, sampleTenant, []job.Name{jobA.Spec().Name()}, true).Return(nil) @@ -1891,7 +1910,7 @@ func TestJobService(t *testing.T) { logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.ErrorContains(t, err, "internal error") }) @@ -1905,7 +1924,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1922,8 +1942,8 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobAUpstreamName := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} incomingSpecs := []*job.Spec{specA} @@ -1935,13 +1955,13 @@ func TestJobService(t *testing.T) { jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(existingSpecs, nil) - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) jobRepo.On("Update", ctx, mock.Anything).Return([]*job.Job{}, errors.New("internal error")) logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.ErrorContains(t, err, "internal error") }) @@ -1955,7 +1975,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -1972,10 +1993,10 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resource.ZeroURN(), nil, false) + jobA := job.NewJob(sampleTenant, specA, "", nil, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) incomingSpecs := []*job.Spec{specA} @@ -1988,7 +2009,7 @@ func TestJobService(t *testing.T) { logWriter.On("Write", mock.Anything, mock.Anything).Return(nil).Times(3) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.ErrorContains(t, err, "internal error") }) @@ -2002,7 +2023,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -2019,10 +2041,10 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resource.ZeroURN(), nil, false) + jobA := job.NewJob(sampleTenant, specA, "", nil, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) incomingSpecs := []*job.Spec{specA} @@ -2034,7 +2056,7 @@ func TestJobService(t *testing.T) { logWriter.On("Write", mock.Anything, mock.Anything).Return(nil).Twice() - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.ErrorContains(t, err, "internal error") }) @@ -2048,7 +2070,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -2070,10 +2093,10 @@ func TestJobService(t *testing.T) { specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() specD, _ := job.NewSpecBuilder(jobVersion, "job-D", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resource.ZeroURN(), nil, false) - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) - jobC := job.NewJob(sampleTenant, specC, resource.ZeroURN(), nil, false) - jobD := job.NewJob(sampleTenant, specD, resource.ZeroURN(), nil, false) + jobA := job.NewJob(sampleTenant, specA, "", nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) + jobC := job.NewJob(sampleTenant, specC, "", nil, false) + jobD := job.NewJob(sampleTenant, specD, "", nil, false) incomingSpecs := []*job.Spec{} @@ -2102,7 +2125,7 @@ func TestJobService(t *testing.T) { jobNamesToRemove := []string{specD.Name().String()} jobDeploymentService.On("UploadJobs", ctx, sampleTenant, jobNamesToUpload, jobNamesToRemove).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.Error(t, err) }) @@ -2116,7 +2139,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -2138,7 +2162,7 @@ func TestJobService(t *testing.T) { errorMsg := "project/namespace error" tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(nil, errors.New(errorMsg)) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.ErrorContains(t, err, errorMsg) @@ -2153,7 +2177,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -2172,12 +2197,12 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobAUpstreamName := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobB := job.NewJob(sampleTenant, specB, resource.ZeroURN(), nil, false) + jobB := job.NewJob(sampleTenant, specB, "", nil, false) incomingSpecs := []*job.Spec{specA, specB} @@ -2185,13 +2210,13 @@ func TestJobService(t *testing.T) { jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(existingJobs, nil) - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) jobRepo.On("Add", ctx, mock.Anything).Return([]*job.Job{jobA}, nil) jobRepo.On("SetDirty", ctx, sampleTenant, []job.Name{jobA.Spec().Name()}, true).Return(nil) - upstream := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstream := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstream}) upstreamResolver.On("BulkResolve", ctx, project.Name(), []*job.Job{jobA}, mock.Anything).Return([]*job.WithUpstream{jobWithUpstream}, nil, nil) @@ -2206,7 +2231,7 @@ func TestJobService(t *testing.T) { var emptyNmaesList []string jobDeploymentService.On("UploadJobs", ctx, sampleTenant, jobNamesToUpload, emptyNmaesList).Return(errors.New(errorMsg)) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.ReplaceAll(ctx, sampleTenant, incomingSpecs, jobNamesWithInvalidSpec, logWriter) assert.ErrorContains(t, err, errorMsg) }) @@ -2223,7 +2248,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -2240,12 +2266,12 @@ func TestJobService(t *testing.T) { eventHandler := newEventHandler(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobAUpstreamName := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) - var jobBDestination resource.URN - jobBUpstreamName := []resource.URN{resourceURNC} + var jobBDestination job.ResourceURN + jobBUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableC"} specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() jobB := job.NewJob(sampleTenant, specB, jobBDestination, jobBUpstreamName, false) @@ -2253,17 +2279,17 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil) - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobBUpstreamName, nil) + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobBUpstreamName), nil) jobRepo.On("Update", ctx, mock.Anything).Return([]*job.Job{jobA, jobB}, nil) - upstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstreamB := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamB}) - upstreamC := job.NewUpstreamResolved("job-C", "", resourceURNC, sampleTenant, "static", taskName, false) + upstreamC := job.NewUpstreamResolved("job-C", "", "bigquery://project:dataset.tableC", sampleTenant, "static", taskName, false) jobBWithUpstream := job.NewWithUpstream(jobB, []*job.Upstream{upstreamC}) upstreamResolver.On("BulkResolve", ctx, project.Name(), []*job.Job{jobA, jobB}, mock.Anything).Return([]*job.WithUpstream{jobAWithUpstream, jobBWithUpstream}, nil) @@ -2276,7 +2302,7 @@ func TestJobService(t *testing.T) { jobNamesToUpload := []string{jobA.GetName(), jobB.GetName()} jobDeploymentService.On("UploadJobs", ctx, sampleTenant, jobNamesToUpload, jobNamesToRemove).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Refresh(ctx, project.Name(), []string{namespace.Name().String()}, nil, logWriter) assert.NoError(t, err) }) @@ -2290,7 +2316,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -2307,13 +2334,13 @@ func TestJobService(t *testing.T) { eventHandler := newEventHandler(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobAUpstreamName := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobAUpstreamName, false) jobsTenant1 := []*job.Job{jobA} - var jobBDestination resource.URN - var jobBUpstreamName []resource.URN + var jobBDestination job.ResourceURN + var jobBUpstreamName []job.ResourceURN specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() jobB := job.NewJob(otherTenant, specB, jobBDestination, jobBUpstreamName, false) jobsTenant2 := []*job.Job{jobB} @@ -2324,17 +2351,17 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil).Once() tenantDetailsGetter.On("GetDetails", ctx, otherTenant).Return(detailedOtherTenant, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, nil).Once() - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobBUpstreamName, nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), nil).Once() + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobBUpstreamName), nil).Once() jobRepo.On("Update", ctx, mock.Anything).Return([]*job.Job{jobA}, nil).Once() jobRepo.On("Update", ctx, mock.Anything).Return([]*job.Job{jobB}, nil).Once() - upstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstreamB := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamB}) - upstreamC := job.NewUpstreamResolved("job-C", "", resourceURNC, otherTenant, "static", taskName, false) + upstreamC := job.NewUpstreamResolved("job-C", "", "bigquery://project:dataset.tableC", otherTenant, "static", taskName, false) jobBWithUpstream := job.NewWithUpstream(jobB, []*job.Upstream{upstreamC}) upstreamResolver.On("BulkResolve", ctx, project.Name(), []*job.Job{jobA}, mock.Anything).Return([]*job.WithUpstream{jobAWithUpstream}, nil) @@ -2350,7 +2377,7 @@ func TestJobService(t *testing.T) { jobDeploymentService.On("UploadJobs", ctx, sampleTenant, []string{jobA.GetName()}, jobNamesToRemove).Return(nil) jobDeploymentService.On("UploadJobs", ctx, otherTenant, []string{jobB.GetName()}, jobNamesToRemove).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.Refresh(ctx, project.Name(), []string{namespace.Name().String(), otherNamespace.Name().String()}, nil, logWriter) assert.NoError(t, err) }) @@ -2364,7 +2391,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -2374,7 +2402,7 @@ func TestJobService(t *testing.T) { jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(nil, errors.New("internal error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) err := jobService.Refresh(ctx, project.Name(), []string{namespace.Name().String()}, nil, nil) assert.ErrorContains(t, err, "internal error") }) @@ -2391,7 +2419,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -2407,9 +2436,9 @@ func TestJobService(t *testing.T) { eventHandler := newEventHandler(t) - resourceURNs := []resource.URN{resourceURNA} + resourceURNs := []job.ResourceURN{"bigquery://project:dataset.table"} - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) downstreamRepo.On("GetDownstreamBySources", ctx, resourceURNs).Return(nil, errors.New("internal error")) @@ -2427,7 +2456,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -2444,15 +2474,16 @@ func TestJobService(t *testing.T) { eventHandler := newEventHandler(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobAUpstreamName := resourceURNB - jobA := job.NewJob(sampleTenant, specA, jobADestination, []resource.URN{jobAUpstreamName}, false) + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobAUpstreamName := job.ResourceURN("bigquery://project:dataset.tableB") + jobA := job.NewJob(sampleTenant, specA, jobADestination, []job.ResourceURN{jobAUpstreamName}, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobBDestination := resourceURNB - jobB := job.NewJob(sampleTenant, specB, jobBDestination, []resource.URN{resourceURNC}, false) + jobBDestination := job.ResourceURN("bigquery://project:dataset.tableB") + jobBUpstreamName := job.ResourceURN("bigquery://project:dataset.tableC") + jobB := job.NewJob(sampleTenant, specB, jobBDestination, []job.ResourceURN{jobBUpstreamName}, false) - resourceURNs := []resource.URN{jobAUpstreamName, resourceURNC} + resourceURNs := []job.ResourceURN{jobAUpstreamName, jobBUpstreamName} jobDownstreams := []*job.Downstream{ job.NewDownstream(jobA.Spec().Name(), sampleTenant.ProjectName(), sampleTenant.NamespaceName(), jobTask.Name()), @@ -2465,17 +2496,17 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return([]resource.URN{jobAUpstreamName}, nil) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString([]job.ResourceURN{jobAUpstreamName}), nil) - pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination, nil).Once() - pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return([]resource.URN{resourceURNC}, nil) + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(jobBDestination.String(), nil).Once() + pluginService.On("IdentifyUpstreams", ctx, specB.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString([]job.ResourceURN{jobBUpstreamName}), nil) jobRepo.On("Update", ctx, mock.Anything).Return([]*job.Job{jobA, jobB}, nil) - upstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "static", taskName, false) + upstreamB := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "static", taskName, false) jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamB}) - upstreamC := job.NewUpstreamResolved("job-C", "", resourceURNC, sampleTenant, "static", taskName, false) + upstreamC := job.NewUpstreamResolved("job-C", "", "bigquery://project:dataset.tableC", sampleTenant, "static", taskName, false) jobBWithUpstream := job.NewWithUpstream(jobB, []*job.Upstream{upstreamC}) upstreamResolver.On("BulkResolve", ctx, project.Name(), []*job.Job{jobA, jobB}, mock.Anything).Return([]*job.WithUpstream{jobAWithUpstream, jobBWithUpstream}, nil) @@ -2488,7 +2519,7 @@ func TestJobService(t *testing.T) { jobNamesToUpload := []string{jobA.GetName(), jobB.GetName()} jobDeploymentService.On("UploadJobs", ctx, sampleTenant, jobNamesToUpload, jobNamesToRemove).Return(nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, eventHandler, log, jobDeploymentService, compiler.NewEngine()) err := jobService.RefreshResourceDownstream(ctx, resourceURNs, logWriter) assert.NoError(t, err) @@ -2509,7 +2540,7 @@ func TestJobService(t *testing.T) { jobName, _ := job.NameFrom("job-A") jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), jobName).Return(nil, errors.New("error when fetch job")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.Get(ctx, sampleTenant, jobName) assert.Error(t, err, "error when fetch job") @@ -2526,10 +2557,10 @@ func TestJobService(t *testing.T) { defer downstreamRepo.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specA.Name()).Return(jobA, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.Get(ctx, sampleTenant, specA.Name()) assert.NoError(t, err, "error when fetch job") @@ -2544,10 +2575,10 @@ func TestJobService(t *testing.T) { jobRepo := new(JobRepository) defer jobRepo.AssertExpectations(t) - jobRepo.On("GetAllByResourceDestination", ctx, resourceURNA).Return(nil, errors.New("error encountered")) + jobRepo.On("GetAllByResourceDestination", ctx, job.ResourceURN("bigquery://project:dataset.example")).Return(nil, errors.New("error encountered")) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) - actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ResourceDestination, "bigquery://project:dataset.tableA")) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) + actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ResourceDestination, "bigquery://project:dataset.example")) assert.Error(t, err, "error encountered") assert.Nil(t, actual) }) @@ -2556,10 +2587,10 @@ func TestJobService(t *testing.T) { defer jobRepo.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) - jobRepo.On("GetAllByResourceDestination", ctx, resourceURNA).Return([]*job.Job{jobA}, nil) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) + jobRepo.On("GetAllByResourceDestination", ctx, job.ResourceURN("bigquery://project:dataset.tableA")).Return([]*job.Job{jobA}, nil) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ResourceDestination, "bigquery://project:dataset.tableA")) assert.NoError(t, err) assert.NotNil(t, actual) @@ -2575,7 +2606,7 @@ func TestJobService(t *testing.T) { jobName, _ := job.NameFrom("job-A") jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), jobName).Return(nil, errors.New("error encountered")) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithStringArray(filter.JobNames, []string{jobName.String()}), @@ -2588,12 +2619,12 @@ func TestJobService(t *testing.T) { defer jobRepo.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specA.Name()).Return(jobA, nil) jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specB.Name()).Return(nil, errors.New("error encountered")) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithStringArray(filter.JobNames, []string{specA.Name().String(), specB.Name().String()}), @@ -2610,7 +2641,7 @@ func TestJobService(t *testing.T) { specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specA.Name()).Return(nil, optErrors.NotFound(job.EntityJob, "job not found")) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithStringArray(filter.JobNames, []string{specA.Name().String()}), @@ -2623,10 +2654,10 @@ func TestJobService(t *testing.T) { defer jobRepo.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specA.Name()).Return(jobA, nil) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithStringArray(filter.JobNames, []string{specA.Name().String()}), @@ -2645,7 +2676,7 @@ func TestJobService(t *testing.T) { jobName, _ := job.NameFrom("job-A") jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), jobName).Return(nil, errors.New("error encountered")) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithString(filter.JobName, jobName.String()), @@ -2660,7 +2691,7 @@ func TestJobService(t *testing.T) { specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specA.Name()).Return(nil, optErrors.NotFound(job.EntityJob, "job not found")) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithString(filter.JobName, specA.Name().String()), @@ -2673,10 +2704,10 @@ func TestJobService(t *testing.T) { defer jobRepo.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), specA.Name()).Return(jobA, nil) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithString(filter.JobName, specA.Name().String()), @@ -2694,7 +2725,7 @@ func TestJobService(t *testing.T) { jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(nil, errors.New("error encountered")) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithStringArray(filter.NamespaceNames, []string{sampleTenant.NamespaceName().String()}), @@ -2706,7 +2737,7 @@ func TestJobService(t *testing.T) { jobRepo := new(JobRepository) defer jobRepo.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithStringArray(filter.NamespaceNames, []string{""}), @@ -2719,10 +2750,10 @@ func TestJobService(t *testing.T) { defer jobRepo.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA}, nil) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithStringArray(filter.NamespaceNames, []string{sampleTenant.NamespaceName().String()}), @@ -2740,7 +2771,7 @@ func TestJobService(t *testing.T) { jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(nil, errors.New("error encountered")) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithString(filter.NamespaceName, sampleTenant.NamespaceName().String()), @@ -2753,10 +2784,10 @@ func TestJobService(t *testing.T) { defer jobRepo.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA}, nil) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), filter.WithString(filter.NamespaceName, sampleTenant.NamespaceName().String()), @@ -2774,7 +2805,7 @@ func TestJobService(t *testing.T) { jobRepo.On("GetAllByProjectName", ctx, sampleTenant.ProjectName()).Return(nil, errors.New("error encountered")) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), ) @@ -2786,10 +2817,10 @@ func TestJobService(t *testing.T) { defer jobRepo.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) jobRepo.On("GetAllByProjectName", ctx, sampleTenant.ProjectName()).Return([]*job.Job{jobA}, nil) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx, filter.WithString(filter.ProjectName, sampleTenant.ProjectName().String()), ) @@ -2800,7 +2831,7 @@ func TestJobService(t *testing.T) { }) }) t.Run("return error when there's no filter", func(t *testing.T) { - jobService := service.NewJobService(nil, nil, nil, nil, nil, nil, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(nil, nil, nil, nil, nil, nil, nil, log, nil, nil) actual, err := jobService.GetByFilter(ctx) assert.Error(t, err, "no filter matched") assert.Nil(t, actual) @@ -2809,18 +2840,20 @@ func TestJobService(t *testing.T) { t.Run("GetTaskInfo", func(t *testing.T) { t.Run("return error when plugin could not retrieve info", func(t *testing.T) { - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) pluginService.On("Info", ctx, jobTask.Name().String()).Return(nil, errors.New("error encountered")) - jobService := service.NewJobService(nil, nil, nil, pluginService, nil, nil, nil, nil, nil, nil, nil, nil) + jobService := service.NewJobService(nil, nil, nil, pluginService, nil, nil, nil, nil, nil, nil) actual, err := jobService.GetTaskInfo(ctx, jobTask) assert.Error(t, err, "error encountered") assert.Nil(t, actual) }) t.Run("return task with information included when success", func(t *testing.T) { - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) pluginInfoResp := &plugin.Info{ Name: "bq2bq", @@ -2829,7 +2862,7 @@ func TestJobService(t *testing.T) { } pluginService.On("Info", ctx, jobTask.Name().String()).Return(pluginInfoResp, nil) - jobService := service.NewJobService(nil, nil, nil, pluginService, nil, nil, nil, nil, nil, nil, nil, nil) + jobService := service.NewJobService(nil, nil, nil, pluginService, nil, nil, nil, nil, nil, nil) actual, err := jobService.GetTaskInfo(ctx, jobTask) assert.NoError(t, err) @@ -2839,926 +2872,471 @@ func TestJobService(t *testing.T) { }) t.Run("Validate", func(t *testing.T) { - t.Run("job preparation", func(t *testing.T) { - t.Run("validate request", func(t *testing.T) { - t.Run("returns nil and error if job names length is more than zero while job specs length is also more than zero", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + t.Run("returns error when get tenant details if failed", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(nil, errors.New("get tenant details fail")) + defer tenantDetailsGetter.AssertExpectations(t) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + jobService := service.NewJobService(nil, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) + err := jobService.Validate(ctx, sampleTenant, []*job.Spec{}, jobNamesWithInvalidSpec, nil) + assert.Error(t, err) + assert.Equal(t, "get tenant details fail", err.Error()) + }) + t.Run("returns error when validate duplicate jobs detected", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(detailedTenant, nil) + defer tenantDetailsGetter.AssertExpectations(t) - jobService := service.NewJobService(nil, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: []*job.Spec{{}, {}}, - JobNames: []string{"", ""}, - DeletionMode: false, - } + upstreamResolver := new(UpstreamResolver) + defer upstreamResolver.AssertExpectations(t) - actualResult, actualError := jobService.Validate(ctx, request) + jobRepo := new(JobRepository) + defer jobRepo.AssertExpectations(t) - assert.Nil(t, actualResult) - assert.ErrorContains(t, actualError, "job names and specs can not be specified together") - }) + upstreamRepo := new(UpstreamRepository) + defer upstreamRepo.AssertExpectations(t) - t.Run("returns nil and error if job names and job specs are both empty", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + downstreamRepo := new(DownstreamRepository) + defer downstreamRepo.AssertExpectations(t) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + logWriter := new(mockWriter) + defer logWriter.AssertExpectations(t) - jobService := service.NewJobService(nil, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + specA1, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() + specA2, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: nil, - DeletionMode: false, - } + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) - actualResult, actualError := jobService.Validate(ctx, request) + err := jobService.Validate(ctx, sampleTenant, []*job.Spec{specA1, specA2}, jobNamesWithInvalidSpec, logWriter) + assert.Error(t, err) + assert.Equal(t, "validate specs errors:\n duplicate job-A", err.Error()) + }) + t.Run("returns error when generate jobs", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(detailedTenant, nil) + defer tenantDetailsGetter.AssertExpectations(t) - assert.Nil(t, actualResult) - assert.ErrorContains(t, actualError, "job names and job specs are both empty") - }) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) - t.Run("returns nil and error if using deletion mode while job names length is zero", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + upstreamResolver := new(UpstreamResolver) + defer upstreamResolver.AssertExpectations(t) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + jobRepo := new(JobRepository) + defer jobRepo.AssertExpectations(t) - jobService := service.NewJobService(nil, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + upstreamRepo := new(UpstreamRepository) + defer upstreamRepo.AssertExpectations(t) - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: []*job.Spec{{}, {}}, - JobNames: nil, - DeletionMode: true, - } + downstreamRepo := new(DownstreamRepository) + defer downstreamRepo.AssertExpectations(t) - actualResult, actualError := jobService.Validate(ctx, request) + logWriter := new(mockWriter) + defer logWriter.AssertExpectations(t) - assert.Nil(t, actualResult) - assert.ErrorContains(t, actualError, "deletion job only accepts job names") - }) - }) + specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - t.Run("validate duplication", func(t *testing.T) { - t.Run("returns nil and error if job names are duplicated", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return(nil, nil) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return("", errors.New("some error on generate destination")) - jobService := service.NewJobService(nil, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + upstreamResolver.On("CheckStaticResolvable", ctx, detailedTenant.ToTenant(), mock.Anything, mock.Anything).Return(nil) - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{"job1", "job2", "job1"}, - DeletionMode: false, - } + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) - actualResult, actualError := jobService.Validate(ctx, request) + logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) - assert.Nil(t, actualResult) - assert.ErrorContains(t, actualError, "the following jobs are duplicated: [job1]") - }) + err := jobService.Validate(ctx, sampleTenant, []*job.Spec{specA}, jobNamesWithInvalidSpec, logWriter) + assert.Error(t, err) + assert.Equal(t, "validate specs errors:\n some error on generate destination", err.Error()) + }) + t.Run("returns error when a job fail a deletion check", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(detailedTenant, nil) + defer tenantDetailsGetter.AssertExpectations(t) - t.Run("returns nil and error if job specs are duplicated based on name", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + upstreamResolver := new(UpstreamResolver) + defer upstreamResolver.AssertExpectations(t) - jobService := service.NewJobService(nil, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + jobRepo := new(JobRepository) + defer jobRepo.AssertExpectations(t) - jobSpec1, err := job.NewSpecBuilder(1, "job1", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) - jobSpec2, err := job.NewSpecBuilder(1, "job2", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) - jobSpec3, err := job.NewSpecBuilder(1, "job1", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() // intentional duplication - assert.NoError(t, err) + upstreamRepo := new(UpstreamRepository) + defer upstreamRepo.AssertExpectations(t) - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: []*job.Spec{jobSpec1, jobSpec2, jobSpec3}, - JobNames: nil, - DeletionMode: false, - } + downstreamRepo := new(DownstreamRepository) + defer downstreamRepo.AssertExpectations(t) - actualResult, actualError := jobService.Validate(ctx, request) + logWriter := new(mockWriter) + defer logWriter.AssertExpectations(t) - assert.Nil(t, actualResult) - assert.ErrorContains(t, actualError, "the following jobs are duplicated: [job1]") - }) - }) + jobTaskConfigA, _ := job.ConfigFrom(map[string]string{"table": "table-A", "BQ_SERVICE_ACCOUNT": "secret_account"}) + jobTaskConfigB, _ := job.ConfigFrom(map[string]string{"table": "table-B", "BQ_SERVICE_ACCOUNT": "secret_account"}) + jobTaskConfigC, _ := job.ConfigFrom(map[string]string{"table": "table-C", "BQ_SERVICE_ACCOUNT": "secret_account"}) + jobTaskConfigD, _ := job.ConfigFrom(map[string]string{"table": "table-D", "BQ_SERVICE_ACCOUNT": "secret_account"}) + jobTaskA := job.NewTask(taskName, jobTaskConfigA) + jobTaskB := job.NewTask(taskName, jobTaskConfigB) + jobTaskC := job.NewTask(taskName, jobTaskConfigC) + jobTaskD := job.NewTask(taskName, jobTaskConfigD) - t.Run("returns result and nil if error when getting tenant details", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).Build() + specAUpdated, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner-updated", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).Build() + specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTaskB).WithAsset(jobAsset).Build() + specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTaskC).WithAsset(jobAsset).Build() + specD, _ := job.NewSpecBuilder(jobVersion, "job-D", "sample-owner", jobSchedule, jobWindow, jobTaskD).WithAsset(jobAsset).Build() + specs := []*job.Spec{specAUpdated, specB, specD} - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableZ"}, false) + jobB := job.NewJob(sampleTenant, specB, "table-B", []job.ResourceURN{"bigquery://project:dataset.tableA", "bigquery://project:dataset.tableD"}, false) + jobC := job.NewJob(sampleTenant, specC, "table-C", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) + jobD := job.NewJob(sampleTenant, specD, "table-D", nil, false) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA, jobB, jobC, jobD}, nil) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(job.ResourceURN("bigquery://project:dataset.tableA").String(), nil) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString([]job.ResourceURN{"bigquery://project:dataset.tableZ"}), nil) - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(nil, errors.New("unexpected error in tenant")) + upstreamResolver.On("CheckStaticResolvable", ctx, detailedTenant.ToTenant(), mock.Anything, mock.Anything).Return(nil) - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{"job1", "job2"}, - DeletionMode: false, - } + jobCDownstream := []*job.Downstream{ + job.NewDownstream("job-B", project.Name(), namespace.Name(), taskName), + } + jobBDownstream := []*job.Downstream{ + job.NewDownstream("job-D", project.Name(), namespace.Name(), taskName), + } + downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specC.Name()).Return(jobCDownstream, nil) + downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specB.Name()).Return(jobBDownstream, nil) + downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specD.Name()).Return([]*job.Downstream{}, nil) - actualResult, actualError := jobService.Validate(ctx, request) + logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) - assert.Nil(t, actualResult) - assert.ErrorContains(t, actualError, "unexpected error in tenant") - }) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) + err := jobService.Validate(ctx, sampleTenant, specs, jobNamesWithInvalidSpec, logWriter) + assert.Error(t, err) + assert.ErrorContains(t, err, "deletion of job job-C will fail. job is being used by test-proj/job-B, test-proj/job-D") + }) - t.Run("returns nil and error if one or more of job names are invalid", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + t.Run("returns no error when delete the job and its downstreams", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(detailedTenant, nil) + defer tenantDetailsGetter.AssertExpectations(t) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) - jobService := service.NewJobService(nil, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + upstreamResolver := new(UpstreamResolver) + defer upstreamResolver.AssertExpectations(t) - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{""}, // intentional empty - DeletionMode: false, - } + jobRepo := new(JobRepository) + defer jobRepo.AssertExpectations(t) - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) + upstreamRepo := new(UpstreamRepository) + defer upstreamRepo.AssertExpectations(t) - actualResult, actualError := jobService.Validate(ctx, request) + downstreamRepo := new(DownstreamRepository) + defer downstreamRepo.AssertExpectations(t) - assert.Nil(t, actualResult) - assert.ErrorContains(t, actualError, "name is empty") - }) + logWriter := new(mockWriter) + defer logWriter.AssertExpectations(t) - t.Run("returns nil and error if one or more of job names are error when being fetched from repository", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + jobTaskConfigA, _ := job.ConfigFrom(map[string]string{"table": "table-A", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskConfigB, _ := job.ConfigFrom(map[string]string{"table": "table-B", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskConfigC, _ := job.ConfigFrom(map[string]string{"table": "table-C", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskA := job.NewTask(taskName, jobTaskConfigA) + jobTaskB := job.NewTask(taskName, jobTaskConfigB) + jobTaskC := job.NewTask(taskName, jobTaskConfigC) - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) + specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).Build() + specAUpdated, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner-updated", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).Build() + specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTaskB).WithAsset(jobAsset).Build() + specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTaskC).WithAsset(jobAsset).Build() + specs := []*job.Spec{specAUpdated} - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + jobA := job.NewJob(sampleTenant, specA, "table-A", []job.ResourceURN{"bigquery://project:dataset.tableZ"}, false) + jobB := job.NewJob(sampleTenant, specB, "table-B", nil, false) + jobC := job.NewJob(sampleTenant, specC, "table-C", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA, jobB, jobC}, nil) - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{"job1"}, - DeletionMode: false, - } + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(job.ResourceURN("bigquery://project:dataset.tableA").String(), nil) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString([]job.ResourceURN{"bigquery://project:dataset.tableZ"}), nil) + upstreamResolver.On("CheckStaticResolvable", ctx, detailedTenant.ToTenant(), mock.Anything, mock.Anything).Return(nil) - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) + jobCDownstream := []*job.Downstream{ + job.NewDownstream("job-B", project.Name(), namespace.Name(), taskName), + } + downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specC.Name()).Return(jobCDownstream, nil) + downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specB.Name()).Return([]*job.Downstream{}, nil) - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job1")).Return(nil, errors.New("unknown repository error")) + logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) - actualResult, actualError := jobService.Validate(ctx, request) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) + err := jobService.Validate(ctx, sampleTenant, specs, jobNamesWithInvalidSpec, logWriter) + assert.NoError(t, err) + }) - assert.Nil(t, actualResult) - assert.ErrorContains(t, actualError, "unknown repository error") - }) + t.Run("returns error when there's a cyclic", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(detailedTenant, nil) + defer tenantDetailsGetter.AssertExpectations(t) - t.Run("returns result and nil if one or more jobs do not have the same tenant as the root", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) + upstreamResolver := new(UpstreamResolver) + defer upstreamResolver.AssertExpectations(t) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + jobRepo := new(JobRepository) + defer jobRepo.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + upstreamRepo := new(UpstreamRepository) + defer upstreamRepo.AssertExpectations(t) - jobSpec1, err := job.NewSpecBuilder(1, "job1", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) - jobSpec2, err := job.NewSpecBuilder(1, "job2", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) + downstreamRepo := new(DownstreamRepository) + defer downstreamRepo.AssertExpectations(t) - job1 := job.NewJob(sampleTenant, jobSpec1, resource.ZeroURN(), nil, false) - job2 := job.NewJob(otherTenant, jobSpec2, resource.ZeroURN(), nil, false) // intentional wrong tenant - - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job1")).Return(job1, nil) - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job2")).Return(job2, nil) - - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{"job1", "job2"}, - DeletionMode: false, - } - - expectedResult := map[job.Name][]dto.ValidateResult{ - "job2": { - { - Stage: "tenant validation", - Messages: []string{ - fmt.Sprintf("current tenant is [%s.%s]", otherTenant.ProjectName(), otherTenant.NamespaceName()), - fmt.Sprintf("expected tenant is [%s.%s]", sampleTenant.ProjectName(), sampleTenant.NamespaceName()), - }, - Success: false, - }, - }, - } - - actualResult, actualError := jobService.Validate(ctx, request) - - assert.EqualValues(t, expectedResult, actualResult) - assert.NoError(t, actualError) - }) + logWriter := new(mockWriter) + defer logWriter.AssertExpectations(t) + + specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() + specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() + specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() + + jobB := job.NewJob(sampleTenant, specB, "bigquery://project:dataset.tableB", []job.ResourceURN{"bigquery://project:dataset.tableA"}, false) + jobC := job.NewJob(sampleTenant, specC, "bigquery://project:dataset.tableC", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) + + jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobB, jobC}, nil) + + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return("bigquery://project:dataset.tableA", nil) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString([]job.ResourceURN{"bigquery://project:dataset.tableC"}), nil) + + upstreamResolver.On("CheckStaticResolvable", ctx, detailedTenant.ToTenant(), mock.Anything, mock.Anything).Return(nil) + + logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) + + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) + err := jobService.Validate(ctx, sampleTenant, []*job.Spec{specA, specB, specC}, jobNamesWithInvalidSpec, logWriter) + assert.Error(t, err) + assert.ErrorContains(t, err, "a cycle dependency encountered in the tree:") }) + t.Run("returns error when there's a cyclic on the incoming job request", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(detailedTenant, nil) + defer tenantDetailsGetter.AssertExpectations(t) - t.Run("deletion mode validation", func(t *testing.T) { - t.Run("returns result and nil if error is encountered when getting downstream jobs", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) + upstreamResolver := new(UpstreamResolver) + defer upstreamResolver.AssertExpectations(t) - downstreamRepo := new(DownstreamRepository) - defer downstreamRepo.AssertExpectations(t) + jobRepo := new(JobRepository) + defer jobRepo.AssertExpectations(t) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + upstreamRepo := new(UpstreamRepository) + defer upstreamRepo.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, downstreamRepo, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + downstreamRepo := new(DownstreamRepository) + defer downstreamRepo.AssertExpectations(t) - jobSpec1, err := job.NewSpecBuilder(1, "job1", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) - jobSpec2, err := job.NewSpecBuilder(1, "job2", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) + logWriter := new(mockWriter) + defer logWriter.AssertExpectations(t) - job1 := job.NewJob(sampleTenant, jobSpec1, resource.ZeroURN(), nil, false) - job2 := job.NewJob(sampleTenant, jobSpec2, resource.ZeroURN(), nil, false) - - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job1")).Return(job1, nil) - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job2")).Return(job2, nil) - - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - - downstreamRepo.On("GetDownstreamByJobName", ctx, sampleTenant.ProjectName(), jobSpec1.Name()).Return([]*job.Downstream{}, nil) - downstreamRepo.On("GetDownstreamByJobName", ctx, sampleTenant.ProjectName(), jobSpec2.Name()).Return(nil, errors.New("unknown error")) - - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{"job1", "job2"}, - DeletionMode: true, - } - - expectedResult := map[job.Name][]dto.ValidateResult{ - "job1": { - { - Stage: "validation for deletion", - Messages: []string{"job is safe for deletion"}, - Success: true, - }, - }, - "job2": { - { - Stage: "validation for deletion", - Messages: []string{ - "downstreams can not be fetched", - "unknown error", - }, - Success: false, - }, - }, - } - - actualResult, actualError := jobService.Validate(ctx, request) - - assert.EqualValues(t, expectedResult["job1"], actualResult["job1"]) - assert.EqualValues(t, expectedResult["job2"], actualResult["job2"]) - assert.NoError(t, actualError) - }) + jobTaskConfigA, _ := job.ConfigFrom(map[string]string{"table": "table-A", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskConfigB, _ := job.ConfigFrom(map[string]string{"table": "table-B", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskConfigC, _ := job.ConfigFrom(map[string]string{"table": "table-C", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskA := job.NewTask(taskName, jobTaskConfigA) + jobTaskB := job.NewTask(taskName, jobTaskConfigB) + jobTaskC := job.NewTask(taskName, jobTaskConfigC) - t.Run("returns result and nil if job is safe for deletion", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).Build() + specAUpdated, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner-updated", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).Build() + specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTaskB).WithAsset(jobAsset).Build() + specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTaskC).WithAsset(jobAsset).Build() - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) + jobA := job.NewJob(sampleTenant, specA, "bigquery://project:dataset.tableA", []job.ResourceURN{"bigquery://project:dataset.tableZ"}, false) + jobB := job.NewJob(sampleTenant, specB, "bigquery://project:dataset.tableB", []job.ResourceURN{"bigquery://project:dataset.tableA"}, false) + jobC := job.NewJob(sampleTenant, specC, "bigquery://project:dataset.tableC", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) - downstreamRepo := new(DownstreamRepository) - defer downstreamRepo.AssertExpectations(t) + jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA, jobB, jobC}, nil) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + upstreamResolver.On("CheckStaticResolvable", ctx, detailedTenant.ToTenant(), mock.Anything, mock.Anything).Return(nil) - jobService := service.NewJobService(jobRepo, nil, downstreamRepo, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(job.ResourceURN("bigquery://project:dataset.tableA").String(), nil) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString([]job.ResourceURN{"bigquery://project:dataset.tableC", "bigquery://project:dataset.tableZ"}), nil) - jobSpec1, err := job.NewSpecBuilder(1, "job1", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) - jobSpec2, err := job.NewSpecBuilder(1, "job2", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) + logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) - job1 := job.NewJob(sampleTenant, jobSpec1, resource.ZeroURN(), nil, false) - job2 := job.NewJob(sampleTenant, jobSpec2, resource.ZeroURN(), nil, false) - - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job1")).Return(job1, nil) - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job2")).Return(job2, nil) - - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - - downstreamRepo.On("GetDownstreamByJobName", ctx, sampleTenant.ProjectName(), jobSpec1.Name()).Return([]*job.Downstream{}, nil) - downstreamRepo.On("GetDownstreamByJobName", ctx, sampleTenant.ProjectName(), jobSpec2.Name()).Return([]*job.Downstream{}, nil) - - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{"job1", "job2"}, - DeletionMode: true, - } - - expectedResult := map[job.Name][]dto.ValidateResult{ - "job1": { - { - Stage: "validation for deletion", - Messages: []string{"job is safe for deletion"}, - Success: true, - }, - }, - "job2": { - { - Stage: "validation for deletion", - Messages: []string{"job is safe for deletion"}, - Success: true, - }, - }, - } - - actualResult, actualError := jobService.Validate(ctx, request) - - assert.EqualValues(t, expectedResult["job1"], actualResult["job1"]) - assert.EqualValues(t, expectedResult["job2"], actualResult["job2"]) - assert.NoError(t, actualError) - }) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) + err := jobService.Validate(ctx, sampleTenant, []*job.Spec{specAUpdated, specB, specC}, jobNamesWithInvalidSpec, logWriter) + assert.Error(t, err) + assert.ErrorContains(t, err, "a cycle dependency encountered in the tree:") + }) + t.Run("returns error when there's a cyclic without resource destination", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(detailedTenant, nil) + defer tenantDetailsGetter.AssertExpectations(t) - t.Run("returns result and nil if job is not safe for deletion", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) + upstreamResolver := new(UpstreamResolver) + defer upstreamResolver.AssertExpectations(t) + + jobRepo := new(JobRepository) + defer jobRepo.AssertExpectations(t) - downstreamRepo := new(DownstreamRepository) - defer downstreamRepo.AssertExpectations(t) + upstreamRepo := new(UpstreamRepository) + defer upstreamRepo.AssertExpectations(t) + + downstreamRepo := new(DownstreamRepository) + defer downstreamRepo.AssertExpectations(t) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + logWriter := new(mockWriter) + defer logWriter.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, downstreamRepo, nil, nil, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + jobTaskConfigA, _ := job.ConfigFrom(map[string]string{"table": "table-A", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskConfigB, _ := job.ConfigFrom(map[string]string{"table": "table-B", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskConfigC, _ := job.ConfigFrom(map[string]string{"example": "value"}) + jobTaskA := job.NewTask(taskName, jobTaskConfigA) + jobTaskB := job.NewTask(taskName, jobTaskConfigB) + jobTaskC := job.NewTask("python", jobTaskConfigC) - jobSpec1, err := job.NewSpecBuilder(1, "job1", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) - jobSpec2, err := job.NewSpecBuilder(1, "job2", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) + upstreamsSpecA, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"test-proj/job-C"}).Build() + upstreamsSpecC, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"test-proj/job-B"}).Build() + upstreamsSpecB, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"test-proj/job-A"}).Build() + specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).WithSpecUpstream(upstreamsSpecA).Build() + specAUpdated, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner-updated", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).WithSpecUpstream(upstreamsSpecA).Build() + specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTaskB).WithAsset(jobAsset).Build() + specBUpdated, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner-updated", jobSchedule, jobWindow, jobTaskB).WithAsset(jobAsset).WithSpecUpstream(upstreamsSpecB).Build() + specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTaskC).WithSpecUpstream(upstreamsSpecC).Build() - job1 := job.NewJob(sampleTenant, jobSpec1, resource.ZeroURN(), nil, false) - job2 := job.NewJob(sampleTenant, jobSpec2, resource.ZeroURN(), nil, false) - job2Downstream := job.NewDownstream("job3", sampleTenant.ProjectName(), sampleTenant.NamespaceName(), taskName) - - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job1")).Return(job1, nil) - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("job2")).Return(job2, nil) - - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - - downstreamRepo.On("GetDownstreamByJobName", ctx, sampleTenant.ProjectName(), jobSpec1.Name()).Return([]*job.Downstream{}, nil) - downstreamRepo.On("GetDownstreamByJobName", ctx, sampleTenant.ProjectName(), jobSpec2.Name()).Return([]*job.Downstream{job2Downstream}, nil) - downstreamRepo.On("GetDownstreamByJobName", ctx, sampleTenant.ProjectName(), job2Downstream.Name()).Return([]*job.Downstream{}, nil) - - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{"job1", "job2"}, - DeletionMode: true, - } - - expectedResult := map[job.Name][]dto.ValidateResult{ - "job1": { - { - Stage: "validation for deletion", - Messages: []string{"job is safe for deletion"}, - Success: true, - }, - }, - "job2": { - { - Stage: "validation for deletion", - Messages: []string{"job is not safe for deletion", "validating job for deletion errors:\n failed precondition for entity job: deletion of job job2 will fail. job is being used by test-proj/job3"}, - Success: false, - }, - }, - } - - actualResult, actualError := jobService.Validate(ctx, request) - - assert.EqualValues(t, expectedResult["job1"], actualResult["job1"]) - assert.EqualValues(t, expectedResult["job2"], actualResult["job2"]) - assert.NoError(t, actualError) - }) + jobA := job.NewJob(sampleTenant, specA, "bigquery://project:dataset.tableA", []job.ResourceURN{"bigquery://project:dataset.tableZ"}, false) + jobB := job.NewJob(sampleTenant, specB, "bigquery://project:dataset.tableB", []job.ResourceURN{"bigquery://project:dataset.tableA"}, false) + + upstreamResolver.On("CheckStaticResolvable", ctx, detailedTenant.ToTenant(), mock.Anything, mock.Anything).Return(nil) + + jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA, jobB}, nil) + + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(job.ResourceURN("bigquery://project:dataset.tableA").String(), nil) + pluginService.On("ConstructDestinationURN", ctx, specB.Task().Name().String(), mock.Anything).Return(job.ResourceURN("bigquery://project:dataset.tableB").String(), nil) + pluginService.On("ConstructDestinationURN", ctx, specC.Task().Name().String(), mock.Anything).Return("", nil) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString([]job.ResourceURN{"bigquery://project:dataset.tableZ"}), nil) + pluginService.On("IdentifyUpstreams", ctx, specBUpdated.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString([]job.ResourceURN{"bigquery://project:dataset.tableA"}), nil) + pluginService.On("IdentifyUpstreams", ctx, specC.Task().Name().String(), mock.Anything, mock.Anything).Return([]string{}, nil) + + logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) + + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) + err := jobService.Validate(ctx, sampleTenant, []*job.Spec{specAUpdated, specBUpdated, specC}, jobNamesWithInvalidSpec, logWriter) + assert.Error(t, err) + assert.ErrorContains(t, err, "a cycle dependency encountered in the tree:") }) + t.Run("returns error when there's a cyclic for static upstreams", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(detailedTenant, nil) + defer tenantDetailsGetter.AssertExpectations(t) - t.Run("non-deletion mode validation", func(t *testing.T) { - t.Run("validate cyclic", func(t *testing.T) { - t.Run("returns result and nil if cyclic dependency is detected", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) - - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) - - upstreamResolver := new(UpstreamResolver) - defer upstreamResolver.AssertExpectations(t) - - downstreamRepo := new(DownstreamRepository) - defer downstreamRepo.AssertExpectations(t) - - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) - - jobService := service.NewJobService(jobRepo, nil, downstreamRepo, nil, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) - - jobSpecA, err := job.NewSpecBuilder(1, "jobA", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) - jobSpecB, err := job.NewSpecBuilder(1, "jobB", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) - jobSpecC, err := job.NewSpecBuilder(1, "jobC", "optimus@goto", jobSchedule, jobWindow, jobTask).Build() - assert.NoError(t, err) - - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC}, false) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURNA}, false) - jobC := job.NewJob(sampleTenant, jobSpecC, resourceURNC, []resource.URN{resourceURNB}, false) - - upstreamB := job.NewUpstreamResolved(jobSpecB.Name(), "", resourceURNB, sampleTenant, "static", taskName, false) - jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamB}) - upstreamC := job.NewUpstreamResolved(jobSpecC.Name(), "", resourceURNC, sampleTenant, "static", taskName, false) - jobBWithUpstream := job.NewWithUpstream(jobB, []*job.Upstream{upstreamC}) - upstreamA := job.NewUpstreamResolved(jobSpecA.Name(), "", resourceURNA, sampleTenant, "static", taskName, false) - jobCWithUpstream := job.NewWithUpstream(jobC, []*job.Upstream{upstreamA}) - - jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA, jobB, jobC}, nil) - - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("jobA")).Return(jobA, nil) - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("jobB")).Return(jobB, nil) - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("jobC")).Return(jobC, nil) - - upstreamResolver.On("CheckStaticResolvable", ctx, sampleTenant, mock.Anything, mock.Anything).Return(nil) - upstreamResolver.On("BulkResolve", ctx, sampleTenant.ProjectName(), mock.Anything, mock.Anything).Return([]*job.WithUpstream{jobAWithUpstream, jobBWithUpstream, jobCWithUpstream}, nil) - - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{"jobA", "jobB", "jobC"}, - DeletionMode: false, - } - - expectedResult := map[job.Name][]dto.ValidateResult{ - "jobA": { - { - Stage: "cyclic validation", - Messages: []string{"cyclic dependency is detected", "jobA", "jobC", "jobB", "jobA"}, - Success: false, - }, - }, - "jobB": { - { - Stage: "cyclic validation", - Messages: []string{"cyclic dependency is detected", "jobA", "jobC", "jobB", "jobA"}, - Success: false, - }, - }, - "jobC": { - { - Stage: "cyclic validation", - Messages: []string{"cyclic dependency is detected", "jobA", "jobC", "jobB", "jobA"}, - Success: false, - }, - }, - } - - actualResult, actualError := jobService.Validate(ctx, request) - - assert.EqualValues(t, expectedResult["jobA"], actualResult["jobA"]) - assert.EqualValues(t, expectedResult["jobB"], actualResult["jobB"]) - assert.EqualValues(t, expectedResult["jobC"], actualResult["jobC"]) - assert.NoError(t, actualError) - }) - }) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) + + upstreamResolver := new(UpstreamResolver) + defer upstreamResolver.AssertExpectations(t) + + jobRepo := new(JobRepository) + defer jobRepo.AssertExpectations(t) + + upstreamRepo := new(UpstreamRepository) + defer upstreamRepo.AssertExpectations(t) - t.Run("validate jobs", func(t *testing.T) { - t.Run("returns result and nil if error is encountered when generating destination urn", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) + downstreamRepo := new(DownstreamRepository) + defer downstreamRepo.AssertExpectations(t) + + logWriter := new(mockWriter) + defer logWriter.AssertExpectations(t) + + jobTaskPython := job.NewTask("python", jobTaskConfig) + upstreamsSpecA, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"test-proj/job-C"}).Build() + upstreamsSpecB, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"test-proj/job-A"}).Build() + upstreamsSpecC, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"test-proj/job-B"}).Build() + specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTaskPython).WithSpecUpstream(upstreamsSpecA).Build() + specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTaskPython).WithSpecUpstream(upstreamsSpecB).Build() + specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTaskPython).WithSpecUpstream(upstreamsSpecC).Build() + specs := []*job.Spec{specA, specB, specC} + + jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{}, nil) + + logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) + upstreamResolver.On("CheckStaticResolvable", ctx, detailedTenant.ToTenant(), mock.Anything, mock.Anything).Return(nil) - upstreamResolver := new(UpstreamResolver) - defer upstreamResolver.AssertExpectations(t) + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return("", nil) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return([]string{}, nil) - downstreamRepo := new(DownstreamRepository) - defer downstreamRepo.AssertExpectations(t) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) + err := jobService.Validate(ctx, sampleTenant, specs, jobNamesWithInvalidSpec, logWriter) + assert.Error(t, err) + assert.ErrorContains(t, err, "a cycle dependency encountered in the tree:") + }) + t.Run("returns no error when success", func(t *testing.T) { + tenantDetailsGetter := new(TenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, mock.Anything).Return(detailedTenant, nil) + defer tenantDetailsGetter.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) + upstreamResolver := new(UpstreamResolver) + defer upstreamResolver.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), jobRunInputCompiler, resourceExistenceChecker) + jobRepo := new(JobRepository) + defer jobRepo.AssertExpectations(t) - jobSpecA, err := job.NewSpecBuilder(1, "jobA", "optimus@goto", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - assert.NoError(t, err) - jobSpecB, err := job.NewSpecBuilder(1, "jobB", "optimus@goto", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - assert.NoError(t, err) + upstreamRepo := new(UpstreamRepository) + defer upstreamRepo.AssertExpectations(t) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC}, false) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURNA}, false) + downstreamRepo := new(DownstreamRepository) + defer downstreamRepo.AssertExpectations(t) - upstreamB := job.NewUpstreamResolved(jobSpecB.Name(), "", resourceURNB, sampleTenant, "static", taskName, false) - jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamB}) - jobBWithUpstream := job.NewWithUpstream(jobB, []*job.Upstream{}) + logWriter := new(mockWriter) + defer logWriter.AssertExpectations(t) - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("jobA")).Return(jobA, nil) - jobRepo.On("GetByJobName", ctx, sampleTenant.ProjectName(), job.Name("jobB")).Return(jobB, nil) - jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA, jobB}, nil) + jobTaskConfigA, _ := job.ConfigFrom(map[string]string{"table": "table-A", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskConfigB, _ := job.ConfigFrom(map[string]string{"table": "table-B", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskConfigC, _ := job.ConfigFrom(map[string]string{"table": "table-C", "BQ_SERVICE_ACCOUNT": "service_account"}) + jobTaskA := job.NewTask(taskName, jobTaskConfigA) + jobTaskB := job.NewTask(taskName, jobTaskConfigB) + jobTaskC := job.NewTask(taskName, jobTaskConfigC) - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) + specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).Build() + specAUpdated, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner-updated", jobSchedule, jobWindow, jobTaskA).WithAsset(jobAsset).Build() + specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTaskB).WithAsset(jobAsset).Build() + specC, _ := job.NewSpecBuilder(jobVersion, "job-C", "sample-owner", jobSchedule, jobWindow, jobTaskC).WithAsset(jobAsset).Build() + specs := []*job.Spec{specAUpdated, specB} - upstreamResolver.On("CheckStaticResolvable", ctx, sampleTenant, mock.Anything, mock.Anything).Return(nil) - upstreamResolver.On("BulkResolve", ctx, sampleTenant.ProjectName(), mock.Anything, mock.Anything).Return([]*job.WithUpstream{jobAWithUpstream, jobBWithUpstream}, nil) + jobA := job.NewJob(sampleTenant, specA, "bigquery://project:dataset.tableA", []job.ResourceURN{"bigquery://project:dataset.tableZ"}, false) + jobB := job.NewJob(sampleTenant, specB, "bigquery://project:dataset.tableB", []job.ResourceURN{"bigquery://project:dataset.tableA"}, false) + jobC := job.NewJob(sampleTenant, specC, "bigquery://project:dataset.tableC", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) - pluginService.On("ConstructDestinationURN", ctx, jobTask.Name().String(), mock.Anything).Return(resource.ZeroURN(), errors.New("unknown error")).Twice() + jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA, jobB, jobC}, nil) - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: nil, - JobNames: []string{"jobA", "jobB"}, - DeletionMode: false, - } + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(job.ResourceURN("bigquery://project:dataset.tableA").String(), nil) + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString([]job.ResourceURN{"bigquery://project:dataset.tableZ"}), nil) - expectedResult := map[job.Name][]dto.ValidateResult{ - "jobA": { - { - Stage: "destination validation", - Messages: []string{"can not generate destination resource", "unknown error"}, - Success: false, - }, - }, - "jobB": { - { - Stage: "destination validation", - Messages: []string{"can not generate destination resource", "unknown error"}, - Success: false, - }, - }, - } + upstreamResolver.On("CheckStaticResolvable", ctx, detailedTenant.ToTenant(), mock.Anything, mock.Anything).Return(nil) - actualResult, actualError := jobService.Validate(ctx, request) + downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specC.Name()).Return([]*job.Downstream{}, nil) - assert.EqualValues(t, expectedResult["jobA"], actualResult["jobA"]) - assert.EqualValues(t, expectedResult["jobB"], actualResult["jobB"]) - assert.NoError(t, actualError) - }) - - t.Run("returns unsuccessful result and nil if one or more validations failed", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) - - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) - - downstreamRepo := new(DownstreamRepository) - defer downstreamRepo.AssertExpectations(t) - - upstreamResolver := new(UpstreamResolver) - defer upstreamResolver.AssertExpectations(t) - - pluginService := NewPluginService(t) - - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) - - jobService := service.NewJobService(jobRepo, nil, downstreamRepo, - pluginService, upstreamResolver, tenantDetailsGetter, nil, - log, nil, compiler.NewEngine(), - jobRunInputCompiler, resourceExistenceChecker, - ) - - resourceURND, err := resource.ParseURN("bigquery://project:dataset.tableD") - assert.NoError(t, err) - - jobSpecA, err := job.NewSpecBuilder(1, "jobA", "optimus@goto", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - assert.NoError(t, err) - jobSpecB, err := job.NewSpecBuilder(1, "jobB", "optimus@goto", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - assert.NoError(t, err) - - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC}, false) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURNA}, false) - - upstreamB := job.NewUpstreamResolved(jobSpecB.Name(), "", resourceURNB, sampleTenant, "static", taskName, false) - jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamB}) - jobBWithUpstream := job.NewWithUpstream(jobB, []*job.Upstream{}) - - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - - pluginService.On("ConstructDestinationURN", ctx, jobTask.Name().String(), mock.Anything).Return(resource.ZeroURN(), nil) - sourcesToValidate := []resource.URN{resourceURNA, resourceURNB, resourceURNC, resourceURND} - pluginService.On("IdentifyUpstreams", ctx, jobTask.Name().String(), mock.Anything, mock.Anything).Return(sourcesToValidate, nil) - - jobRunInputCompiler.On("Compile", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("unexpected compile error")) - - resourceExistenceChecker.On("GetByURN", ctx, sampleTenant, resourceURNA).Return(nil, errors.New("unexpected get by urn error")) - resourceExistenceChecker.On("ExistInStore", ctx, sampleTenant, resourceURNA).Return(false, errors.New("unexpected exist in store error")) - - rsc, err := resource.NewResource("resource_1", "table", resource.Bigquery, sampleTenant, &resource.Metadata{Description: "table for test"}, map[string]any{"version": 1}) - assert.NoError(t, err) - deletedRsc := resource.FromExisting(rsc, resource.ReplaceStatus(resource.StatusDeleted)) - - jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA, jobB}, nil) - - resourceExistenceChecker.On("GetByURN", ctx, sampleTenant, resourceURNB).Return(deletedRsc, nil) - resourceExistenceChecker.On("ExistInStore", ctx, sampleTenant, resourceURNB).Return(false, nil) - - resourceExistenceChecker.On("GetByURN", ctx, sampleTenant, resourceURNC).Return(rsc, nil) - resourceExistenceChecker.On("ExistInStore", ctx, sampleTenant, resourceURNC).Return(false, nil) - - resourceExistenceChecker.On("GetByURN", ctx, sampleTenant, resourceURND).Return(rsc, nil) - resourceExistenceChecker.On("ExistInStore", ctx, sampleTenant, resourceURND).Return(true, nil) - - upstreamResolver.On("CheckStaticResolvable", ctx, sampleTenant, mock.Anything, mock.Anything).Return(nil) - upstreamResolver.On("BulkResolve", ctx, sampleTenant.ProjectName(), mock.Anything, mock.Anything).Return([]*job.WithUpstream{jobAWithUpstream, jobBWithUpstream}, nil) - - upstreamResolver.On("Resolve", ctx, mock.Anything, mock.Anything).Return(nil, errors.New("unexpected errors in upstream resolve")).Once() - upstreamResolver.On("Resolve", ctx, mock.Anything, mock.Anything).Return(nil, nil).Once() - - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: []*job.Spec{jobSpecA, jobSpecB}, - JobNames: nil, - DeletionMode: false, - } - - expectedResult := map[job.Name][]dto.ValidateResult{ - "jobA": { - { - Stage: "destination validation", - Messages: []string{"no issue"}, - Success: true, - }, - { - Stage: "source validation", - Messages: []string{ - "bigquery://project:dataset.tableA: unexpected exist in store error", - "bigquery://project:dataset.tableB: resource does not exist in both db and store", - "bigquery://project:dataset.tableC: resource exists in db but not in store", - "bigquery://project:dataset.tableD: no issue", - }, - Success: false, - }, - { - Stage: "window validation", - Messages: []string{"no issue"}, - Success: true, - }, - { - Stage: "compile validation for run", - Messages: []string{ - "compiling [bq2bq] with type [task] failed with error: unexpected compile error", - }, - Success: false, - }, - { - Stage: "upstream validation", - Messages: []string{ - "can not resolve upstream", - "unexpected errors in upstream resolve", - }, - Success: false, - }, - }, - "jobB": { - { - Stage: "destination validation", - Messages: []string{"no issue"}, - Success: true, - }, - { - Stage: "source validation", - Messages: []string{ - "bigquery://project:dataset.tableA: unexpected exist in store error", - "bigquery://project:dataset.tableB: resource does not exist in both db and store", - "bigquery://project:dataset.tableC: resource exists in db but not in store", - "bigquery://project:dataset.tableD: no issue", - }, - Success: false, - }, - { - Stage: "window validation", - Messages: []string{"no issue"}, - Success: true, - }, - { - Stage: "compile validation for run", - Messages: []string{ - "compiling [bq2bq] with type [task] failed with error: unexpected compile error", - }, - Success: false, - }, - { - Stage: "upstream validation", - Messages: []string{"no issue"}, - Success: true, - }, - }, - } - - actualResult, actualError := jobService.Validate(ctx, request) - - assert.EqualValues(t, expectedResult["jobA"], actualResult["jobA"]) - assert.EqualValues(t, expectedResult["jobB"], actualResult["jobB"]) - assert.NoError(t, actualError) - }) - - t.Run("returns successful result and nil if no validation failed", func(t *testing.T) { - tenantDetailsGetter := new(TenantDetailsGetter) - defer tenantDetailsGetter.AssertExpectations(t) - - jobRepo := new(JobRepository) - defer jobRepo.AssertExpectations(t) - - downstreamRepo := new(DownstreamRepository) - defer downstreamRepo.AssertExpectations(t) - - upstreamResolver := new(UpstreamResolver) - defer upstreamResolver.AssertExpectations(t) - - pluginService := NewPluginService(t) - - jobRunInputCompiler := NewJobRunInputCompiler(t) - resourceExistenceChecker := NewResourceExistenceChecker(t) - - jobService := service.NewJobService(jobRepo, nil, downstreamRepo, - pluginService, upstreamResolver, tenantDetailsGetter, nil, - log, nil, compiler.NewEngine(), - jobRunInputCompiler, resourceExistenceChecker, - ) - - resourceURND, err := resource.ParseURN("bigquery://project:dataset.tableD") - assert.NoError(t, err) - - jobSpecA, err := job.NewSpecBuilder(1, "jobA", "optimus@goto", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - assert.NoError(t, err) - jobSpecB, err := job.NewSpecBuilder(1, "jobB", "optimus@goto", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - assert.NoError(t, err) - - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC}, false) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURNA}, false) - - upstreamB := job.NewUpstreamResolved(jobSpecB.Name(), "", resourceURNB, sampleTenant, "static", taskName, false) - jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamB}) - jobBWithUpstream := job.NewWithUpstream(jobB, []*job.Upstream{}) - - jobRepo.On("GetAllByTenant", ctx, sampleTenant).Return([]*job.Job{jobA, jobB}, nil) - - tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - - pluginService.On("ConstructDestinationURN", ctx, jobTask.Name().String(), mock.Anything).Return(resource.ZeroURN(), nil) - sourcesToValidate := []resource.URN{resourceURNA, resourceURNB, resourceURNC, resourceURND} - pluginService.On("IdentifyUpstreams", ctx, jobTask.Name().String(), mock.Anything, mock.Anything).Return(sourcesToValidate, nil) - - jobRunInputCompiler.On("Compile", ctx, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) - - rsc, err := resource.NewResource("resource_1", "table", resource.Bigquery, sampleTenant, &resource.Metadata{Description: "table for test"}, map[string]any{"version": 1}) - assert.NoError(t, err) - - resourceExistenceChecker.On("GetByURN", ctx, sampleTenant, resourceURNA).Return(rsc, nil) - resourceExistenceChecker.On("ExistInStore", ctx, sampleTenant, resourceURNA).Return(true, nil) - - resourceExistenceChecker.On("GetByURN", ctx, sampleTenant, resourceURNB).Return(rsc, nil) - resourceExistenceChecker.On("ExistInStore", ctx, sampleTenant, resourceURNB).Return(true, nil) - - resourceExistenceChecker.On("GetByURN", ctx, sampleTenant, resourceURNC).Return(rsc, nil) - resourceExistenceChecker.On("ExistInStore", ctx, sampleTenant, resourceURNC).Return(true, nil) - - upstreamResolver.On("CheckStaticResolvable", ctx, sampleTenant, mock.Anything, mock.Anything).Return(nil) - upstreamResolver.On("BulkResolve", ctx, sampleTenant.ProjectName(), mock.Anything, mock.Anything).Return([]*job.WithUpstream{jobAWithUpstream, jobBWithUpstream}, nil) - - resourceExistenceChecker.On("GetByURN", ctx, sampleTenant, resourceURND).Return(rsc, nil) - resourceExistenceChecker.On("ExistInStore", ctx, sampleTenant, resourceURND).Return(true, nil) - - upstreamResolver.On("Resolve", ctx, mock.Anything, mock.Anything).Return(nil, nil) - - request := dto.ValidateRequest{ - Tenant: sampleTenant, - JobSpecs: []*job.Spec{jobSpecA, jobSpecB}, - JobNames: nil, - DeletionMode: false, - } - - expectedResult := map[job.Name][]dto.ValidateResult{ - "jobA": { - { - Stage: "destination validation", - Messages: []string{"no issue"}, - Success: true, - }, - { - Stage: "source validation", - Messages: []string{ - "bigquery://project:dataset.tableA: no issue", - "bigquery://project:dataset.tableB: no issue", - "bigquery://project:dataset.tableC: no issue", - "bigquery://project:dataset.tableD: no issue", - }, - Success: true, - }, - { - Stage: "window validation", - Messages: []string{"no issue"}, - Success: true, - }, - { - Stage: "compile validation for run", - Messages: []string{"compiling [bq2bq] with type [task] contains no issue"}, - Success: true, - }, - { - Stage: "upstream validation", - Messages: []string{"no issue"}, - Success: true, - }, - }, - "jobB": { - { - Stage: "destination validation", - Messages: []string{"no issue"}, - Success: true, - }, - { - Stage: "source validation", - Messages: []string{ - "bigquery://project:dataset.tableA: no issue", - "bigquery://project:dataset.tableB: no issue", - "bigquery://project:dataset.tableC: no issue", - "bigquery://project:dataset.tableD: no issue", - }, - Success: true, - }, - { - Stage: "window validation", - Messages: []string{"no issue"}, - Success: true, - }, - { - Stage: "compile validation for run", - Messages: []string{"compiling [bq2bq] with type [task] contains no issue"}, - Success: true, - }, - { - Stage: "upstream validation", - Messages: []string{"no issue"}, - Success: true, - }, - }, - } - - actualResult, actualError := jobService.Validate(ctx, request) - - assert.EqualValues(t, expectedResult["jobA"], actualResult["jobA"]) - assert.EqualValues(t, expectedResult["jobB"], actualResult["jobB"]) - assert.NoError(t, actualError) - }) - }) + logWriter.On("Write", mock.Anything, mock.Anything).Return(nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) + err := jobService.Validate(ctx, sampleTenant, specs, jobNamesWithInvalidSpec, logWriter) + assert.NoError(t, err) }) }) @@ -3773,7 +3351,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -3782,13 +3361,13 @@ func TestJobService(t *testing.T) { defer tenantDetailsGetter.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "bigquery://project:dataset.tableA", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) - upstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "inferred", taskName, false) + upstreamB := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "inferred", taskName, false) upstreamRepo.On("GetUpstreams", ctx, project.Name(), jobA.Spec().Name()).Return([]*job.Upstream{upstreamB}, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, nil) result, err := jobService.GetUpstreamsToInspect(ctx, jobA, false) assert.NoError(t, err) assert.EqualValues(t, []*job.Upstream{upstreamB}, result) @@ -3803,7 +3382,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -3815,13 +3395,13 @@ func TestJobService(t *testing.T) { defer logWriter.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, specA, "bigquery://project:dataset.tableA", []job.ResourceURN{"bigquery://project:dataset.tableB"}, false) - upstreamB := job.NewUpstreamResolved("job-B", "", resourceURNB, sampleTenant, "inferred", taskName, false) + upstreamB := job.NewUpstreamResolved("job-B", "", "bigquery://project:dataset.tableB", sampleTenant, "inferred", taskName, false) upstreamResolver.On("Resolve", ctx, jobA, mock.Anything).Return([]*job.Upstream{upstreamB}, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, nil) result, err := jobService.GetUpstreamsToInspect(ctx, jobA, true) assert.NoError(t, err) assert.EqualValues(t, []*job.Upstream{upstreamB}, result) @@ -3839,7 +3419,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -3851,17 +3432,17 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - jobADestination := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil) + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil) - jobASources := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobASources, nil) + jobASources := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobASources), nil) jobRepo.On("GetAllByResourceDestination", ctx, jobADestination).Return([]*job.Job{}, nil) jobA := job.NewJob(sampleTenant, specA, jobADestination, jobASources, false) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) result, logger := jobService.GetJobBasicInfo(ctx, sampleTenant, "", specA) assert.Nil(t, logger.Messages) assert.Equal(t, jobA, result) @@ -3876,7 +3457,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -3885,14 +3467,14 @@ func TestJobService(t *testing.T) { defer tenantDetailsGetter.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobASources := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobASources := []job.ResourceURN{"bigquery://project:dataset.tableB"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobASources, false) jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(jobA, nil) jobRepo.On("GetAllByResourceDestination", ctx, jobADestination).Return([]*job.Job{}, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) result, logger := jobService.GetJobBasicInfo(ctx, sampleTenant, specA.Name(), nil) assert.Nil(t, logger.Messages) assert.Equal(t, jobA, result) @@ -3907,7 +3489,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -3919,7 +3502,7 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, errors.New("sample error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) result, logger := jobService.GetJobBasicInfo(ctx, sampleTenant, "", specA) assert.Contains(t, logger.Messages[0].Message, "sample error") assert.Nil(t, result) @@ -3934,7 +3517,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -3946,13 +3530,13 @@ func TestJobService(t *testing.T) { tenantDetailsGetter.On("GetDetails", ctx, sampleTenant).Return(detailedTenant, nil) - jobADestination := resourceURNA - pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination, nil).Once() + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + pluginService.On("ConstructDestinationURN", ctx, specA.Task().Name().String(), mock.Anything).Return(jobADestination.String(), nil).Once() - jobAUpstreamName := []resource.URN{resourceURNB} - pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobAUpstreamName, errors.New("sample error")) + jobAUpstreamName := []job.ResourceURN{"bigquery://project:dataset.tableB"} + pluginService.On("IdentifyUpstreams", ctx, specA.Task().Name().String(), mock.Anything, mock.Anything).Return(jobResourceURNsToString(jobAUpstreamName), errors.New("sample error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) result, logger := jobService.GetJobBasicInfo(ctx, sampleTenant, "", specA) assert.Contains(t, logger.Messages[0].Message, "sample error") assert.Nil(t, result) @@ -3967,7 +3551,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -3979,13 +3564,13 @@ func TestJobService(t *testing.T) { assert.NoError(t, err) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", specASchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") jobA := job.NewJob(sampleTenant, specA, jobADestination, nil, false) jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(jobA, nil) jobRepo.On("GetAllByResourceDestination", ctx, jobADestination).Return([]*job.Job{}, errors.New("sample-error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) result, logger := jobService.GetJobBasicInfo(ctx, sampleTenant, specA.Name(), nil) assert.Contains(t, logger.Messages[0].Message, "no job sources detected") assert.Contains(t, logger.Messages[1].Message, "catchup is enabled") @@ -4002,7 +3587,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -4014,7 +3600,7 @@ func TestJobService(t *testing.T) { jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(nil, errors.New("internal error")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) result, logger := jobService.GetJobBasicInfo(ctx, sampleTenant, specA.Name(), nil) assert.Contains(t, logger.Messages[0].Message, "internal error") assert.Nil(t, result) @@ -4029,7 +3615,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -4041,7 +3628,7 @@ func TestJobService(t *testing.T) { jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(nil, errors.New("job not found")) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) result, logger := jobService.GetJobBasicInfo(ctx, sampleTenant, specA.Name(), nil) assert.Contains(t, logger.Messages[0].Message, "job not found") assert.Nil(t, result) @@ -4056,7 +3643,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -4065,19 +3653,19 @@ func TestJobService(t *testing.T) { defer tenantDetailsGetter.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobASources := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobASources := []job.ResourceURN{"bigquery://project:dataset.tableB"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobASources, false) specB, _ := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobBDestination := resourceURNB - jobBSources := []resource.URN{resourceURNC} + jobBDestination := job.ResourceURN("bigquery://project:dataset.tableB") + jobBSources := []job.ResourceURN{"bigquery://project:dataset.tableC"} jobB := job.NewJob(sampleTenant, specB, jobBDestination, jobBSources, false) jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(jobA, nil) jobRepo.On("GetAllByResourceDestination", ctx, jobADestination).Return([]*job.Job{jobB, jobA}, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) result, logger := jobService.GetJobBasicInfo(ctx, sampleTenant, specA.Name(), nil) assert.Contains(t, logger.Messages[0].Message, "job already exists with same Destination") assert.Equal(t, jobA, result) @@ -4092,7 +3680,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -4101,14 +3690,14 @@ func TestJobService(t *testing.T) { defer tenantDetailsGetter.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobADestination := resourceURNA - jobASources := []resource.URN{resourceURNB} + jobADestination := job.ResourceURN("bigquery://project:dataset.tableA") + jobASources := []job.ResourceURN{"bigquery://project:dataset.tableB"} jobA := job.NewJob(sampleTenant, specA, jobADestination, jobASources, false) jobRepo.On("GetByJobName", ctx, project.Name(), specA.Name()).Return(jobA, nil) jobRepo.On("GetAllByResourceDestination", ctx, jobADestination).Return([]*job.Job{jobA}, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine(), nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, compiler.NewEngine()) result, logger := jobService.GetJobBasicInfo(ctx, sampleTenant, specA.Name(), nil) assert.Nil(t, logger.Messages) assert.Equal(t, jobA, result) @@ -4126,7 +3715,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -4135,14 +3725,14 @@ func TestJobService(t *testing.T) { defer tenantDetailsGetter.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, nil, false) + jobA := job.NewJob(sampleTenant, specA, "bigquery://project:dataset.tableA", nil, false) jobADownstream := []*job.Downstream{ job.NewDownstream("job-B", project.Name(), namespace.Name(), taskName), } downstreamRepo.On("GetDownstreamByDestination", ctx, project.Name(), jobA.Destination()).Return(jobADownstream, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, nil) result, err := jobService.GetDownstream(ctx, jobA, true) assert.NoError(t, err) assert.Equal(t, jobADownstream, result) @@ -4157,7 +3747,8 @@ func TestJobService(t *testing.T) { downstreamRepo := new(DownstreamRepository) defer downstreamRepo.AssertExpectations(t) - pluginService := NewPluginService(t) + pluginService := new(PluginService) + defer pluginService.AssertExpectations(t) upstreamResolver := new(UpstreamResolver) defer upstreamResolver.AssertExpectations(t) @@ -4166,14 +3757,14 @@ func TestJobService(t *testing.T) { defer tenantDetailsGetter.AssertExpectations(t) specA, _ := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).WithAsset(jobAsset).Build() - jobA := job.NewJob(sampleTenant, specA, resourceURNA, nil, false) + jobA := job.NewJob(sampleTenant, specA, "bigquery://project:dataset.tableA", nil, false) jobADownstream := []*job.Downstream{ job.NewDownstream("job-B", project.Name(), namespace.Name(), taskName), } downstreamRepo.On("GetDownstreamByJobName", ctx, project.Name(), specA.Name()).Return(jobADownstream, nil) - jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, nil, nil, nil) + jobService := service.NewJobService(jobRepo, upstreamRepo, downstreamRepo, pluginService, upstreamResolver, tenantDetailsGetter, nil, log, nil, nil) result, err := jobService.GetDownstream(ctx, jobA, false) assert.NoError(t, err) assert.Equal(t, jobADownstream, result) @@ -4189,7 +3780,7 @@ func TestJobService(t *testing.T) { jobDeploymentService.On("UpdateJobScheduleState", ctx, sampleTenant, jobsToUpdateState, state.String()).Return(fmt.Errorf("some error in update Job State")) defer jobDeploymentService.AssertExpectations(t) - jobService := service.NewJobService(nil, nil, nil, nil, nil, nil, nil, nil, jobDeploymentService, nil, nil, nil) + jobService := service.NewJobService(nil, nil, nil, nil, nil, nil, nil, nil, jobDeploymentService, nil) err := jobService.UpdateState(ctx, sampleTenant, jobsToUpdateState, state, "job disable remark") assert.ErrorContains(t, err, "some error in update Job State") }) @@ -4203,7 +3794,7 @@ func TestJobService(t *testing.T) { jobRepo.On("UpdateState", ctx, sampleTenant, jobsToUpdateState, state, updateRemark).Return(fmt.Errorf("some error in update Job State repo")) defer jobRepo.AssertExpectations(t) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, jobDeploymentService, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, nil, nil, jobDeploymentService, nil) err := jobService.UpdateState(ctx, sampleTenant, jobsToUpdateState, state, updateRemark) assert.ErrorContains(t, err, "some error in update Job State repo") }) @@ -4220,7 +3811,7 @@ func TestJobService(t *testing.T) { eventHandler := newEventHandler(t) eventHandler.On("HandleEvent", mock.Anything).Times(1) - jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, eventHandler, nil, jobDeploymentService, nil, nil, nil) + jobService := service.NewJobService(jobRepo, nil, nil, nil, nil, nil, eventHandler, nil, jobDeploymentService, nil) err := jobService.UpdateState(ctx, sampleTenant, jobsToUpdateState, state, updateRemark) assert.Nil(t, err) }) @@ -4228,9 +3819,7 @@ func TestJobService(t *testing.T) { t.Run("GetDownstreamByResourceURN", func(t *testing.T) { tnnt, _ := tenant.NewTenant("proj123", "name123") - urn, err := resource.ParseURN("bigquery://dataset.table_test") - assert.NoError(t, err) - + urn := job.ResourceURN("bigquery://dataset.table_test") downstreamJobs := []*job.Downstream{ job.NewDownstream("jobA", tnnt.ProjectName(), tnnt.NamespaceName(), "jobA"), job.NewDownstream("jobA", "proj501", "name4012", "jobA"), @@ -4239,10 +3828,10 @@ func TestJobService(t *testing.T) { t.Run("success found dependent job", func(t *testing.T) { var ( downstreamRepo = new(DownstreamRepository) - jobService = service.NewJobService(nil, nil, downstreamRepo, nil, nil, nil, nil, nil, nil, nil, nil, nil) + jobService = service.NewJobService(nil, nil, downstreamRepo, nil, nil, nil, nil, nil, nil, nil) ) - downstreamRepo.On("GetDownstreamBySources", ctx, []resource.URN{urn}).Return(downstreamJobs, nil) + downstreamRepo.On("GetDownstreamBySources", ctx, []job.ResourceURN{urn}).Return(downstreamJobs, nil) actual, err := jobService.GetDownstreamByResourceURN(ctx, tnnt, urn) assert.NoError(t, err) @@ -4253,10 +3842,10 @@ func TestJobService(t *testing.T) { t.Run("return error when GetDownstreamBySources", func(t *testing.T) { var ( downstreamRepo = new(DownstreamRepository) - jobService = service.NewJobService(nil, nil, downstreamRepo, nil, nil, nil, nil, nil, nil, nil, nil, nil) + jobService = service.NewJobService(nil, nil, downstreamRepo, nil, nil, nil, nil, nil, nil, nil) ) - downstreamRepo.On("GetDownstreamBySources", ctx, []resource.URN{urn}).Return(nil, context.DeadlineExceeded) + downstreamRepo.On("GetDownstreamBySources", ctx, []job.ResourceURN{urn}).Return(nil, context.DeadlineExceeded) actual, err := jobService.GetDownstreamByResourceURN(ctx, tnnt, urn) assert.Error(t, err) @@ -4355,11 +3944,11 @@ func (_m *JobRepository) GetAllByProjectName(ctx context.Context, projectName te } // GetAllByResourceDestination provides a mock function with given fields: ctx, resourceDestination -func (_m *JobRepository) GetAllByResourceDestination(ctx context.Context, resourceDestination resource.URN) ([]*job.Job, error) { +func (_m *JobRepository) GetAllByResourceDestination(ctx context.Context, resourceDestination job.ResourceURN) ([]*job.Job, error) { ret := _m.Called(ctx, resourceDestination) var r0 []*job.Job - if rf, ok := ret.Get(0).(func(context.Context, resource.URN) []*job.Job); ok { + if rf, ok := ret.Get(0).(func(context.Context, job.ResourceURN) []*job.Job); ok { r0 = rf(ctx, resourceDestination) } else { if ret.Get(0) != nil { @@ -4368,7 +3957,7 @@ func (_m *JobRepository) GetAllByResourceDestination(ctx context.Context, resour } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, resource.URN) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, job.ResourceURN) error); ok { r1 = rf(ctx, resourceDestination) } else { r1 = ret.Error(1) @@ -4451,11 +4040,11 @@ type DownstreamRepository struct { } // GetDownstreamByDestination provides a mock function with given fields: ctx, projectName, destination -func (_d *DownstreamRepository) GetDownstreamByDestination(ctx context.Context, projectName tenant.ProjectName, destination resource.URN) ([]*job.Downstream, error) { +func (_d *DownstreamRepository) GetDownstreamByDestination(ctx context.Context, projectName tenant.ProjectName, destination job.ResourceURN) ([]*job.Downstream, error) { ret := _d.Called(ctx, projectName, destination) var r0 []*job.Downstream - if rf, ok := ret.Get(0).(func(context.Context, tenant.ProjectName, resource.URN) []*job.Downstream); ok { + if rf, ok := ret.Get(0).(func(context.Context, tenant.ProjectName, job.ResourceURN) []*job.Downstream); ok { r0 = rf(ctx, projectName, destination) } else { if ret.Get(0) != nil { @@ -4464,7 +4053,7 @@ func (_d *DownstreamRepository) GetDownstreamByDestination(ctx context.Context, } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, tenant.ProjectName, resource.URN) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, tenant.ProjectName, job.ResourceURN) error); ok { r1 = rf(ctx, projectName, destination) } else { r1 = ret.Error(1) @@ -4497,11 +4086,11 @@ func (_d *DownstreamRepository) GetDownstreamByJobName(ctx context.Context, proj } // GetDownstreamBySources provides a mock function with given fields: ctx, sources -func (_d *DownstreamRepository) GetDownstreamBySources(ctx context.Context, sources []resource.URN) ([]*job.Downstream, error) { +func (_d *DownstreamRepository) GetDownstreamBySources(ctx context.Context, sources []job.ResourceURN) ([]*job.Downstream, error) { ret := _d.Called(ctx, sources) var r0 []*job.Downstream - if rf, ok := ret.Get(0).(func(context.Context, []resource.URN) []*job.Downstream); ok { + if rf, ok := ret.Get(0).(func(context.Context, []job.ResourceURN) []*job.Downstream); ok { r0 = rf(ctx, sources) } else { if ret.Get(0) != nil { @@ -4510,7 +4099,7 @@ func (_d *DownstreamRepository) GetDownstreamBySources(ctx context.Context, sour } var r1 error - if rf, ok := ret.Get(1).(func(context.Context, []resource.URN) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, []job.ResourceURN) error); ok { r1 = rf(ctx, sources) } else { r1 = ret.Error(1) @@ -4588,27 +4177,23 @@ type PluginService struct { mock.Mock } -// ConstructDestinationURN provides a mock function with given fields: ctx, taskName, compiledConfig -func (_m *PluginService) ConstructDestinationURN(ctx context.Context, taskName string, compiledConfig map[string]string) (resource.URN, error) { - ret := _m.Called(ctx, taskName, compiledConfig) +// ConstructDestinationURN provides a mock function with given fields: ctx, taskName, config +func (_m *PluginService) ConstructDestinationURN(ctx context.Context, taskName string, config map[string]string) (string, error) { + ret := _m.Called(ctx, taskName, config) - if len(ret) == 0 { - panic("no return value specified for ConstructDestinationURN") - } - - var r0 resource.URN + var r0 string var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, map[string]string) (resource.URN, error)); ok { - return rf(ctx, taskName, compiledConfig) + if rf, ok := ret.Get(0).(func(context.Context, string, map[string]string) (string, error)); ok { + return rf(ctx, taskName, config) } - if rf, ok := ret.Get(0).(func(context.Context, string, map[string]string) resource.URN); ok { - r0 = rf(ctx, taskName, compiledConfig) + if rf, ok := ret.Get(0).(func(context.Context, string, map[string]string) string); ok { + r0 = rf(ctx, taskName, config) } else { - r0 = ret.Get(0).(resource.URN) + r0 = ret.Get(0).(string) } if rf, ok := ret.Get(1).(func(context.Context, string, map[string]string) error); ok { - r1 = rf(ctx, taskName, compiledConfig) + r1 = rf(ctx, taskName, config) } else { r1 = ret.Error(1) } @@ -4616,29 +4201,25 @@ func (_m *PluginService) ConstructDestinationURN(ctx context.Context, taskName s return r0, r1 } -// IdentifyUpstreams provides a mock function with given fields: ctx, taskName, compiledConfig, assets -func (_m *PluginService) IdentifyUpstreams(ctx context.Context, taskName string, compiledConfig, assets map[string]string) ([]resource.URN, error) { - ret := _m.Called(ctx, taskName, compiledConfig, assets) +// IdentifyUpstreams provides a mock function with given fields: ctx, taskName, config, assets +func (_m *PluginService) IdentifyUpstreams(ctx context.Context, taskName string, config, assets map[string]string) ([]string, error) { + ret := _m.Called(ctx, taskName, config, assets) - if len(ret) == 0 { - panic("no return value specified for IdentifyUpstreams") - } - - var r0 []resource.URN + var r0 []string var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, map[string]string, map[string]string) ([]resource.URN, error)); ok { - return rf(ctx, taskName, compiledConfig, assets) + if rf, ok := ret.Get(0).(func(context.Context, string, map[string]string, map[string]string) ([]string, error)); ok { + return rf(ctx, taskName, config, assets) } - if rf, ok := ret.Get(0).(func(context.Context, string, map[string]string, map[string]string) []resource.URN); ok { - r0 = rf(ctx, taskName, compiledConfig, assets) + if rf, ok := ret.Get(0).(func(context.Context, string, map[string]string, map[string]string) []string); ok { + r0 = rf(ctx, taskName, config, assets) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]resource.URN) + r0 = ret.Get(0).([]string) } } if rf, ok := ret.Get(1).(func(context.Context, string, map[string]string, map[string]string) error); ok { - r1 = rf(ctx, taskName, compiledConfig, assets) + r1 = rf(ctx, taskName, config, assets) } else { r1 = ret.Error(1) } @@ -4650,10 +4231,6 @@ func (_m *PluginService) IdentifyUpstreams(ctx context.Context, taskName string, func (_m *PluginService) Info(ctx context.Context, taskName string) (*plugin.Info, error) { ret := _m.Called(ctx, taskName) - if len(ret) == 0 { - panic("no return value specified for Info") - } - var r0 *plugin.Info var r1 error if rf, ok := ret.Get(0).(func(context.Context, string) (*plugin.Info, error)); ok { @@ -4676,21 +4253,6 @@ func (_m *PluginService) Info(ctx context.Context, taskName string) (*plugin.Inf return r0, r1 } -// NewPluginService creates a new instance of PluginService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewPluginService(t interface { - mock.TestingT - Cleanup(func()) -}, -) *PluginService { - mock := &PluginService{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} - // UpstreamResolver is an autogenerated mock type for the UpstreamResolver type type UpstreamResolver struct { mock.Mock @@ -4829,130 +4391,10 @@ func (_m *JobDeploymentService) UpdateJobScheduleState(ctx context.Context, tnnt return args.Error(0) } -// JobRunInputCompiler is an autogenerated mock type for the JobRunInputCompiler type -type JobRunInputCompiler struct { - mock.Mock -} - -// Compile provides a mock function with given fields: ctx, job, config, executedAt -func (_m *JobRunInputCompiler) Compile(ctx context.Context, job *scheduler.JobWithDetails, config scheduler.RunConfig, executedAt time.Time) (*scheduler.ExecutorInput, error) { - ret := _m.Called(ctx, job, config, executedAt) - - if len(ret) == 0 { - panic("no return value specified for Compile") - } - - var r0 *scheduler.ExecutorInput - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *scheduler.JobWithDetails, scheduler.RunConfig, time.Time) (*scheduler.ExecutorInput, error)); ok { - return rf(ctx, job, config, executedAt) - } - if rf, ok := ret.Get(0).(func(context.Context, *scheduler.JobWithDetails, scheduler.RunConfig, time.Time) *scheduler.ExecutorInput); ok { - r0 = rf(ctx, job, config, executedAt) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*scheduler.ExecutorInput) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *scheduler.JobWithDetails, scheduler.RunConfig, time.Time) error); ok { - r1 = rf(ctx, job, config, executedAt) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewJobRunInputCompiler creates a new instance of JobRunInputCompiler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewJobRunInputCompiler(t interface { - mock.TestingT - Cleanup(func()) -}, -) *JobRunInputCompiler { - mock := &JobRunInputCompiler{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} - -// ResourceExistenceChecker is an autogenerated mock type for the ResourceExistenceChecker type -type ResourceExistenceChecker struct { - mock.Mock -} - -// ExistInStore provides a mock function with given fields: ctx, tnnt, urn -func (_m *ResourceExistenceChecker) ExistInStore(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) { - ret := _m.Called(ctx, tnnt, urn) - - if len(ret) == 0 { - panic("no return value specified for ExistInStore") - } - - var r0 bool - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, tenant.Tenant, resource.URN) (bool, error)); ok { - return rf(ctx, tnnt, urn) - } - if rf, ok := ret.Get(0).(func(context.Context, tenant.Tenant, resource.URN) bool); ok { - r0 = rf(ctx, tnnt, urn) - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func(context.Context, tenant.Tenant, resource.URN) error); ok { - r1 = rf(ctx, tnnt, urn) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetByURN provides a mock function with given fields: ctx, tnnt, urn -func (_m *ResourceExistenceChecker) GetByURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (*resource.Resource, error) { - ret := _m.Called(ctx, tnnt, urn) - - if len(ret) == 0 { - panic("no return value specified for GetByURN") - } - - var r0 *resource.Resource - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, tenant.Tenant, resource.URN) (*resource.Resource, error)); ok { - return rf(ctx, tnnt, urn) - } - if rf, ok := ret.Get(0).(func(context.Context, tenant.Tenant, resource.URN) *resource.Resource); ok { - r0 = rf(ctx, tnnt, urn) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*resource.Resource) - } +func jobResourceURNsToString(resourceURNs []job.ResourceURN) []string { + output := make([]string, len(resourceURNs)) + for i, urn := range resourceURNs { + output[i] = urn.String() } - - if rf, ok := ret.Get(1).(func(context.Context, tenant.Tenant, resource.URN) error); ok { - r1 = rf(ctx, tnnt, urn) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// NewResourceExistenceChecker creates a new instance of ResourceExistenceChecker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewResourceExistenceChecker(t interface { - mock.TestingT - Cleanup(func()) -}, -) *ResourceExistenceChecker { - mock := &ResourceExistenceChecker{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock + return output } diff --git a/core/job/spec.go b/core/job/spec.go index baef01d2b8..d9af85d2e1 100644 --- a/core/job/spec.go +++ b/core/job/spec.go @@ -202,6 +202,20 @@ func (s Specs) ToFullNameAndSpecMap(projectName tenant.ProjectName) map[FullName return fullnameAndSpecMap } +func (s Specs) Validate() error { + me := errors.NewMultiError("validate specs errors") + jobNameCount := s.getJobNameCount() + isJobNameVisited := map[Name]bool{} + for _, spec := range s { + if jobNameCount[spec.Name()] > 1 && !isJobNameVisited[spec.Name()] { + me.Append(fmt.Errorf("duplicate %s", spec.Name())) + } + isJobNameVisited[spec.Name()] = true + } + + return me.ToErr() +} + func (s Specs) GetValid() []*Spec { jobNameCount := s.getJobNameCount() validSpecs := []*Spec{} diff --git a/core/job/spec_test.go b/core/job/spec_test.go index 5edec3f928..5b9c5c105b 100644 --- a/core/job/spec_test.go +++ b/core/job/spec_test.go @@ -174,6 +174,21 @@ func TestEntitySpec(t *testing.T) { assert.EqualValues(t, expectedMap, resultMap) }) + t.Run("Validate should return error for duplicate jobs", func(t *testing.T) { + specA1, err := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).Build() + assert.NoError(t, err) + + specA2, err := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).Build() + assert.NoError(t, err) + + specB, err := job.NewSpecBuilder(jobVersion, "job-B", "sample-owner", jobSchedule, jobWindow, jobTask).Build() + assert.NoError(t, err) + + specs := job.Specs([]*job.Spec{specA1, specA2, specB}) + err = specs.Validate() + + assert.ErrorContains(t, err, "duplicate job-A") + }) t.Run("GetValid should return valid specs", func(t *testing.T) { specA1, err := job.NewSpecBuilder(jobVersion, "job-A", "sample-owner", jobSchedule, jobWindow, jobTask).Build() assert.NoError(t, err) diff --git a/core/resource/resource.go b/core/resource/resource.go index 5cad42b914..7cbf1485b2 100644 --- a/core/resource/resource.go +++ b/core/resource/resource.go @@ -48,13 +48,7 @@ func NameFrom(name string) (Name, error) { return "", errors.InvalidArgument(EntityResource, "resource name is empty") } - cleaned := strings.ReplaceAll(name, ":", ".") // TODO: design flaw, needs to be refactored - - return Name(cleaned), nil -} - -func (n Name) Sections() []string { - return strings.Split(n.String(), nameSectionSeparator) + return Name(name), nil } func (n Name) String() string { @@ -66,7 +60,7 @@ type Resource struct { kind string store Store - urn URN + urn string tenant tenant.Tenant @@ -117,12 +111,12 @@ func (r *Resource) FullName() string { return r.name.String() } -func (r *Resource) URN() URN { +func (r *Resource) URN() string { return r.urn } -func (r *Resource) UpdateURN(urn URN) error { - if r.urn.IsZero() { +func (r *Resource) UpdateURN(urn string) error { + if r.urn == "" { r.urn = urn return nil } @@ -138,6 +132,10 @@ func (r *Resource) Metadata() *Metadata { return r.metadata } +func (r *Resource) NameSections() []string { + return strings.Split(r.name.String(), nameSectionSeparator) +} + func (r *Resource) Kind() string { return r.kind } diff --git a/core/resource/resource_test.go b/core/resource/resource_test.go index 4b1df5fb58..92f345b460 100644 --- a/core/resource/resource_test.go +++ b/core/resource/resource_test.go @@ -120,16 +120,14 @@ func TestNewResource(t *testing.T) { res, err := resource.NewResource("proj.set.res_name", "table", resource.Bigquery, tnnt, meta, spec) assert.Nil(t, err) - urn, err := resource.ParseURN("bigquery://proj:set.res_name") - assert.NoError(t, err) - err = res.UpdateURN(urn) + err = res.UpdateURN("bigquery://proj:set.res_name") assert.Nil(t, err) assert.Equal(t, "proj.set.res_name", res.FullName()) assert.Equal(t, "proj.set.res_name", res.Name().String()) - assert.Equal(t, urn, res.URN()) + assert.Equal(t, "bigquery://proj:set.res_name", res.URN()) assert.EqualValues(t, meta, res.Metadata()) - assert.Equal(t, 3, len(res.Name().Sections())) + assert.Equal(t, 3, len(res.NameSections())) assert.Equal(t, "table", res.Kind()) assert.EqualValues(t, tnnt, res.Tenant()) assert.Equal(t, resource.Bigquery.String(), res.Store().String()) @@ -147,7 +145,7 @@ func TestNewResource(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "proj.dataset", res.FullName()) - assert.Equal(t, 2, len(res.Name().Sections())) + assert.Equal(t, 2, len(res.NameSections())) assert.EqualValues(t, meta, res.Metadata()) assert.Equal(t, "dataset", res.Kind()) assert.EqualValues(t, tnnt, res.Tenant()) @@ -386,8 +384,7 @@ func TestResource(t *testing.T) { validResource, err := resource.NewResource("project.dataset.table", "table", resource.Bigquery, tnnt, metadata, spec) assert.NoError(t, err) - urn, err := resource.ParseURN("bigquery://project:dataset.table") - assert.NoError(t, err) + urn := "bigquery://project:dataset.table" err = validResource.UpdateURN(urn) assert.NoError(t, err) assert.Equal(t, urn, validResource.URN()) @@ -396,8 +393,7 @@ func TestResource(t *testing.T) { validResource, err := resource.NewResource("project.dataset.table", "table", resource.Bigquery, tnnt, metadata, spec) assert.NoError(t, err) - urn, err := resource.ParseURN("bigquery://project:dataset.table") - assert.NoError(t, err) + urn := "bigquery://project:dataset.table" err = validResource.UpdateURN(urn) assert.NoError(t, err) diff --git a/core/resource/service/resource_manager.go b/core/resource/service/resource_manager.go index 01710bb62e..d0401de5d2 100644 --- a/core/resource/service/resource_manager.go +++ b/core/resource/service/resource_manager.go @@ -7,7 +7,6 @@ import ( "github.com/goto/salt/log" "github.com/goto/optimus/core/resource" - "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/errors" ) @@ -16,9 +15,8 @@ type DataStore interface { Update(context.Context, *resource.Resource) error BatchUpdate(context.Context, []*resource.Resource) error Validate(*resource.Resource) error - GetURN(res *resource.Resource) (resource.URN, error) + GetURN(res *resource.Resource) (string, error) Backup(context.Context, *resource.Backup, []*resource.Resource) (*resource.BackupResult, error) - Exist(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) } type ResourceStatusRepo interface { @@ -119,13 +117,13 @@ func (m *ResourceMgr) Validate(res *resource.Resource) error { return datastore.Validate(res) } -func (m *ResourceMgr) GetURN(res *resource.Resource) (resource.URN, error) { +func (m *ResourceMgr) GetURN(res *resource.Resource) (string, error) { store := res.Store() datastore, ok := m.datastoreMap[store] if !ok { msg := fmt.Sprintf("datastore [%s] for resource [%s] is not found", store.String(), res.FullName()) m.logger.Error(msg) - return resource.ZeroURN(), errors.InternalError(resource.EntityResource, msg, nil) + return "", errors.InternalError(resource.EntityResource, msg, nil) } return datastore.GetURN(res) @@ -159,26 +157,6 @@ func (m *ResourceMgr) RegisterDatastore(store resource.Store, dataStore DataStor m.datastoreMap[store] = dataStore } -func (m *ResourceMgr) Exist(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) { - if urn.IsZero() { - return false, errors.InvalidArgument(resource.EntityResource, "urn is zero-valued") - } - - store, err := resource.FromStringToStore(urn.GetStore()) - if err != nil { - return false, err - } - - datastore, ok := m.datastoreMap[store] - if !ok { - msg := fmt.Sprintf("datastore [%s] is not found", store) - m.logger.Error(msg) - return false, errors.InternalError(resource.EntityResource, msg, nil) - } - - return datastore.Exist(ctx, tnnt, urn) -} - func NewResourceManager(repo ResourceStatusRepo, logger log.Logger) *ResourceMgr { return &ResourceMgr{ repo: repo, diff --git a/core/resource/service/resource_manager_test.go b/core/resource/service/resource_manager_test.go index b2488cb9d0..c3258df4ac 100644 --- a/core/resource/service/resource_manager_test.go +++ b/core/resource/service/resource_manager_test.go @@ -52,7 +52,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Create", ctx, createRequest).Return(errors.InternalError("resource", "error in create", nil)) defer storeService.AssertExpectations(t) @@ -80,7 +80,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Create", ctx, createRequest).Return(errors.InvalidArgument("res", "error in create")) defer storeService.AssertExpectations(t) @@ -108,7 +108,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Create", ctx, createRequest).Return(errors.AlreadyExists("resource", "error in create")) defer storeService.AssertExpectations(t) @@ -134,7 +134,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Create", ctx, createRequest).Return(nil) defer storeService.AssertExpectations(t) @@ -175,7 +175,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Update", ctx, updateRequest).Return(errors.InternalError("resource", "error in update", nil)) defer storeService.AssertExpectations(t) @@ -204,7 +204,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Update", ctx, updateRequest).Return(errors.InvalidArgument("res", "error in update")) defer storeService.AssertExpectations(t) @@ -232,7 +232,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Update", ctx, updateRequest).Return(nil) defer storeService.AssertExpectations(t) @@ -266,7 +266,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(nil, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Validate", updateRequest).Return(nil) defer storeService.AssertExpectations(t) @@ -300,18 +300,15 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(nil, logger) - urn, err := resource.ParseURN("snowflake://db.schema.table") - assert.NoError(t, err) - - storeService := NewDataStore(t) - storeService.On("GetURN", updateRequest).Return(urn, nil) + storeService := new(mockDataStore) + storeService.On("GetURN", updateRequest).Return("snowflake://db.schema.table", nil) defer storeService.AssertExpectations(t) manager.RegisterDatastore(store, storeService) - actualURN, err := manager.GetURN(updateRequest) + urn, err := manager.GetURN(updateRequest) assert.NoError(t, err) - assert.Equal(t, urn, actualURN) + assert.Equal(t, "snowflake://db.schema.table", urn) }) }) t.Run("BatchUpdate", func(t *testing.T) { @@ -361,7 +358,7 @@ func TestResourceManager(t *testing.T) { }) me2 := errors.NewMultiError("error in db update") me.Append(errors.InternalError("resource", "enable to update state in db", nil)) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("BatchUpdate", ctx, matcher).Return(me2) defer storeService.AssertExpectations(t) @@ -397,7 +394,7 @@ func TestResourceManager(t *testing.T) { } return false }) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("BatchUpdate", ctx, matcher).Return(nil) defer storeService.AssertExpectations(t) @@ -438,7 +435,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(nil, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Backup", ctx, backup, []*resource.Resource{res}).Return(&resource.BackupResult{ ResourceNames: []string{"proj.ds.name1"}, }, nil) @@ -481,7 +478,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Create", ctx, res).Return(errors.InternalError("resource", "error in create", nil)) defer storeService.AssertExpectations(t) @@ -508,7 +505,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Create", ctx, res).Return(errors.AlreadyExists(resource.EntityResource, "table already exists")) storeService.On("Update", ctx, res).Return(errors.InternalError("resource", "error in update", nil)) defer storeService.AssertExpectations(t) @@ -536,7 +533,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Create", ctx, res).Return(nil) defer storeService.AssertExpectations(t) @@ -563,7 +560,7 @@ func TestResourceManager(t *testing.T) { logger := log.NewLogrus() manager := service.NewResourceManager(repo, logger) - storeService := NewDataStore(t) + storeService := new(mockDataStore) storeService.On("Create", ctx, res).Return(errors.AlreadyExists(resource.EntityResource, "table already exists")) storeService.On("Update", ctx, res).Return(nil) defer storeService.AssertExpectations(t) @@ -585,180 +582,35 @@ func (m *mockRepo) UpdateStatus(ctx context.Context, res ...*resource.Resource) return args.Error(0) } -// DataStore is an autogenerated mock type for the DataStore type -type DataStore struct { +type mockDataStore struct { mock.Mock } -// Backup provides a mock function with given fields: _a0, _a1, _a2 -func (_m *DataStore) Backup(_a0 context.Context, _a1 *resource.Backup, _a2 []*resource.Resource) (*resource.BackupResult, error) { - ret := _m.Called(_a0, _a1, _a2) - - if len(ret) == 0 { - panic("no return value specified for Backup") - } - - var r0 *resource.BackupResult - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *resource.Backup, []*resource.Resource) (*resource.BackupResult, error)); ok { - return rf(_a0, _a1, _a2) - } - if rf, ok := ret.Get(0).(func(context.Context, *resource.Backup, []*resource.Resource) *resource.BackupResult); ok { - r0 = rf(_a0, _a1, _a2) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*resource.BackupResult) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *resource.Backup, []*resource.Resource) error); ok { - r1 = rf(_a0, _a1, _a2) - } else { - r1 = ret.Error(1) - } - - return r0, r1 +func (m *mockDataStore) Create(ctx context.Context, r *resource.Resource) error { + return m.Called(ctx, r).Error(0) } -// BatchUpdate provides a mock function with given fields: _a0, _a1 -func (_m *DataStore) BatchUpdate(_a0 context.Context, _a1 []*resource.Resource) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for BatchUpdate") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, []*resource.Resource) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 +func (m *mockDataStore) Update(ctx context.Context, r *resource.Resource) error { + return m.Called(ctx, r).Error(0) } -// Create provides a mock function with given fields: _a0, _a1 -func (_m *DataStore) Create(_a0 context.Context, _a1 *resource.Resource) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for Create") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *resource.Resource) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 +func (m *mockDataStore) BatchUpdate(ctx context.Context, resources []*resource.Resource) error { + return m.Called(ctx, resources).Error(0) } -// Exist provides a mock function with given fields: ctx, tnnt, urn -func (_m *DataStore) Exist(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) { - ret := _m.Called(ctx, tnnt, urn) - - if len(ret) == 0 { - panic("no return value specified for Exist") - } - - var r0 bool - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, tenant.Tenant, resource.URN) (bool, error)); ok { - return rf(ctx, tnnt, urn) - } - if rf, ok := ret.Get(0).(func(context.Context, tenant.Tenant, resource.URN) bool); ok { - r0 = rf(ctx, tnnt, urn) - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func(context.Context, tenant.Tenant, resource.URN) error); ok { - r1 = rf(ctx, tnnt, urn) - } else { - r1 = ret.Error(1) - } - - return r0, r1 +func (m *mockDataStore) Validate(r *resource.Resource) error { + return m.Called(r).Error(0) } -// GetURN provides a mock function with given fields: res -func (_m *DataStore) GetURN(res *resource.Resource) (resource.URN, error) { - ret := _m.Called(res) - - if len(ret) == 0 { - panic("no return value specified for GetURN") - } - - var r0 resource.URN - var r1 error - if rf, ok := ret.Get(0).(func(*resource.Resource) (resource.URN, error)); ok { - return rf(res) - } - if rf, ok := ret.Get(0).(func(*resource.Resource) resource.URN); ok { - r0 = rf(res) - } else { - r0 = ret.Get(0).(resource.URN) - } - - if rf, ok := ret.Get(1).(func(*resource.Resource) error); ok { - r1 = rf(res) - } else { - r1 = ret.Error(1) - } - - return r0, r1 +func (m *mockDataStore) GetURN(r *resource.Resource) (string, error) { + args := m.Called(r) + return args.Get(0).(string), args.Error(1) } -// Update provides a mock function with given fields: _a0, _a1 -func (_m *DataStore) Update(_a0 context.Context, _a1 *resource.Resource) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for Update") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *resource.Resource) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) +func (m *mockDataStore) Backup(ctx context.Context, backup *resource.Backup, resources []*resource.Resource) (*resource.BackupResult, error) { + args := m.Called(ctx, backup, resources) + if args.Get(0) == nil { + return nil, args.Error(1) } - - return r0 -} - -// Validate provides a mock function with given fields: _a0 -func (_m *DataStore) Validate(_a0 *resource.Resource) error { - ret := _m.Called(_a0) - - if len(ret) == 0 { - panic("no return value specified for Validate") - } - - var r0 error - if rf, ok := ret.Get(0).(func(*resource.Resource) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// NewDataStore creates a new instance of DataStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewDataStore(t interface { - mock.TestingT - Cleanup(func()) -}, -) *DataStore { - mock := &DataStore{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock + return args.Get(0).(*resource.BackupResult), args.Error(1) } diff --git a/core/resource/service/resource_service.go b/core/resource/service/resource_service.go index bcde2f6df1..43de8fbba7 100644 --- a/core/resource/service/resource_service.go +++ b/core/resource/service/resource_service.go @@ -33,16 +33,15 @@ type ResourceManager interface { SyncResource(ctx context.Context, res *resource.Resource) error BatchUpdate(ctx context.Context, store resource.Store, resources []*resource.Resource) error Validate(res *resource.Resource) error - GetURN(res *resource.Resource) (resource.URN, error) - Exist(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) + GetURN(res *resource.Resource) (string, error) } type DownstreamRefresher interface { - RefreshResourceDownstream(ctx context.Context, resourceURNs []resource.URN, logWriter writer.LogWriter) error + RefreshResourceDownstream(ctx context.Context, resourceURNs []job.ResourceURN, logWriter writer.LogWriter) error } type DownstreamResolver interface { - GetDownstreamByResourceURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (job.DownstreamList, error) + GetDownstreamByResourceURN(ctx context.Context, tnnt tenant.Tenant, urn job.ResourceURN) (job.DownstreamList, error) } type EventHandler interface { @@ -190,7 +189,7 @@ func (rs ResourceService) Delete(ctx context.Context, req *resource.DeleteReques return nil, err } - downstreamList, err := rs.downstreamResolver.GetDownstreamByResourceURN(ctx, existing.Tenant(), existing.URN()) + downstreamList, err := rs.downstreamResolver.GetDownstreamByResourceURN(ctx, existing.Tenant(), job.ResourceURN(existing.URN())) if err != nil { return nil, err } @@ -276,33 +275,6 @@ func (rs ResourceService) SyncResources(ctx context.Context, tnnt tenant.Tenant, return synced, nil } -func (rs ResourceService) GetByURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (*resource.Resource, error) { - if urn.IsZero() { - rs.logger.Error("urn is zero value") - return nil, errors.InvalidArgument(resource.EntityResource, "urn is zero value") - } - - store, err := resource.FromStringToStore(urn.GetStore()) - if err != nil { - return nil, err - } - - name, err := resource.NameFrom(urn.GetName()) - if err != nil { - return nil, err - } - - return rs.repo.ReadByFullName(ctx, tnnt, store, name.String(), false) -} - -func (rs ResourceService) ExistInStore(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) { - if urn.IsZero() { - return false, errors.NewError(errors.ErrInvalidArgument, resource.EntityResource, "urn is zero-valued") - } - - return rs.mgr.Exist(ctx, tnnt, urn) -} - func (rs ResourceService) Deploy(ctx context.Context, tnnt tenant.Tenant, store resource.Store, incomings []*resource.Resource, logWriter writer.LogWriter) error { // nolint:gocritic multiError := errors.NewMultiError("error batch updating resources") for _, r := range incomings { @@ -485,7 +457,7 @@ func (rs ResourceService) handleRefreshDownstream( // nolint:gocritic existingMappedByFullName map[string]*resource.Resource, logWriter writer.LogWriter, ) error { - var resourceURNsToRefresh []resource.URN + var resourceURNsToRefresh []job.ResourceURN for _, incoming := range incomings { if incoming.Status() != resource.StatusSuccess { continue @@ -501,7 +473,7 @@ func (rs ResourceService) handleRefreshDownstream( // nolint:gocritic } if rs.isToRefreshDownstream(incoming, existing) { - resourceURNsToRefresh = append(resourceURNsToRefresh, incoming.URN()) + resourceURNsToRefresh = append(resourceURNsToRefresh, job.ResourceURN(incoming.URN())) } else { rs.logger.Warn(skipMessage) logWriter.Write(writer.LogLevelWarning, skipMessage) diff --git a/core/resource/service/resource_service_test.go b/core/resource/service/resource_service_test.go index a7a9e814b4..9b13b14654 100644 --- a/core/resource/service/resource_service_test.go +++ b/core/resource/service/resource_service_test.go @@ -33,16 +33,11 @@ func TestResourceService(t *testing.T) { "description": "test spec", } - datasetURN, err := resource.ParseURN("bigquery://project:dataset") - assert.NoError(t, err) - tableURN, err := resource.ParseURN("bigquery://project:dataset.table") - assert.NoError(t, err) - t.Run("Create", func(t *testing.T) { t.Run("returns error if resource is invalid", func(t *testing.T) { invalid := &resource.Resource{} - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", invalid).Return(errors.New("validation error")) rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil) @@ -55,9 +50,9 @@ func TestResourceService(t *testing.T) { incoming, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) - mgr.On("GetURN", incoming).Return(resource.ZeroURN(), errors.New("urn error")) + mgr.On("GetURN", incoming).Return("", errors.New("urn error")) rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil) @@ -68,12 +63,13 @@ func TestResourceService(t *testing.T) { t.Run("returns error if cannot update resource urn", func(t *testing.T) { incoming, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - err = incoming.UpdateURN(tableURN) + urn := "bigquery://project:dataset.table" + err = incoming.UpdateURN(urn) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) - mgr.On("GetURN", incoming).Return(tableURN, nil) + mgr.On("GetURN", incoming).Return(urn, nil) rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil) @@ -88,9 +84,9 @@ func TestResourceService(t *testing.T) { incoming, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) - mgr.On("GetURN", incoming).Return(tableURN, nil) + mgr.On("GetURN", incoming).Return("bigquery://project:dataset", nil) repo := newResourceRepository(t) repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(nil, errors.New("unknown error")) @@ -110,9 +106,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(nil, oErrors.NotFound(resource.EntityResource, "resource not found")) repo.On("Create", ctx, mock.Anything).Return(errors.New("error creating resource")) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) - mgr.On("GetURN", incoming).Return(tableURN, nil) + mgr.On("GetURN", incoming).Return("bigquery://project:dataset", nil) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -126,9 +122,9 @@ func TestResourceService(t *testing.T) { existing, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset", nil) statusToTest := []resource.Status{ resource.StatusExistInStore, @@ -155,9 +151,9 @@ func TestResourceService(t *testing.T) { existing, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset", nil) unacceptableStatuses := []resource.Status{ resource.StatusUnknown, @@ -196,9 +192,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(existing, nil) repo.On("Update", ctx, incoming).Return(errors.New("error updating resource")) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) - mgr.On("GetURN", incoming).Return(tableURN, nil) + mgr.On("GetURN", incoming).Return("bigquery://project:dataset", nil) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -215,9 +211,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(nil, oErrors.NotFound(resource.EntityResource, "resource not found")) repo.On("Create", ctx, incoming).Return(nil) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) - mgr.On("GetURN", incoming).Return(tableURN, nil) + mgr.On("GetURN", incoming).Return("bigquery://project:dataset", nil) mgr.On("CreateResource", ctx, incoming).Return(errors.New("error creating to store")) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -234,9 +230,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(nil, oErrors.NotFound(resource.EntityResource, "resource not found")) repo.On("Create", ctx, incoming).Return(nil) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) - mgr.On("GetURN", incoming).Return(tableURN, nil) + mgr.On("GetURN", incoming).Return("bigquery://project:dataset", nil) mgr.On("CreateResource", ctx, incoming).Return(nil) eventHandler := newEventHandler(t) @@ -253,7 +249,7 @@ func TestResourceService(t *testing.T) { t.Run("returns error if resource is invalid", func(t *testing.T) { invalidResource := &resource.Resource{} - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", invalidResource).Return(errors.New("validation error")) rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil) @@ -265,9 +261,9 @@ func TestResourceService(t *testing.T) { incoming, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) - mgr.On("GetURN", incoming).Return(resource.ZeroURN(), errors.New("urn error")) + mgr.On("GetURN", incoming).Return("", errors.New("urn error")) rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil) @@ -278,12 +274,11 @@ func TestResourceService(t *testing.T) { t.Run("returns error if cannot update resource urn", func(t *testing.T) { incoming, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - urn, err := resource.ParseURN("bigquery://project:dataset.table") - assert.NoError(t, err) + urn := "bigquery://project:dataset.table" err = incoming.UpdateURN(urn) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(urn, nil) @@ -302,9 +297,9 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, fullName, onlyActive).Return(nil, errors.New("unknown error")) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", resourceToUpdate).Return(nil) - mgr.On("GetURN", resourceToUpdate).Return(datasetURN, nil) + mgr.On("GetURN", resourceToUpdate).Return("bigquery://project:dataset", nil) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -316,9 +311,9 @@ func TestResourceService(t *testing.T) { existing, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset", nil) repo := newResourceRepository(t) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -353,9 +348,9 @@ func TestResourceService(t *testing.T) { assert.NoError(t, err) existingResource = resource.FromExisting(existingResource, resource.ReplaceStatus(resource.StatusToUpdate)) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", resourceToUpdate).Return(nil) - mgr.On("GetURN", resourceToUpdate).Return(datasetURN, nil) + mgr.On("GetURN", resourceToUpdate).Return("bigquery://project:dataset", nil) repo := newResourceRepository(t) repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, fullName, onlyActive).Return(existingResource, nil) @@ -379,9 +374,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, fullName, onlyActive).Return(existingResource, nil) repo.On("Update", ctx, mock.Anything).Return(nil) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset", nil) mgr.On("UpdateResource", ctx, mock.Anything).Return(errors.New("unknown error")) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -404,9 +399,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, fullName, onlyActive).Return(existingResource, nil) repo.On("Update", ctx, mock.Anything).Return(nil) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset", nil) mgr.On("UpdateResource", ctx, mock.Anything).Run(func(args mock.Arguments) { res, ok := args[1].(*resource.Resource) if ok { @@ -440,9 +435,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, fullName, onlyActive).Return(existingResource, nil) repo.On("Update", ctx, mock.Anything).Return(nil) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset", nil) mgr.On("UpdateResource", ctx, mock.Anything).Return(nil) eventHandler := newEventHandler(t) @@ -550,7 +545,7 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{}, nil) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", invalidResourceToUpdate).Return(errors.New("error validating")) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -564,9 +559,9 @@ func TestResourceService(t *testing.T) { incoming, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) - mgr.On("GetURN", incoming).Return(resource.ZeroURN(), errors.New("urn error")) + mgr.On("GetURN", incoming).Return("", errors.New("urn error")) repo := newResourceRepository(t) repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{}, nil) @@ -582,12 +577,11 @@ func TestResourceService(t *testing.T) { t.Run("returns error if cannot update resource urn", func(t *testing.T) { incoming, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) - urn, err := resource.ParseURN("bigquery://project:dataset.table") - assert.NoError(t, err) + urn := "bigquery://project:dataset.table" err = incoming.UpdateURN(urn) assert.NoError(t, err) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(urn, nil) @@ -609,12 +603,9 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return(nil, errors.New("error while read all")) - urn, err := resource.ParseURN("bigquery://project:dataset.table1") - assert.NoError(t, err) - - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", incomingResourceToUpdate).Return(nil) - mgr.On("GetURN", incomingResourceToUpdate).Return(urn, nil) + mgr.On("GetURN", incomingResourceToUpdate).Return("bigquery://project:dataset.table1", nil) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -630,12 +621,9 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{existing}, nil) - urn, err := resource.ParseURN("bigquery://project:dataset.view1") - assert.NoError(t, err) - - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(urn, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset.view1", nil) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -655,9 +643,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{}, nil) repo.On("Create", ctx, incomingResourceToUpdate).Return(errors.New("error in create")) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset", nil) eventHandler := newEventHandler(t) @@ -682,12 +670,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{existing}, nil) repo.On("Update", ctx, incomingResourceToUpdate).Return(errors.New("error in update")) - urn, err := resource.ParseURN("bigquery://project:dataset.view1") - assert.NoError(t, err) - - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(urn, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset.view1", nil) eventHandler := newEventHandler(t) rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil) @@ -711,12 +696,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{existing}, nil) repo.On("Update", ctx, incomingResourceToUpdate).Return(nil) - urn, err := resource.ParseURN("bigquery://project:dataset.view1") - assert.NoError(t, err) - - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(urn, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset.view1", nil) mgr.On("BatchUpdate", ctx, resource.Bigquery, mock.Anything).Return(errors.New("unknown error")) eventHandler := newEventHandler(t) @@ -743,12 +725,9 @@ func TestResourceService(t *testing.T) { repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{existingResource}, nil) repo.On("Update", ctx, incomingResourceToUpdate).Return(nil) - urn, err := resource.ParseURN("bigquery://project:dataset.view1") - assert.NoError(t, err) - - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(urn, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset.view1", nil) mgr.On("BatchUpdate", ctx, resource.Bigquery, mock.Anything).Return(errors.New("unknown error")) eventHandler := newEventHandler(t) @@ -781,12 +760,9 @@ func TestResourceService(t *testing.T) { repo.On("Update", ctx, incomingToUpdate).Return(nil) repo.On("Update", ctx, incomingToCreateExisting).Return(nil) - urn, err := resource.ParseURN("bigquery://project:dataset.view1") - assert.NoError(t, err) - - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(urn, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset.view1", nil) mgr.On("BatchUpdate", ctx, resource.Bigquery, []*resource.Resource{incomingToCreate, incomingToUpdate, incomingToCreateExisting}).Run(func(args mock.Arguments) { res := args.Get(2).([]*resource.Resource) for _, r := range res { @@ -833,12 +809,9 @@ func TestResourceService(t *testing.T) { repo.On("Update", ctx, incomingToUpdate).Return(nil) repo.On("Update", ctx, incomingToCreateExisting).Return(nil) - urn, err := resource.ParseURN("bigquery://project:dataset.view1") - assert.NoError(t, err) - - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(urn, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset.view1", nil) mgr.On("BatchUpdate", ctx, resource.Bigquery, []*resource.Resource{incomingToCreate, incomingToUpdate, incomingToCreateExisting}).Run(func(args mock.Arguments) { res := args.Get(2).([]*resource.Resource) for _, r := range res { @@ -888,12 +861,9 @@ func TestResourceService(t *testing.T) { repo.On("Update", ctx, incomingToCreateExisting).Return(nil) repo.On("Update", ctx, incomingToDelete).Return(nil) - urn, err := resource.ParseURN("bigquery://project:dataset.view1") - assert.NoError(t, err) - - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(urn, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset.view1", nil) mgr.On("BatchUpdate", ctx, resource.Bigquery, []*resource.Resource{incomingToCreate, incomingToUpdate, incomingToCreateExisting, incomingToDelete}). Run(func(args mock.Arguments) { res := args.Get(2).([]*resource.Resource) @@ -950,12 +920,9 @@ func TestResourceService(t *testing.T) { repo.On("Update", ctx, incomingToCreateExisting).Return(nil) repo.On("Update", ctx, incomingToRecreate).Return(nil) - urn, err := resource.ParseURN("bigquery://project:dataset.view1") - assert.NoError(t, err) - - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("Validate", mock.Anything).Return(nil) - mgr.On("GetURN", mock.Anything).Return(urn, nil) + mgr.On("GetURN", mock.Anything).Return("bigquery://project:dataset.view1", nil) mgr.On("BatchUpdate", ctx, resource.Bigquery, []*resource.Resource{incomingToCreate, incomingToUpdate, incomingToCreateExisting, incomingToRecreate}). Run(func(args mock.Arguments) { res := args.Get(2).([]*resource.Resource) @@ -992,7 +959,7 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("GetResources", ctx, tnnt, resource.Bigquery, []string{fullName}).Return(nil, errors.New("unknown error")) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -1005,7 +972,7 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("GetResources", ctx, tnnt, resource.Bigquery, []string{fullName}).Return([]*resource.Resource{}, nil) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -1024,7 +991,7 @@ func TestResourceService(t *testing.T) { repo.On("GetResources", ctx, tnnt, resource.Bigquery, []string{fullName}). Return([]*resource.Resource{incoming}, nil) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("SyncResource", ctx, incoming).Return(errors.New("unable to create")) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -1044,7 +1011,7 @@ func TestResourceService(t *testing.T) { repo.On("GetResources", ctx, tnnt, resource.Bigquery, []string{fullName}). Return([]*resource.Resource{incoming}, nil) - mgr := NewResourceManager(t) + mgr := newResourceManager(t) mgr.On("SyncResource", ctx, incoming).Return(nil) rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil) @@ -1076,7 +1043,7 @@ func TestResourceService(t *testing.T) { defer mockDepResolver.AssertExpectations(t) repo.On("ReadByFullName", ctx, req.Tenant, req.Datastore, req.FullName, onlyActive).Return(existingResource, nil) - mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), existingResource.URN()). + mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), job.ResourceURN(existingResource.URN())). Return(nil, nil) repo.On("Delete", ctx, existingResource).Return(nil) @@ -1102,7 +1069,7 @@ func TestResourceService(t *testing.T) { defer mockDepResolver.AssertExpectations(t) repo.On("ReadByFullName", ctx, req.Tenant, req.Datastore, req.FullName, onlyActive).Return(existingResource, nil) - mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), existingResource.URN()). + mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), job.ResourceURN(existingResource.URN())). Return(downstreamList, nil) repo.On("Delete", ctx, existingResource).Return(nil) @@ -1134,7 +1101,7 @@ func TestResourceService(t *testing.T) { defer mockDepResolver.AssertExpectations(t) repo.On("ReadByFullName", ctx, req.Tenant, req.Datastore, req.FullName, onlyActive).Return(existingResource, nil) - mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), existingResource.URN()). + mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), job.ResourceURN(existingResource.URN())). Return(nil, nil) repo.On("Delete", ctx, existingResource).Return(context.DeadlineExceeded) @@ -1159,7 +1126,7 @@ func TestResourceService(t *testing.T) { defer mockDepResolver.AssertExpectations(t) repo.On("ReadByFullName", ctx, req.Tenant, req.Datastore, req.FullName, onlyActive).Return(existingResource, nil) - mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), existingResource.URN()). + mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), job.ResourceURN(existingResource.URN())). Return(downstreamList, nil) actual, err := rscService.Delete(ctx, req) @@ -1184,7 +1151,7 @@ func TestResourceService(t *testing.T) { defer mockDepResolver.AssertExpectations(t) repo.On("ReadByFullName", ctx, req.Tenant, req.Datastore, req.FullName, onlyActive).Return(existingResource, nil) - mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), existingResource.URN()). + mockDepResolver.On("GetDownstreamByResourceURN", ctx, existingResource.Tenant(), job.ResourceURN(existingResource.URN())). Return(downstreamList, context.DeadlineExceeded) actual, err := rscService.Delete(ctx, req) @@ -1273,207 +1240,84 @@ func newResourceRepository(t mockConstructorTestingTNewResourceRepository) *mock return mock } -type mockEventHandler struct { +type mockResourceManager struct { mock.Mock } -func (m *mockEventHandler) HandleEvent(e moderator.Event) { - m.Called(e) +func (m *mockResourceManager) BatchUpdate(ctx context.Context, store resource.Store, resources []*resource.Resource) error { + return m.Called(ctx, store, resources).Error(0) } -type mockConstructorEventHandler interface { - mock.TestingT - Cleanup(func()) +func (m *mockResourceManager) CreateResource(ctx context.Context, res *resource.Resource) error { + return m.Called(ctx, res).Error(0) } -func newEventHandler(t mockConstructorEventHandler) *mockEventHandler { - mock := &mockEventHandler{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock +func (m *mockResourceManager) UpdateResource(ctx context.Context, res *resource.Resource) error { + return m.Called(ctx, res).Error(0) } -type mockDownstreamRefresher struct { - mock.Mock +func (m *mockResourceManager) Validate(res *resource.Resource) error { + return m.Called(res).Error(0) } -func (m *mockDownstreamRefresher) RefreshResourceDownstream(ctx context.Context, resourceURNs []resource.URN, logWriter writer.LogWriter) error { - return m.Called(ctx, resourceURNs, logWriter).Error(0) +func (m *mockResourceManager) GetURN(res *resource.Resource) (string, error) { + args := m.Called(res) + return args.Get(0).(string), args.Error(1) } -// ResourceManager is an autogenerated mock type for the ResourceManager type -type ResourceManager struct { - mock.Mock +func (m *mockResourceManager) SyncResource(ctx context.Context, res *resource.Resource) error { + return m.Called(ctx, res).Error(0) } -// BatchUpdate provides a mock function with given fields: ctx, store, resources -func (_m *ResourceManager) BatchUpdate(ctx context.Context, store resource.Store, resources []*resource.Resource) error { - ret := _m.Called(ctx, store, resources) - - if len(ret) == 0 { - panic("no return value specified for BatchUpdate") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, resource.Store, []*resource.Resource) error); ok { - r0 = rf(ctx, store, resources) - } else { - r0 = ret.Error(0) - } - - return r0 +type mockConstructorTestingTNewResourceManager interface { + mock.TestingT + Cleanup(func()) } -// CreateResource provides a mock function with given fields: ctx, res -func (_m *ResourceManager) CreateResource(ctx context.Context, res *resource.Resource) error { - ret := _m.Called(ctx, res) - - if len(ret) == 0 { - panic("no return value specified for CreateResource") - } +func newResourceManager(t mockConstructorTestingTNewResourceManager) *mockResourceManager { + mock := &mockResourceManager{} + mock.Mock.Test(t) - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *resource.Resource) error); ok { - r0 = rf(ctx, res) - } else { - r0 = ret.Error(0) - } + t.Cleanup(func() { mock.AssertExpectations(t) }) - return r0 + return mock } -// Exist provides a mock function with given fields: ctx, tnnt, urn -func (_m *ResourceManager) Exist(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) { - ret := _m.Called(ctx, tnnt, urn) - - if len(ret) == 0 { - panic("no return value specified for Exist") - } - - var r0 bool - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, tenant.Tenant, resource.URN) (bool, error)); ok { - return rf(ctx, tnnt, urn) - } - if rf, ok := ret.Get(0).(func(context.Context, tenant.Tenant, resource.URN) bool); ok { - r0 = rf(ctx, tnnt, urn) - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func(context.Context, tenant.Tenant, resource.URN) error); ok { - r1 = rf(ctx, tnnt, urn) - } else { - r1 = ret.Error(1) - } - - return r0, r1 +type mockEventHandler struct { + mock.Mock } -// GetURN provides a mock function with given fields: res -func (_m *ResourceManager) GetURN(res *resource.Resource) (resource.URN, error) { - ret := _m.Called(res) - - if len(ret) == 0 { - panic("no return value specified for GetURN") - } - - var r0 resource.URN - var r1 error - if rf, ok := ret.Get(0).(func(*resource.Resource) (resource.URN, error)); ok { - return rf(res) - } - if rf, ok := ret.Get(0).(func(*resource.Resource) resource.URN); ok { - r0 = rf(res) - } else { - r0 = ret.Get(0).(resource.URN) - } - - if rf, ok := ret.Get(1).(func(*resource.Resource) error); ok { - r1 = rf(res) - } else { - r1 = ret.Error(1) - } - - return r0, r1 +func (m *mockEventHandler) HandleEvent(e moderator.Event) { + m.Called(e) } -// SyncResource provides a mock function with given fields: ctx, res -func (_m *ResourceManager) SyncResource(ctx context.Context, res *resource.Resource) error { - ret := _m.Called(ctx, res) - - if len(ret) == 0 { - panic("no return value specified for SyncResource") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *resource.Resource) error); ok { - r0 = rf(ctx, res) - } else { - r0 = ret.Error(0) - } - - return r0 +type mockConstructorEventHandler interface { + mock.TestingT + Cleanup(func()) } -// UpdateResource provides a mock function with given fields: ctx, res -func (_m *ResourceManager) UpdateResource(ctx context.Context, res *resource.Resource) error { - ret := _m.Called(ctx, res) - - if len(ret) == 0 { - panic("no return value specified for UpdateResource") - } +func newEventHandler(t mockConstructorEventHandler) *mockEventHandler { + mock := &mockEventHandler{} + mock.Mock.Test(t) - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *resource.Resource) error); ok { - r0 = rf(ctx, res) - } else { - r0 = ret.Error(0) - } + t.Cleanup(func() { mock.AssertExpectations(t) }) - return r0 + return mock } -// Validate provides a mock function with given fields: res -func (_m *ResourceManager) Validate(res *resource.Resource) error { - ret := _m.Called(res) - - if len(ret) == 0 { - panic("no return value specified for Validate") - } - - var r0 error - if rf, ok := ret.Get(0).(func(*resource.Resource) error); ok { - r0 = rf(res) - } else { - r0 = ret.Error(0) - } - - return r0 +type mockDownstreamRefresher struct { + mock.Mock } -// NewResourceManager creates a new instance of ResourceManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewResourceManager(t interface { - mock.TestingT - Cleanup(func()) -}, -) *ResourceManager { - mock := &ResourceManager{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock +func (m *mockDownstreamRefresher) RefreshResourceDownstream(ctx context.Context, resourceURNs []job.ResourceURN, logWriter writer.LogWriter) error { + return m.Called(ctx, resourceURNs, logWriter).Error(0) } type mockDownstreamResolver struct { mock.Mock } -func (m *mockDownstreamResolver) GetDownstreamByResourceURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (job.DownstreamList, error) { +func (m *mockDownstreamResolver) GetDownstreamByResourceURN(ctx context.Context, tnnt tenant.Tenant, urn job.ResourceURN) (job.DownstreamList, error) { args := m.Called(ctx, tnnt, urn) if args.Get(0) != nil { return args.Get(0).(job.DownstreamList), args.Error(1) diff --git a/core/resource/urn.go b/core/resource/urn.go deleted file mode 100644 index 1cf14fab24..0000000000 --- a/core/resource/urn.go +++ /dev/null @@ -1,87 +0,0 @@ -package resource - -import ( - "errors" - "fmt" - "strings" -) - -const ( - urnSeparator = "://" - urnComponentLength = 2 -) - -var ( - errURNWrongPattern = fmt.Errorf("urn does not follow pattern %s", urnSeparator) - errURNStoreNotSpecified = errors.New("urn store is not specified") - errURNNameNotSpecified = errors.New("urn name is not specified") - errURNStoreContainsWhitespace = errors.New("urn store contains whitespace") - errURNNameContainsWhitespace = errors.New("urn name contains whitespace") -) - -var zeroURN URN - -type URN struct { - store string - name string - - raw string -} - -func NewURN(store, name string) (URN, error) { - rawURN := store + urnSeparator + name - - return ParseURN(rawURN) -} - -func (u URN) GetStore() string { - return u.store -} - -func (u URN) GetName() string { - return u.name -} - -func (u URN) String() string { - return u.raw -} - -func (u URN) IsZero() bool { - return u == zeroURN -} - -func ParseURN(urn string) (URN, error) { - splitURN := strings.Split(urn, urnSeparator) - if len(splitURN) != urnComponentLength { - return URN{}, errURNWrongPattern - } - - store := splitURN[0] - name := splitURN[1] - - if store == "" { - return URN{}, errURNStoreNotSpecified - } - - if name == "" { - return URN{}, errURNNameNotSpecified - } - - if trimmedStore := strings.TrimSpace(store); len(trimmedStore) != len(store) { - return URN{}, errURNStoreContainsWhitespace - } - - if trimmedName := strings.TrimSpace(name); len(trimmedName) != len(name) { - return URN{}, errURNNameContainsWhitespace - } - - return URN{ - store: store, - name: name, - raw: urn, - }, nil -} - -func ZeroURN() URN { - return zeroURN -} diff --git a/core/resource/urn_test.go b/core/resource/urn_test.go deleted file mode 100644 index 0fbb062617..0000000000 --- a/core/resource/urn_test.go +++ /dev/null @@ -1,161 +0,0 @@ -package resource_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/goto/optimus/core/resource" -) - -func TestNewURN(t *testing.T) { - t.Run("should return zero urn and error if parsing fails", func(t *testing.T) { - testTable := []struct { - conditionName string - store string - name string - errorMessage string - }{ - { - conditionName: "empty store", - store: "", - name: "project.dataset.name", - errorMessage: "urn store is not specified", - }, - { - conditionName: "empty name", - store: "store", - name: "", - errorMessage: "urn name is not specified", - }, - { - conditionName: "store contains whitespace", - store: "store ", - name: "project.dataset.name", - errorMessage: "urn store contains whitespace", - }, - { - conditionName: "name contains whitespace", - store: "store", - name: "project.dataset.name ", - errorMessage: "urn name contains whitespace", - }, - } - - for _, testCase := range testTable { - actualURN, actualError := resource.NewURN(testCase.store, testCase.name) - - assert.Zero(t, actualURN, testCase.conditionName) - assert.EqualError(t, actualError, testCase.errorMessage, testCase.conditionName) - } - }) - - t.Run("should return urn and nil if parsing succeeds", func(t *testing.T) { - store := "store" - name := "project.dataset.name" - - actualURN, actualError := resource.NewURN(store, name) - - assert.NotZero(t, actualURN) - assert.NoError(t, actualError) - }) -} - -func TestParseURN(t *testing.T) { - t.Run("should return zero urn and error if parsing fails", func(t *testing.T) { - testTable := []struct { - conditionName string - inputURN string - errorMessage string - }{ - { - conditionName: "empty urn", - inputURN: "", - errorMessage: "urn does not follow pattern ://", - }, - { - conditionName: "store is not specified", - inputURN: "://project.dataset.name", - errorMessage: "urn store is not specified", - }, - { - conditionName: "name is not specified", - inputURN: "store://", - errorMessage: "urn name is not specified", - }, - { - conditionName: "store contains whitespace", - inputURN: "store ://project.dataset.name", - errorMessage: "urn store contains whitespace", - }, - { - conditionName: "name contains whitespace", - inputURN: "store://project.dataset.name ", - errorMessage: "urn name contains whitespace", - }, - } - - for _, testCase := range testTable { - actualURN, actualError := resource.ParseURN(testCase.inputURN) - - assert.Zero(t, actualURN, testCase.conditionName) - assert.EqualError(t, actualError, testCase.errorMessage, testCase.conditionName) - } - }) - - t.Run("should return urn and nil if parsing succeeds", func(t *testing.T) { - urn := "store://project.dataset.name" - - actualURN, actualError := resource.ParseURN(urn) - - assert.NotZero(t, actualURN) - assert.NoError(t, actualError) - }) -} - -func TestZeroURN(t *testing.T) { - t.Run("should return zero urn", func(t *testing.T) { - assert.Zero(t, resource.ZeroURN()) - }) -} - -func TestURN(t *testing.T) { - t.Run("IsZero", func(t *testing.T) { - t.Run("should return true if urn is zero valued", func(t *testing.T) { - var urn resource.URN - - assert.True(t, urn.IsZero()) - }) - - t.Run("should return false if urn is not zero valued", func(t *testing.T) { - rawURN := "store://project.dataset.name" - - urn, err := resource.ParseURN(rawURN) - assert.NoError(t, err) - - assert.False(t, urn.IsZero()) - }) - }) - - t.Run("should return empty member if urn is not initialized", func(t *testing.T) { - var urn resource.URN - - assert.Empty(t, urn.GetStore()) - assert.Empty(t, urn.GetName()) - assert.Empty(t, urn.String()) - }) - - t.Run("should return proper member if the urn is parsed", func(t *testing.T) { - rawURN := "store://project.dataset.name" - - expectedStore := "store" - expectedName := "project.dataset.name" - - urn, err := resource.ParseURN(rawURN) - assert.NoError(t, err) - - assert.EqualValues(t, expectedStore, urn.GetStore()) - assert.EqualValues(t, expectedName, urn.GetName()) - assert.EqualValues(t, rawURN, urn.String()) - }) -} diff --git a/core/scheduler/job.go b/core/scheduler/job.go index 0ca4ec4176..cfb9dfd837 100644 --- a/core/scheduler/job.go +++ b/core/scheduler/job.go @@ -7,7 +7,6 @@ import ( "github.com/google/uuid" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/errors" "github.com/goto/optimus/internal/lib/window" @@ -52,7 +51,7 @@ type Job struct { Name JobName Tenant tenant.Tenant - Destination resource.URN + Destination string Task *Task Hooks []*Hook @@ -233,7 +232,7 @@ type JobUpstream struct { JobName string Host string TaskName string // TODO: remove after airflow migration - DestinationURN resource.URN //- bigquery://pilot.playground.table + DestinationURN string //- bigquery://pilot.playground.table Tenant tenant.Tenant // Current or external tenant Type string External bool diff --git a/core/scheduler/job_run.go b/core/scheduler/job_run.go index 415884a517..8a9b786830 100644 --- a/core/scheduler/job_run.go +++ b/core/scheduler/job_run.go @@ -5,7 +5,6 @@ import ( "github.com/google/uuid" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/errors" ) @@ -50,7 +49,7 @@ type JobRun struct { type JobRunMeta struct { Labels map[string]string - DestinationURN resource.URN + DestinationURN string } func (j *JobRun) HasSLABreached() bool { diff --git a/core/scheduler/service/deployment_service_test.go b/core/scheduler/service/deployment_service_test.go index ab6a4c5f6e..92cbad1cbe 100644 --- a/core/scheduler/service/deployment_service_test.go +++ b/core/scheduler/service/deployment_service_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/scheduler" "github.com/goto/optimus/core/scheduler/service" "github.com/goto/optimus/core/tenant" @@ -25,14 +24,11 @@ func TestDeploymentService(t *testing.T) { tnnt1, _ := tenant.NewTenant(proj1Name.String(), namespace1Name.String()) tnnt2, _ := tenant.NewTenant(proj1Name.String(), namespace2Name.String()) - urn, err := resource.ParseURN("bigquery://some-resource") - assert.NoError(t, err) - jobUpstreamStatic := scheduler.JobUpstream{ JobName: "job3", Host: "some-host", TaskName: "bq2bq", - DestinationURN: urn, + DestinationURN: "bigquery://some-resource", Tenant: tenant.Tenant{}, Type: scheduler.UpstreamTypeStatic, External: true, @@ -42,7 +38,7 @@ func TestDeploymentService(t *testing.T) { JobName: "job3", Host: "some-host", TaskName: "bq2bq", - DestinationURN: urn, + DestinationURN: "bigquery://some-resource", Tenant: tenant.Tenant{}, Type: scheduler.UpstreamTypeInferred, External: true, diff --git a/core/scheduler/service/events_service_test.go b/core/scheduler/service/events_service_test.go index b873e2dc18..f7430d5844 100644 --- a/core/scheduler/service/events_service_test.go +++ b/core/scheduler/service/events_service_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/scheduler" "github.com/goto/optimus/core/scheduler/service" "github.com/goto/optimus/core/tenant" @@ -110,7 +109,7 @@ func TestNotificationService(t *testing.T) { JobEvent: event, Meta: &scheduler.JobRunMeta{ Labels: nil, - DestinationURN: resource.ZeroURN(), + DestinationURN: "", }, Route: "http://someDomain.com/endpoint", Headers: map[string]string{ diff --git a/core/scheduler/service/executor_input_compiler.go b/core/scheduler/service/executor_input_compiler.go index a083298433..be4dc690ce 100644 --- a/core/scheduler/service/executor_input_compiler.go +++ b/core/scheduler/service/executor_input_compiler.go @@ -222,7 +222,7 @@ func getSystemDefinedConfigs(job *scheduler.Job, interval interval.Interval, exe configDstart: interval.Start().Format(TimeISOFormat), configDend: interval.End().Format(TimeISOFormat), configExecutionTime: executedAt.Format(TimeISOFormat), - configDestination: job.Destination.String(), + configDestination: job.Destination, } // TODO: remove this condition after v1/v2 removal, add to map directly if job.WindowConfig.GetVersion() == window.NewWindowVersion { diff --git a/core/scheduler/service/executor_input_compiler_test.go b/core/scheduler/service/executor_input_compiler_test.go index 167078bdde..ef94918fbc 100644 --- a/core/scheduler/service/executor_input_compiler_test.go +++ b/core/scheduler/service/executor_input_compiler_test.go @@ -13,7 +13,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/scheduler" "github.com/goto/optimus/core/scheduler/service" "github.com/goto/optimus/core/tenant" @@ -43,9 +42,6 @@ func TestExecutorCompiler(t *testing.T) { logger := log.NewLogrus() - destinationURN, err := resource.ParseURN("store://some_destination_table_name") - assert.NoError(t, err) - t.Run("Compile", func(t *testing.T) { t.Run("should give error if tenant service getDetails fails", func(t *testing.T) { job := scheduler.Job{ @@ -148,7 +144,7 @@ func TestExecutorCompiler(t *testing.T) { "DSTART": interval.Start().Format(time.RFC3339), "DEND": interval.End().Format(time.RFC3339), "EXECUTION_TIME": executedAt.Format(time.RFC3339), - "JOB_DESTINATION": job.Destination.String(), + "JOB_DESTINATION": job.Destination, } taskContext := mock.Anything @@ -172,7 +168,7 @@ func TestExecutorCompiler(t *testing.T) { job := scheduler.Job{ Name: "job1", Tenant: tnnt, - Destination: destinationURN, + Destination: "some_destination_table_name", Task: &scheduler.Task{ Name: "bq2bq", Config: map[string]string{ @@ -217,7 +213,7 @@ func TestExecutorCompiler(t *testing.T) { "DSTART": interval.Start().Format(time.RFC3339), "DEND": interval.End().Format(time.RFC3339), "EXECUTION_TIME": executedAt.Format(time.RFC3339), - "JOB_DESTINATION": job.Destination.String(), + "JOB_DESTINATION": job.Destination, } taskContext := mock.Anything @@ -274,7 +270,7 @@ func TestExecutorCompiler(t *testing.T) { "DSTART": interval.Start().Format(time.RFC3339), "DEND": interval.End().Format(time.RFC3339), "EXECUTION_TIME": executedAt.Format(time.RFC3339), - "JOB_DESTINATION": job.Destination.String(), + "JOB_DESTINATION": job.Destination, "some.config.compiled": "val.compiled", }, Secrets: map[string]string{"secret.config.compiled": "a.secret.val.compiled"}, @@ -326,7 +322,7 @@ func TestExecutorCompiler(t *testing.T) { "DSTART": interval.Start().Format(time.RFC3339), "DEND": interval.End().Format(time.RFC3339), "EXECUTION_TIME": executedAt.Format(time.RFC3339), - "JOB_DESTINATION": job.Destination.String(), + "JOB_DESTINATION": job.Destination, "some.config.compiled": "val.compiled", }, Secrets: map[string]string{"secret.config.compiled": "a.secret.val.compiled"}, @@ -354,7 +350,7 @@ func TestExecutorCompiler(t *testing.T) { job := scheduler.Job{ Name: "job1", Tenant: tnnt, - Destination: destinationURN, + Destination: "some_destination_table_name", Task: &scheduler.Task{ Name: "bq2bq", Config: map[string]string{ @@ -400,7 +396,7 @@ func TestExecutorCompiler(t *testing.T) { "DSTART": interval.Start().Format(time.RFC3339), "DEND": interval.End().Format(time.RFC3339), "EXECUTION_TIME": executedAt.Format(time.RFC3339), - "JOB_DESTINATION": job.Destination.String(), + "JOB_DESTINATION": job.Destination, } taskContext := mock.Anything @@ -431,7 +427,7 @@ func TestExecutorCompiler(t *testing.T) { "DSTART": interval.Start().Format(time.RFC3339), "DEND": interval.End().Format(time.RFC3339), "EXECUTION_TIME": executedAt.Format(time.RFC3339), - "JOB_DESTINATION": job.Destination.String(), + "JOB_DESTINATION": job.Destination, "hook.compiled": "hook.val.compiled", "JOB_LABELS": "job_id=00000000-0000-0000-0000-000000000000,job_name=job1,namespace=ns1,project=proj1", }, @@ -453,7 +449,7 @@ func TestExecutorCompiler(t *testing.T) { job := scheduler.Job{ Name: "job1", Tenant: tnnt, - Destination: destinationURN, + Destination: "some_destination_table_name", Task: &scheduler.Task{ Name: "bq2bq", Config: map[string]string{ @@ -499,7 +495,7 @@ func TestExecutorCompiler(t *testing.T) { "DSTART": interval.Start().Format(time.RFC3339), "DEND": interval.End().Format(time.RFC3339), "EXECUTION_TIME": executedAt.Format(time.RFC3339), - "JOB_DESTINATION": job.Destination.String(), + "JOB_DESTINATION": job.Destination, } taskContext := mock.Anything @@ -533,7 +529,7 @@ func TestExecutorCompiler(t *testing.T) { job := scheduler.Job{ Name: "job1", Tenant: tnnt, - Destination: destinationURN, + Destination: "some_destination_table_name", Task: &scheduler.Task{ Name: "bq2bq", Config: map[string]string{ @@ -571,7 +567,7 @@ func TestExecutorCompiler(t *testing.T) { "DSTART": interval.Start().Format(time.RFC3339), "DEND": interval.End().Format(time.RFC3339), "EXECUTION_TIME": executedAt.Format(time.RFC3339), - "JOB_DESTINATION": job.Destination.String(), + "JOB_DESTINATION": job.Destination, } taskContext := mock.Anything diff --git a/core/scheduler/service/job_run_asset_compiler_test.go b/core/scheduler/service/job_run_asset_compiler_test.go index fe37227a8c..74965fd7e0 100644 --- a/core/scheduler/service/job_run_asset_compiler_test.go +++ b/core/scheduler/service/job_run_asset_compiler_test.go @@ -57,7 +57,7 @@ func TestJobAssetsCompiler(t *testing.T) { "DSTART": interval.Start().Format(time.RFC3339), "DEND": interval.End().Format(time.RFC3339), "EXECUTION_TIME": executedAt.Format(time.RFC3339), - "JOB_DESTINATION": job.Destination.String(), + "JOB_DESTINATION": job.Destination, } logger := log.NewNoop() diff --git a/ext/notify/webhook/webhook.go b/ext/notify/webhook/webhook.go index 38388d7ce3..6f64c1ce14 100644 --- a/ext/notify/webhook/webhook.go +++ b/ext/notify/webhook/webhook.go @@ -85,7 +85,7 @@ func fireWebhook(e event) error { JobName: e.meta.JobName.String(), Project: e.meta.Tenant.ProjectName().String(), Namespace: e.meta.Tenant.NamespaceName().String(), - Destination: e.jobMeta.DestinationURN.String(), + Destination: e.jobMeta.DestinationURN, ScheduledAt: e.meta.JobScheduledAt.String(), Status: e.meta.Status.String(), JobLabel: e.jobMeta.Labels, diff --git a/ext/notify/webhook/webhook_test.go b/ext/notify/webhook/webhook_test.go index 4f0fb31f1e..bc828d1eee 100644 --- a/ext/notify/webhook/webhook_test.go +++ b/ext/notify/webhook/webhook_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/scheduler" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/ext/notify/webhook" @@ -19,9 +18,7 @@ import ( func TestWebhook(t *testing.T) { projectName := "ss" namespaceName := "bb" - jobDestinationTableURN, err := resource.ParseURN("store://project-dest-table") - assert.NoError(t, err) - + jobDestinationTableURN := "project-dest-table" jobName := scheduler.JobName("foo-job-spec") tnnt, _ := tenant.NewTenant(projectName, namespaceName) eventTime := time.Now() diff --git a/ext/resourcemanager/optimus.go b/ext/resourcemanager/optimus.go index 45054aea97..d429ca6daa 100644 --- a/ext/resourcemanager/optimus.go +++ b/ext/resourcemanager/optimus.go @@ -14,7 +14,6 @@ import ( "github.com/goto/optimus/config" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" ) @@ -109,7 +108,7 @@ func (o *OptimusResourceManager) constructGetJobSpecificationsRequest(ctx contex if unresolvedDependency.ProjectName() != "" { filters = append(filters, fmt.Sprintf("project_name=%s", unresolvedDependency.ProjectName().String())) } - if !unresolvedDependency.Resource().IsZero() { + if unresolvedDependency.Resource() != "" { filters = append(filters, fmt.Sprintf("resource_destination=%s", unresolvedDependency.Resource().String())) } @@ -153,15 +152,6 @@ func (o *OptimusResourceManager) toOptimusDependency(response *jobSpecificationR if err != nil { return nil, err } - - var resourceURN resource.URN - if response.Job.Destination != "" { - urn, err := resource.ParseURN(response.Job.Destination) - if err != nil { - return nil, err - } - resourceURN = urn - } - + resourceURN := job.ResourceURN(response.Job.Destination) return job.NewUpstreamResolved(jobName, o.config.Host, resourceURN, jobTenant, unresolvedDependency.Type(), taskName, true), nil } diff --git a/ext/resourcemanager/optimus_test.go b/ext/resourcemanager/optimus_test.go index e1b29cd125..27e6c7ae9b 100644 --- a/ext/resourcemanager/optimus_test.go +++ b/ext/resourcemanager/optimus_test.go @@ -11,7 +11,6 @@ import ( "github.com/goto/optimus/config" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/ext/resourcemanager" ) @@ -24,9 +23,6 @@ func (o *OptimusResourceManager) TestGetJobSpecifications() { apiPath := "/api/v1beta1/jobs" sampleTenant, _ := tenant.NewTenant("test-proj", "test-ns") - urn, err := resource.NewURN("store", "sample-resource") - o.NoError(err) - o.Run("should return nil and error if context is nil", func() { conf := config.ResourceManager{ Config: config.ResourceManagerConfigOptimus{ @@ -178,7 +174,7 @@ func (o *OptimusResourceManager) TestGetJobSpecifications() { ctx := context.Background() unresolvedUpstream := job.NewUpstreamUnresolvedStatic("job", "test-proj") - dependency := job.NewUpstreamResolved("job", server.URL, resource.ZeroURN(), sampleTenant, "static", "sample-task", true) + dependency := job.NewUpstreamResolved("job", server.URL, "", sampleTenant, "static", "sample-task", true) expectedDependencies := []*job.Upstream{dependency} actualOptimusDependencies, actualError := manager.GetOptimusUpstreams(ctx, unresolvedUpstream) @@ -211,7 +207,7 @@ func (o *OptimusResourceManager) TestGetJobSpecifications() { actualHeaderValue := r.Header.Get("key") o.EqualValues(expectedHeaderValue, actualHeaderValue) - expectedRawQuery := "resource_destination=store://sample-resource" + expectedRawQuery := "resource_destination=sample-resource" actualRawQuery := r.URL.RawQuery o.EqualValues(expectedRawQuery, actualRawQuery) @@ -236,9 +232,9 @@ func (o *OptimusResourceManager) TestGetJobSpecifications() { }) ctx := context.Background() - unresolvedUpstream := job.NewUpstreamUnresolvedInferred(urn) + unresolvedUpstream := job.NewUpstreamUnresolvedInferred("sample-resource") - dependency := job.NewUpstreamResolved("job", server.URL, resource.ZeroURN(), sampleTenant, "inferred", "sample-task", true) + dependency := job.NewUpstreamResolved("job", server.URL, "", sampleTenant, "inferred", "sample-task", true) expectedDependencies := []*job.Upstream{dependency} actualOptimusDependencies, actualError := manager.GetOptimusUpstreams(ctx, unresolvedUpstream) @@ -292,7 +288,7 @@ func (o *OptimusResourceManager) TestGetJobSpecifications() { }] }], "taskName": "task-1", - "destination": "store://sample-resource" + "destination": "resource" } } ] @@ -305,7 +301,7 @@ func (o *OptimusResourceManager) TestGetJobSpecifications() { ctx := context.Background() unresolvedUpstream := job.NewUpstreamUnresolvedStatic("job", "test-proj") - dependency := job.NewUpstreamResolved("job", server.URL, urn, sampleTenant, "static", "task-1", true) + dependency := job.NewUpstreamResolved("job", server.URL, "resource", sampleTenant, "static", "task-1", true) expectedDependencies := []*job.Upstream{dependency} actualOptimusDependencies, actualError := manager.GetOptimusUpstreams(ctx, unresolvedUpstream) diff --git a/ext/scheduler/airflow/dag/compiler_test.go b/ext/scheduler/airflow/dag/compiler_test.go index f321c6d9d5..a0831db6c5 100644 --- a/ext/scheduler/airflow/dag/compiler_test.go +++ b/ext/scheduler/airflow/dag/compiler_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/assert" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/scheduler" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/ext/scheduler/airflow/dag" @@ -159,16 +158,11 @@ func setupJobDetails(tnnt tenant.Tenant) *scheduler.JobWithDetails { Labels: map[string]string{"orchestrator": "optimus"}, } - urn, err := resource.ParseURN("bigquery://billing:reports.weekly-status") - if err != nil { - panic(err) - } - jobName := scheduler.JobName("infra.billing.weekly-status-reports") job := &scheduler.Job{ Name: jobName, Tenant: tnnt, - Destination: urn, + Destination: "bigquery://billing:reports.weekly-status", Task: &scheduler.Task{Name: "bq-bq"}, Hooks: hooks, WindowConfig: window1, diff --git a/ext/store/bigquery/backup.go b/ext/store/bigquery/backup.go index 162d36383f..8d8fe61f6f 100644 --- a/ext/store/bigquery/backup.go +++ b/ext/store/bigquery/backup.go @@ -41,7 +41,7 @@ func BackupResources(ctx context.Context, backup *resource.Backup, resources []* var backupNames []string if len(tablesToBackup) > 0 { - dataset, err := DataSetFor(tablesToBackup[0].Name()) + dataset, err := DataSetFor(tablesToBackup[0]) if err != nil { return nil, err } @@ -92,11 +92,11 @@ func CreateIfDatasetDoesNotExist(ctx context.Context, client Client, dataset Dat } func BackupTable(ctx context.Context, backup *resource.Backup, source *resource.Resource, client Client) (string, error) { - sourceDataset, err := DataSetFor(source.Name()) + sourceDataset, err := DataSetFor(source) if err != nil { return "", err } - sourceName, err := ResourceNameFor(source.Name(), source.Kind()) + sourceName, err := ResourceNameFor(source) if err != nil { return "", err } diff --git a/ext/store/bigquery/backup_test.go b/ext/store/bigquery/backup_test.go index 5bebbb8fb7..db2515529a 100644 --- a/ext/store/bigquery/backup_test.go +++ b/ext/store/bigquery/backup_test.go @@ -318,7 +318,7 @@ func TestBigqueryBackup(t *testing.T) { backup, err := resource.NewBackup(store, tnnt, []string{fullName}, "", createdAt, nil) assert.NoError(t, err) - dataset, err := bigquery.DataSetFor(source.Name()) + dataset, err := bigquery.DataSetFor(source) assert.NoError(t, err) destinationDS, err := bigquery.DestinationDataset(dataset.Project, backup) @@ -331,10 +331,10 @@ func TestBigqueryBackup(t *testing.T) { backup, err := resource.NewBackup(store, tnnt, []string{fullName}, "", createdAt, nil) assert.NoError(t, err) - dataset, err := bigquery.DataSetFor(source.Name()) + dataset, err := bigquery.DataSetFor(source) assert.NoError(t, err) - name, err := bigquery.ResourceNameFor(source.Name(), source.Kind()) + name, err := bigquery.ResourceNameFor(source) assert.NoError(t, err) destination := bigquery.DestinationName(dataset.DatasetName, name, backup) diff --git a/ext/store/bigquery/batch.go b/ext/store/bigquery/batch.go index 786e1becc5..568df5535a 100644 --- a/ext/store/bigquery/batch.go +++ b/ext/store/bigquery/batch.go @@ -40,11 +40,11 @@ func (b *Batch) QueueJobs(ctx context.Context, account string, runner *parallel. for _, table := range b.Tables { runner.Add(func(res *resource.Resource) func() (interface{}, error) { return func() (interface{}, error) { - ds, err := DataSetFor(res.Name()) + ds, err := DataSetFor(res) if err != nil { return res, err } - resourceName, err := ResourceNameFor(res.Name(), res.Kind()) + resourceName, err := ResourceNameFor(res) if err != nil { return res, err } @@ -58,11 +58,11 @@ func (b *Batch) QueueJobs(ctx context.Context, account string, runner *parallel. for _, extTables := range b.ExternalTables { runner.Add(func(res *resource.Resource) func() (interface{}, error) { return func() (interface{}, error) { - ds, err := DataSetFor(res.Name()) + ds, err := DataSetFor(res) if err != nil { return res, err } - resourceName, err := ResourceNameFor(res.Name(), res.Kind()) + resourceName, err := ResourceNameFor(res) if err != nil { return res, err } @@ -76,11 +76,11 @@ func (b *Batch) QueueJobs(ctx context.Context, account string, runner *parallel. for _, view := range b.Views { runner.Add(func(res *resource.Resource) func() (interface{}, error) { return func() (interface{}, error) { - ds, err := DataSetFor(res.Name()) + ds, err := DataSetFor(res) if err != nil { return res, err } - resourceName, err := ResourceNameFor(res.Name(), res.Kind()) + resourceName, err := ResourceNameFor(res) if err != nil { return res, err } @@ -149,7 +149,7 @@ func BatchesFrom(resources []*resource.Resource, provider ClientProvider) (map[s me := errors.NewMultiError("error while creating batches") for _, res := range resources { - dataset, err := DataSetFor(res.Name()) + dataset, err := DataSetFor(res) if err != nil { me.Append(err) continue diff --git a/ext/store/bigquery/batch_test.go b/ext/store/bigquery/batch_test.go index 4ba2d7491b..3682c46e21 100644 --- a/ext/store/bigquery/batch_test.go +++ b/ext/store/bigquery/batch_test.go @@ -117,9 +117,9 @@ func TestBatches(t *testing.T) { extTableHandle.On("Exists", ctx).Return(true) defer extTableHandle.AssertExpectations(t) - dataset1, err := bigquery.DataSetFor(updateDS.Name()) + dataset1, err := bigquery.DataSetFor(updateDS) assert.NoError(t, err) - dataset2, err := bigquery.DataSetFor(updateExt1.Name()) + dataset2, err := bigquery.DataSetFor(updateExt1) assert.NoError(t, err) client := new(mockClient) @@ -189,7 +189,7 @@ func TestBatches(t *testing.T) { accountSecret := "secret for account" - tab1Dataset, err := bigquery.DataSetFor(tab1.Name()) + tab1Dataset, err := bigquery.DataSetFor(tab1) assert.NoError(t, err) datasetHandle.On("Exists", ctx).Return(false) @@ -220,7 +220,7 @@ func TestBatches(t *testing.T) { accountSecret := "secret for account" - tab1Dataset, err := bigquery.DataSetFor(tab1.Name()) + tab1Dataset, err := bigquery.DataSetFor(tab1) assert.NoError(t, err) datasetHandle.On("Exists", ctx).Return(true) @@ -253,7 +253,7 @@ func TestBatches(t *testing.T) { dsToCreate := resource.FromExisting(ds1, resource.ReplaceStatus(resource.StatusToCreate)) - dsToCreateDataset, err := bigquery.DataSetFor(dsToCreate.Name()) + dsToCreateDataset, err := bigquery.DataSetFor(dsToCreate) assert.NoError(t, err) datasetHandle.On("Exists", ctx).Return(false) diff --git a/ext/store/bigquery/bigquery.go b/ext/store/bigquery/bigquery.go index 62e51c6f4b..17c07c42b6 100644 --- a/ext/store/bigquery/bigquery.go +++ b/ext/store/bigquery/bigquery.go @@ -2,7 +2,6 @@ package bigquery import ( "context" - "fmt" "time" bq "cloud.google.com/go/bigquery" @@ -21,8 +20,6 @@ const ( ConcurrentTicketPerSec = 5 ConcurrentLimit = 20 - - bigqueryID = "bigquery" ) type ResourceHandle interface { @@ -75,11 +72,11 @@ func (s Store) Create(ctx context.Context, res *resource.Resource) error { } defer client.Close() - dataset, err := DataSetFor(res.Name()) + dataset, err := DataSetFor(res) if err != nil { return err } - resourceName, err := ResourceNameFor(res.Name(), res.Kind()) + resourceName, err := ResourceNameFor(res) if err != nil { return err } @@ -121,11 +118,11 @@ func (s Store) Update(ctx context.Context, res *resource.Resource) error { } defer client.Close() - dataset, err := DataSetFor(res.Name()) + dataset, err := DataSetFor(res) if err != nil { return err } - resourceName, err := ResourceNameFor(res.Name(), res.Kind()) + resourceName, err := ResourceNameFor(res) if err != nil { return err } @@ -230,7 +227,7 @@ func (Store) Validate(r *resource.Resource) error { } } -func (Store) GetURN(res *resource.Resource) (resource.URN, error) { +func (Store) GetURN(res *resource.Resource) (string, error) { return URNFor(res) } @@ -249,62 +246,6 @@ func (s Store) Backup(ctx context.Context, backup *resource.Backup, resources [] return BackupResources(ctx, backup, resources, client) } -func (s Store) Exist(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) { - spanCtx, span := startChildSpan(ctx, "bigquery/Exist") - defer span.End() - - if urn.GetStore() != bigqueryID { - msg := fmt.Sprintf("expected store [%s] but received [%s]", bigqueryID, urn.GetStore()) - return false, errors.InvalidArgument(store, msg) - } - - account, err := s.secretProvider.GetSecret(spanCtx, tnnt, accountKey) - if err != nil { - return false, err - } - - client, err := s.clientProvider.Get(spanCtx, account.Value()) - if err != nil { - return false, err - } - defer client.Close() - - name, err := resource.NameFrom(urn.GetName()) - if err != nil { - return false, err - } - - dataset, err := DataSetFor(name) - if err != nil { - return false, err - } - - if !client.DatasetHandleFrom(dataset).Exists(spanCtx) { - return false, nil - } - - kindToHandleFn := map[string]func(ds Dataset, name string) ResourceHandle{ - KindTable: func(ds Dataset, name string) ResourceHandle { - return client.TableHandleFrom(ds, name) - }, - KindExternalTable: client.ExternalTableHandleFrom, - KindView: client.ViewHandleFrom, - } - - for kind, resourceHandleFn := range kindToHandleFn { - resourceName, err := ResourceNameFor(name, kind) - if err != nil { - return true, nil // nolint:nilerr - } - - if resourceHandleFn(dataset, resourceName).Exists(spanCtx) { - return true, nil - } - } - - return false, nil -} - func startChildSpan(ctx context.Context, name string) (context.Context, trace.Span) { tracer := otel.Tracer("datastore/bigquery") diff --git a/ext/store/bigquery/bigquery_test.go b/ext/store/bigquery/bigquery_test.go index df85352a7b..4a8f2052a1 100644 --- a/ext/store/bigquery/bigquery_test.go +++ b/ext/store/bigquery/bigquery_test.go @@ -676,13 +676,10 @@ func TestBigqueryStore(t *testing.T) { tnnt, &resource.Metadata{}, spec) assert.NoError(t, err) - expectedURN, err := resource.ParseURN("bigquery://project:set.view_name1") - assert.NoError(t, err) - bqStore := bigquery.NewBigqueryDataStore(nil, nil) - actualURN, err := bqStore.GetURN(res) + urn, err := bqStore.GetURN(res) assert.NoError(t, err) - assert.Equal(t, expectedURN, actualURN) + assert.Equal(t, "bigquery://project:set.view_name1", urn) }) }) t.Run("Backup", func(t *testing.T) { @@ -751,238 +748,6 @@ func TestBigqueryStore(t *testing.T) { assert.Equal(t, 1, len(result.IgnoredResources)) }) }) - t.Run("Exist", func(t *testing.T) { - t.Run("returns false and error if store is not bigquery", func(t *testing.T) { - secretProvider := new(mockSecretProvider) - defer secretProvider.AssertExpectations(t) - - extTableHandle := new(mockTableResourceHandle) - defer extTableHandle.AssertExpectations(t) - - client := new(mockClient) - defer client.AssertExpectations(t) - - clientProvider := new(mockClientProvider) - defer clientProvider.AssertExpectations(t) - - bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) - - urn, err := resource.NewURN("random_store", "project.dataset.table") - assert.NoError(t, err) - - actualExist, actualError := bqStore.Exist(ctx, tnnt, urn) - - assert.False(t, actualExist) - assert.ErrorContains(t, actualError, "expected store [bigquery] but received [random_store]") - }) - - t.Run("returns false and error if secret provider returned error", func(t *testing.T) { - secretProvider := new(mockSecretProvider) - defer secretProvider.AssertExpectations(t) - - extTableHandle := new(mockTableResourceHandle) - defer extTableHandle.AssertExpectations(t) - - client := new(mockClient) - defer client.AssertExpectations(t) - - clientProvider := new(mockClientProvider) - defer clientProvider.AssertExpectations(t) - - bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) - - urn, err := resource.NewURN("bigquery", "project.dataset.table") - assert.NoError(t, err) - - secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY").Return(nil, errors.New("expected error for testing")) - - actualExist, actualError := bqStore.Exist(ctx, tnnt, urn) - - assert.False(t, actualExist) - assert.ErrorContains(t, actualError, "expected error for testing") - }) - - t.Run("returns false and error if client provider returned error", func(t *testing.T) { - secretProvider := new(mockSecretProvider) - defer secretProvider.AssertExpectations(t) - - extTableHandle := new(mockTableResourceHandle) - defer extTableHandle.AssertExpectations(t) - - client := new(mockClient) - defer client.AssertExpectations(t) - - clientProvider := new(mockClientProvider) - defer clientProvider.AssertExpectations(t) - - bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) - - urn, err := resource.NewURN("bigquery", "project.dataset.table") - assert.NoError(t, err) - - pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") - secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY").Return(pts, nil) - - clientProvider.On("Get", mock.Anything, pts.Value()).Return(nil, errors.New("expected error for testing")) - - actualExist, actualError := bqStore.Exist(ctx, tnnt, urn) - - assert.False(t, actualExist) - assert.ErrorContains(t, actualError, "expected error for testing") - }) - - t.Run("returns false and error if dataset name is invalid", func(t *testing.T) { - secretProvider := new(mockSecretProvider) - defer secretProvider.AssertExpectations(t) - - extTableHandle := new(mockTableResourceHandle) - defer extTableHandle.AssertExpectations(t) - - client := new(mockClient) - defer client.AssertExpectations(t) - - clientProvider := new(mockClientProvider) - defer clientProvider.AssertExpectations(t) - - bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) - - urn, err := resource.NewURN("bigquery", "invalid_name") - assert.NoError(t, err) - - pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") - secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY").Return(pts, nil) - - clientProvider.On("Get", mock.Anything, pts.Value()).Return(client, nil) - - client.On("Close").Return(nil) - - actualExist, actualError := bqStore.Exist(ctx, tnnt, urn) - - assert.False(t, actualExist) - assert.ErrorContains(t, actualError, "invalid dataset name") - }) - - t.Run("returns true and nil if dataset exists", func(t *testing.T) { - secretProvider := new(mockSecretProvider) - defer secretProvider.AssertExpectations(t) - - extTableHandle := new(mockTableResourceHandle) - defer extTableHandle.AssertExpectations(t) - - client := new(mockClient) - defer client.AssertExpectations(t) - - clientProvider := new(mockClientProvider) - defer clientProvider.AssertExpectations(t) - - handle := new(mockTableResourceHandle) - defer handle.AssertExpectations(t) - - bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) - - urn, err := resource.NewURN("bigquery", "project.dataset") - assert.NoError(t, err) - - pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") - secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY").Return(pts, nil) - - clientProvider.On("Get", mock.Anything, pts.Value()).Return(client, nil) - - client.On("Close").Return(nil) - - handle.On("Exists", mock.Anything).Return(true) - - client.On("DatasetHandleFrom", mock.Anything).Return(handle) - - actualExist, actualError := bqStore.Exist(ctx, tnnt, urn) - - assert.True(t, actualExist) - assert.NoError(t, actualError) - }) - - t.Run("returns true and nil if dataset exists and underlying resource does exist", func(t *testing.T) { - secretProvider := new(mockSecretProvider) - defer secretProvider.AssertExpectations(t) - - extTableHandle := new(mockTableResourceHandle) - defer extTableHandle.AssertExpectations(t) - - client := new(mockClient) - defer client.AssertExpectations(t) - - clientProvider := new(mockClientProvider) - defer clientProvider.AssertExpectations(t) - - handle := new(mockTableResourceHandle) - defer handle.AssertExpectations(t) - - bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) - - urn, err := resource.NewURN("bigquery", "project.dataset.table") - assert.NoError(t, err) - - pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") - secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY").Return(pts, nil) - - clientProvider.On("Get", mock.Anything, pts.Value()).Return(client, nil) - - client.On("Close").Return(nil) - - handle.On("Exists", mock.Anything).Return(true) - - client.On("DatasetHandleFrom", mock.Anything, mock.Anything).Return(handle) - client.On("TableHandleFrom", mock.Anything, mock.Anything).Return(handle).Maybe() - client.On("ExternalTableHandleFrom", mock.Anything, mock.Anything).Return(handle).Maybe() - client.On("ViewHandleFrom", mock.Anything, mock.Anything).Return(handle).Maybe() - - actualExist, actualError := bqStore.Exist(ctx, tnnt, urn) - - assert.True(t, actualExist) - assert.NoError(t, actualError) - }) - - t.Run("returns false and nil if dataset exists and underlying resource does not exist", func(t *testing.T) { - secretProvider := new(mockSecretProvider) - defer secretProvider.AssertExpectations(t) - - extTableHandle := new(mockTableResourceHandle) - defer extTableHandle.AssertExpectations(t) - - client := new(mockClient) - defer client.AssertExpectations(t) - - clientProvider := new(mockClientProvider) - defer clientProvider.AssertExpectations(t) - - handle := new(mockTableResourceHandle) - defer handle.AssertExpectations(t) - - bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) - - urn, err := resource.NewURN("bigquery", "project.dataset.table") - assert.NoError(t, err) - - pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") - secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY").Return(pts, nil) - - clientProvider.On("Get", mock.Anything, pts.Value()).Return(client, nil) - - client.On("Close").Return(nil) - - handle.On("Exists", mock.Anything).Return(true).Once() - handle.On("Exists", mock.Anything).Return(false).Times(3) - - client.On("DatasetHandleFrom", mock.Anything, mock.Anything).Return(handle) - client.On("TableHandleFrom", mock.Anything, mock.Anything).Return(handle) - client.On("ExternalTableHandleFrom", mock.Anything, mock.Anything).Return(handle) - client.On("ViewHandleFrom", mock.Anything, mock.Anything).Return(handle) - - actualExist, actualError := bqStore.Exist(ctx, tnnt, urn) - - assert.False(t, actualExist) - assert.NoError(t, actualError) - }) - }) } type mockClientProvider struct { diff --git a/ext/store/bigquery/dataset_spec.go b/ext/store/bigquery/dataset_spec.go index a3550d64df..5a6f80856d 100644 --- a/ext/store/bigquery/dataset_spec.go +++ b/ext/store/bigquery/dataset_spec.go @@ -71,38 +71,38 @@ func (d Dataset) FullName() string { return d.Project + "." + d.DatasetName } -func DataSetFor(name resource.Name) (Dataset, error) { - sections := name.Sections() +func DataSetFor(res *resource.Resource) (Dataset, error) { + sections := res.NameSections() if len(sections) < DatesetNameSections { - return Dataset{}, errors.InvalidArgument(EntityDataset, "invalid dataset name: "+name.String()) + return Dataset{}, errors.InvalidArgument(EntityDataset, "invalid dataset name: "+res.FullName()) } return DataSetFrom(sections[0], sections[1]) } -func ResourceNameFor(name resource.Name, kind string) (string, error) { - sections := name.Sections() +func ResourceNameFor(res *resource.Resource) (string, error) { + sections := res.NameSections() var strName string - if kind == KindDataset { + if res.Kind() == KindDataset { if len(sections) < DatesetNameSections { - return "", errors.InvalidArgument(resource.EntityResource, "invalid resource name: "+name.String()) + return "", errors.InvalidArgument(resource.EntityResource, "invalid resource name: "+res.FullName()) } strName = sections[1] } else { if len(sections) < TableNameSections { - return "", errors.InvalidArgument(resource.EntityResource, "invalid resource name: "+name.String()) + return "", errors.InvalidArgument(resource.EntityResource, "invalid resource name: "+res.FullName()) } strName = sections[2] } if strName == "" { - return "", errors.InvalidArgument(resource.EntityResource, "invalid resource name: "+name.String()) + return "", errors.InvalidArgument(resource.EntityResource, "invalid resource name: "+res.FullName()) } return strName, nil } func ValidateName(res *resource.Resource) error { - sections := res.Name().Sections() + sections := res.NameSections() if len(sections) < DatesetNameSections { return errors.InvalidArgument(resource.EntityResource, "invalid sections in name: "+res.FullName()) } @@ -127,22 +127,21 @@ func ValidateName(res *resource.Resource) error { return nil } -func URNFor(res *resource.Resource) (resource.URN, error) { - dataset, err := DataSetFor(res.Name()) +func URNFor(res *resource.Resource) (string, error) { + dataset, err := DataSetFor(res) if err != nil { - return resource.ZeroURN(), err + return "", err } + datasetURN := string(resource.Bigquery) + "://" + dataset.Project + ":" + dataset.DatasetName if res.Kind() == KindDataset { - name := dataset.Project + ":" + dataset.DatasetName - return resource.NewURN(resource.Bigquery.String(), name) + return datasetURN, nil } - resourceName, err := ResourceNameFor(res.Name(), res.Kind()) + name, err := ResourceNameFor(res) if err != nil { - return resource.ZeroURN(), err + return "", err } - name := dataset.Project + ":" + dataset.DatasetName + "." + resourceName - return resource.NewURN(resource.Bigquery.String(), name) + return datasetURN + "." + name, nil } diff --git a/ext/store/bigquery/dataset_spec_test.go b/ext/store/bigquery/dataset_spec_test.go index 0622b4f4be..0d077723a3 100644 --- a/ext/store/bigquery/dataset_spec_test.go +++ b/ext/store/bigquery/dataset_spec_test.go @@ -60,11 +60,11 @@ func TestDataSet(t *testing.T) { invalidRes, err := resource.NewResource("proj.", bigquery.KindExternalTable, bqStore, tnnt, &metadata, spec) assert.Nil(t, err) - _, err = bigquery.DataSetFor(invalidRes.Name()) + _, err = bigquery.DataSetFor(invalidRes) assert.Error(t, err) }) t.Run("returns dataset when valid", func(t *testing.T) { - ds, err := bigquery.DataSetFor(res.Name()) + ds, err := bigquery.DataSetFor(res) assert.NoError(t, err) assert.Equal(t, "proj", ds.Project) @@ -90,7 +90,7 @@ func TestResourceName(t *testing.T) { dsRes, dsErr := resource.NewResource("proj", bigquery.KindDataset, bqStore, tnnt, &metadata, spec) assert.NoError(t, dsErr) - _, err := bigquery.ResourceNameFor(dsRes.Name(), dsRes.Kind()) + _, err := bigquery.ResourceNameFor(dsRes) assert.Error(t, err) assert.ErrorContains(t, err, "invalid resource name: proj") }) @@ -98,7 +98,7 @@ func TestResourceName(t *testing.T) { dsRes, dsErr := resource.NewResource("proj.dataset", bigquery.KindDataset, bqStore, tnnt, &metadata, spec) assert.NoError(t, dsErr) - name, err := bigquery.ResourceNameFor(dsRes.Name(), dsRes.Kind()) + name, err := bigquery.ResourceNameFor(dsRes) assert.NoError(t, err) assert.Equal(t, "dataset", name) }) @@ -108,7 +108,7 @@ func TestResourceName(t *testing.T) { dsRes, dsErr := resource.NewResource("proj.dataset", bigquery.KindTable, bqStore, tnnt, &metadata, spec) assert.NoError(t, dsErr) - _, err := bigquery.ResourceNameFor(dsRes.Name(), dsRes.Kind()) + _, err := bigquery.ResourceNameFor(dsRes) assert.Error(t, err) assert.ErrorContains(t, err, "invalid resource name") }) @@ -116,12 +116,12 @@ func TestResourceName(t *testing.T) { dsRes, dsErr := resource.NewResource("proj.dataset.", bigquery.KindTable, bqStore, tnnt, &metadata, spec) assert.NoError(t, dsErr) - _, err := bigquery.ResourceNameFor(dsRes.Name(), dsRes.Kind()) + _, err := bigquery.ResourceNameFor(dsRes) assert.Error(t, err) assert.ErrorContains(t, err, "invalid resource name") }) t.Run("name when valid", func(t *testing.T) { - name, err := bigquery.ResourceNameFor(res.Name(), res.Kind()) + name, err := bigquery.ResourceNameFor(res) assert.NoError(t, err) assert.Equal(t, "extTable1", name) }) @@ -221,7 +221,7 @@ func TestURNFor(t *testing.T) { urn, err := bigquery.URNFor(res) assert.NoError(t, err) - assert.Equal(t, "bigquery://p-project:dataset", urn.String()) + assert.Equal(t, "bigquery://p-project:dataset", urn) }) t.Run("returns error when cannot get resource name", func(t *testing.T) { res, err := resource.NewResource("p-project.dataset.", bigquery.KindTable, bqStore, tnnt, &metadata, spec) @@ -237,6 +237,6 @@ func TestURNFor(t *testing.T) { urn, err := bigquery.URNFor(res) assert.NoError(t, err) - assert.Equal(t, "bigquery://p-project:dataset.table1", urn.String()) + assert.Equal(t, "bigquery://p-project:dataset.table1", urn) }) } diff --git a/internal/store/postgres/job/job_repository.go b/internal/store/postgres/job/job_repository.go index fcb4a82162..47d0c8ee1b 100644 --- a/internal/store/postgres/job/job_repository.go +++ b/internal/store/postgres/job/job_repository.go @@ -437,11 +437,9 @@ func (JobRepository) toUpstreams(storeUpstreams []*JobWithUpstream) ([]*job.Upst var upstreams []*job.Upstream for _, storeUpstream := range storeUpstreams { - var resourceURN resource.URN + var resourceURN job.ResourceURN if storeUpstream.UpstreamResourceURN.Valid { - if storeUpstream.UpstreamResourceURN.String != "" { - resourceURN, _ = resource.ParseURN(storeUpstream.UpstreamResourceURN.String) - } + resourceURN = job.ResourceURN(storeUpstream.UpstreamResourceURN.String) } var upstreamName job.Name @@ -550,12 +548,12 @@ func (j JobRepository) GetAllByProjectName(ctx context.Context, projectName tena return jobs, me.ToErr() } -func (j JobRepository) GetAllByResourceDestination(ctx context.Context, resourceDestination resource.URN) ([]*job.Job, error) { +func (j JobRepository) GetAllByResourceDestination(ctx context.Context, resourceDestination job.ResourceURN) ([]*job.Job, error) { me := errors.NewMultiError("get all job specs by resource destination") getAllByDestination := `SELECT ` + jobColumns + ` FROM job WHERE destination = $1 AND deleted_at IS NULL;` - rows, err := j.db.Query(ctx, getAllByDestination, resourceDestination.String()) + rows, err := j.db.Query(ctx, getAllByDestination, resourceDestination) if err != nil { return nil, errors.Wrap(job.EntityJob, "error while jobs for destination: "+resourceDestination.String(), err) } @@ -592,24 +590,11 @@ func specToJob(spec *Spec) (*job.Job, error) { return nil, err } - var destination resource.URN - if spec.Destination != "" { - destination, err = resource.ParseURN(spec.Destination) - if err != nil { - return nil, err - } - } + destination := job.ResourceURN(spec.Destination) - var sources []resource.URN + var sources []job.ResourceURN for _, source := range spec.Sources { - var resourceURN resource.URN - if source != "" { - resourceURN, err = resource.ParseURN(source) - if err != nil { - return nil, err - } - } - + resourceURN := job.ResourceURN(source) sources = append(sources, resourceURN) } @@ -887,7 +872,7 @@ WHERE project_name=$1 AND job_name=$2;` return j.toUpstreams(storeJobsWithUpstreams) } -func (j JobRepository) GetDownstreamByDestination(ctx context.Context, projectName tenant.ProjectName, destination resource.URN) ([]*job.Downstream, error) { +func (j JobRepository) GetDownstreamByDestination(ctx context.Context, projectName tenant.ProjectName, destination job.ResourceURN) ([]*job.Downstream, error) { query := ` SELECT name as job_name, project_name, namespace_name, task_name @@ -895,7 +880,7 @@ FROM job WHERE project_name = $1 AND $2 = ANY(sources) AND deleted_at IS NULL;` - rows, err := j.db.Query(ctx, query, projectName, destination.String()) + rows, err := j.db.Query(ctx, query, projectName, destination) if err != nil { return nil, errors.Wrap(job.EntityJob, "error while getting job downstream", err) } @@ -978,7 +963,7 @@ func fromStoreDownstream(storeDownstreamList []Downstream) ([]*job.Downstream, e return downstreamList, me.ToErr() } -func (j JobRepository) GetDownstreamBySources(ctx context.Context, sources []resource.URN) ([]*job.Downstream, error) { +func (j JobRepository) GetDownstreamBySources(ctx context.Context, sources []job.ResourceURN) ([]*job.Downstream, error) { if len(sources) == 0 { return nil, nil } diff --git a/internal/store/postgres/job/job_repository_test.go b/internal/store/postgres/job/job_repository_test.go index c84afab9f0..ba1afdb2fd 100644 --- a/internal/store/postgres/job/job_repository_test.go +++ b/internal/store/postgres/job/job_repository_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/lib/window" "github.com/goto/optimus/internal/models" @@ -58,31 +57,6 @@ func TestPostgresJobRepository(t *testing.T) { sampleTenant, err := tenant.NewTenant(proj.Name().String(), namespace.Name().String()) assert.NoError(t, err) - resourceURNA, err := resource.ParseURN("store://dev.resource.sample_a") - assert.NoError(t, err) - resourceURNB, err := resource.ParseURN("store://dev.resource.sample_b") - assert.NoError(t, err) - resourceURNC, err := resource.ParseURN("store://dev.resource.sample_c") - assert.NoError(t, err) - resourceURND, err := resource.ParseURN("store://dev.resource.sample_d") - assert.NoError(t, err) - resourceURNE, err := resource.ParseURN("store://dev.resource.sample_e") - assert.NoError(t, err) - resourceURNF, err := resource.ParseURN("store://dev.resource.sample_f") - assert.NoError(t, err) - resourceURNG, err := resource.ParseURN("store://dev.resource.sample_g") - assert.NoError(t, err) - - resourceURNX, err := resource.ParseURN("store://dev.resource.sample_x") - assert.NoError(t, err) - resourceURNY, err := resource.ParseURN("store://dev.resource.sample_y") - assert.NoError(t, err) - - resourceURN3, err := resource.ParseURN("store://dev.resource.sample_3") - assert.NoError(t, err) - resourceURN4, err := resource.ParseURN("store://dev.resource.sample_4") - assert.NoError(t, err) - dbSetup := func() *pgxpool.Pool { pool := setup.TestPool() setup.TruncateTablesWith(pool) @@ -162,7 +136,7 @@ func TestPostgresJobRepository(t *testing.T) { WithMetadata(jobMetadata). Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask). WithDescription(jobDescription). @@ -173,7 +147,7 @@ func TestPostgresJobRepository(t *testing.T) { WithMetadata(jobMetadata). Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobs := []*job.Job{jobA, jobB} @@ -191,7 +165,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobs := []*job.Job{jobA} @@ -205,7 +179,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA}) @@ -213,7 +187,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURN3}, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", []job.ResourceURN{"resource-3"}, false) addedJobs, err := jobRepo.Add(ctx, []*job.Job{jobA, jobB}) assert.ErrorContains(t, err, "already exists") @@ -224,11 +198,11 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURN3}, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", []job.ResourceURN{"resource-3"}, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB}) @@ -279,7 +253,7 @@ func TestPostgresJobRepository(t *testing.T) { WithMetadata(jobMetadata). Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask). WithDescription(jobDescription). @@ -290,7 +264,7 @@ func TestPostgresJobRepository(t *testing.T) { WithMetadata(jobMetadata). Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobs := []*job.Job{jobA, jobB} @@ -347,7 +321,7 @@ func TestPostgresJobRepository(t *testing.T) { WithMetadata(jobMetadata). Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobs := []*job.Job{jobA} @@ -362,7 +336,7 @@ func TestPostgresJobRepository(t *testing.T) { otherTenant, err := tenant.NewTenant(proj.Name().String(), otherNamespace.Name().String()) assert.NoError(t, err) - jobAToReAdd := job.NewJob(otherTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobAToReAdd := job.NewJob(otherTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) addedJobs, err = jobRepo.Add(ctx, []*job.Job{jobAToReAdd}) assert.ErrorContains(t, err, "already exists and soft deleted in namespace test-ns") assert.Nil(t, addedJobs) @@ -375,11 +349,11 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobs := []*job.Job{jobA, jobB} @@ -393,8 +367,8 @@ func TestPostgresJobRepository(t *testing.T) { Build() assert.NoError(t, err) - jobAToUpdate := job.NewJob(sampleTenant, jobSpecAToUpdate, resourceURNA, []resource.URN{resourceURN3}, false) - jobBToUpdate := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURN4}, false) + jobAToUpdate := job.NewJob(sampleTenant, jobSpecAToUpdate, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) + jobBToUpdate := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", []job.ResourceURN{"resource-4"}, false) jobsToUpdate := []*job.Job{jobAToUpdate, jobBToUpdate} updatedJobs, err := jobRepo.Update(ctx, jobsToUpdate) @@ -406,7 +380,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA}) @@ -416,11 +390,11 @@ func TestPostgresJobRepository(t *testing.T) { WithDescription(jobDescription). Build() assert.NoError(t, err) - jobAToUpdate := job.NewJob(sampleTenant, jobSpecAToUpdate, resourceURNA, []resource.URN{resourceURN3}, false) + jobAToUpdate := job.NewJob(sampleTenant, jobSpecAToUpdate, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).Build() assert.NoError(t, err) - jobBToUpdate := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURN4}, false) + jobBToUpdate := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", []job.ResourceURN{"resource-4"}, false) jobsToUpdate := []*job.Job{jobAToUpdate, jobBToUpdate} updatedJobs, err := jobRepo.Update(ctx, jobsToUpdate) @@ -432,11 +406,11 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURN3}, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", []job.ResourceURN{"resource-3"}, false) jobRepo := postgres.NewJobRepository(db) addedJobs, err := jobRepo.Update(ctx, []*job.Job{jobA, jobB}) @@ -448,7 +422,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobs := []*job.Job{jobA} @@ -466,7 +440,7 @@ func TestPostgresJobRepository(t *testing.T) { otherTenant, err := tenant.NewTenant(proj.Name().String(), otherNamespace.Name().String()) assert.NoError(t, err) - jobToUpdate := job.NewJob(otherTenant, jobSpecA, resource.ZeroURN(), nil, false) + jobToUpdate := job.NewJob(otherTenant, jobSpecA, "", nil, false) updatedJobs, err = jobRepo.Update(ctx, []*job.Job{jobToUpdate}) assert.ErrorContains(t, err, "already exists and soft deleted in namespace test-ns") assert.Nil(t, updatedJobs) @@ -476,7 +450,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobs := []*job.Job{jobA} @@ -487,7 +461,7 @@ func TestPostgresJobRepository(t *testing.T) { otherTenant, err := tenant.NewTenant(proj.Name().String(), otherNamespace.Name().String()) assert.NoError(t, err) - jobAToUpdate := job.NewJob(otherTenant, jobSpecA, resourceURNA, []resource.URN{resourceURN3}, false) + jobAToUpdate := job.NewJob(otherTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) updatedJobs, err := jobRepo.Update(ctx, []*job.Job{jobAToUpdate}) assert.ErrorContains(t, err, "job sample-job-A already exists in namespace test-ns") @@ -504,11 +478,11 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_b"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB}) @@ -532,11 +506,11 @@ func TestPostgresJobRepository(t *testing.T) { WithDescription(jobDescription). WithSpecUpstream(jobAUpstream). Build() - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, nil, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", nil, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB}) @@ -560,15 +534,15 @@ func TestPostgresJobRepository(t *testing.T) { WithDescription(jobDescription). WithSpecUpstream(jobAUpstream). Build() - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_c"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobSpecC, err := job.NewSpecBuilder(jobVersion, "sample-job-C", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobC := job.NewJob(sampleTenant, jobSpecC, resourceURNC, nil, false) + jobC := job.NewJob(sampleTenant, jobSpecC, "dev.resource.sample_c", nil, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB, jobC}) @@ -600,21 +574,21 @@ func TestPostgresJobRepository(t *testing.T) { WithDescription(jobDescription). WithSpecUpstream(jobAUpstream). Build() - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC, resourceURNE}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_c", "dev.resource.sample_e"}, false) // internal project, same server jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobSpecC, err := job.NewSpecBuilder(jobVersion, "sample-job-C", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobC := job.NewJob(sampleTenant, jobSpecC, resourceURNC, nil, false) + jobC := job.NewJob(sampleTenant, jobSpecC, "dev.resource.sample_c", nil, false) // external project, same server jobSpecD, _ := job.NewSpecBuilder(jobVersion, "sample-job-D", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() - jobD := job.NewJob(otherTenant, jobSpecD, resourceURND, nil, false) + jobD := job.NewJob(otherTenant, jobSpecD, "dev.resource.sample_d", nil, false) jobSpecE, _ := job.NewSpecBuilder(jobVersion, "sample-job-E", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() - jobE := job.NewJob(otherTenant, jobSpecE, resourceURNE, nil, false) + jobE := job.NewJob(otherTenant, jobSpecE, "dev.resource.sample_e", nil, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB, jobC, jobD, jobE}) @@ -648,11 +622,11 @@ func TestPostgresJobRepository(t *testing.T) { WithDescription(jobDescription). WithSpecUpstream(jobAUpstream). Build() - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_b"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB}) @@ -673,21 +647,21 @@ func TestPostgresJobRepository(t *testing.T) { t.Run("ReplaceUpstreams", func(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_c"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobSpecC, err := job.NewSpecBuilder(jobVersion, "sample-job-C", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobC := job.NewJob(sampleTenant, jobSpecC, resourceURNC, nil, false) + jobC := job.NewJob(sampleTenant, jobSpecC, "dev.resource.sample_c", nil, false) t.Run("inserts job upstreams", func(t *testing.T) { db := dbSetup() - upstreamB := job.NewUpstreamResolved("jobB", host, resourceURNB, sampleTenant, upstreamType, taskName, false) - upstreamC := job.NewUpstreamResolved("jobC", host, resourceURNC, sampleTenant, upstreamType, taskName, false) + upstreamB := job.NewUpstreamResolved("jobB", host, "resource-B", sampleTenant, upstreamType, taskName, false) + upstreamC := job.NewUpstreamResolved("jobC", host, "resource-C", sampleTenant, upstreamType, taskName, false) upstreams := []*job.Upstream{upstreamB, upstreamC} jobWithUpstream := job.NewWithUpstream(jobA, upstreams) @@ -700,9 +674,9 @@ func TestPostgresJobRepository(t *testing.T) { t.Run("inserts job upstreams including unresolved upstreams", func(t *testing.T) { db := dbSetup() - upstreamB := job.NewUpstreamResolved("jobB", host, resourceURNB, sampleTenant, upstreamType, taskName, false) - upstreamC := job.NewUpstreamResolved("jobC", host, resourceURNC, sampleTenant, upstreamType, taskName, false) - upstreamD := job.NewUpstreamUnresolvedInferred(resourceURND) + upstreamB := job.NewUpstreamResolved("jobB", host, "resource-B", sampleTenant, upstreamType, taskName, false) + upstreamC := job.NewUpstreamResolved("jobC", host, "resource-C", sampleTenant, upstreamType, taskName, false) + upstreamD := job.NewUpstreamUnresolvedInferred("resource-D") upstreams := []*job.Upstream{upstreamB, upstreamC, upstreamD} jobWithUpstream := job.NewWithUpstream(jobA, upstreams) @@ -714,8 +688,8 @@ func TestPostgresJobRepository(t *testing.T) { t.Run("deletes existing job upstream and inserts", func(t *testing.T) { db := dbSetup() - upstreamB := job.NewUpstreamResolved("jobB", host, resourceURNB, sampleTenant, upstreamType, taskName, false) - upstreamC := job.NewUpstreamResolved("jobC", host, resourceURNC, sampleTenant, upstreamType, taskName, false) + upstreamB := job.NewUpstreamResolved("jobB", host, "resource-B", sampleTenant, upstreamType, taskName, false) + upstreamC := job.NewUpstreamResolved("jobC", host, "resource-C", sampleTenant, upstreamType, taskName, false) jobUpstreamRepo := postgres.NewJobRepository(db) _, err = jobUpstreamRepo.Add(ctx, []*job.Job{jobA, jobB, jobC}) @@ -738,7 +712,7 @@ func TestPostgresJobRepository(t *testing.T) { t.Run("deletes existing job upstream without inserts if no longer upstream found", func(t *testing.T) { db := dbSetup() - upstreamB := job.NewUpstreamResolved("jobB", host, resourceURNB, sampleTenant, upstreamType, taskName, false) + upstreamB := job.NewUpstreamResolved("jobB", host, "resource-B", sampleTenant, upstreamType, taskName, false) jobUpstreamRepo := postgres.NewJobRepository(db) _, err = jobUpstreamRepo.Add(ctx, []*job.Job{jobA, jobB, jobC}) @@ -761,8 +735,8 @@ func TestPostgresJobRepository(t *testing.T) { t.Run("inserts job upstreams with exact name across projects exists", func(t *testing.T) { db := dbSetup() - upstreamB := job.NewUpstreamResolved("jobB", host, resourceURNB, sampleTenant, upstreamType, taskName, false) - upstreamC := job.NewUpstreamResolved("jobC", host, resourceURNC, sampleTenant, upstreamType, taskName, false) + upstreamB := job.NewUpstreamResolved("jobB", host, "resource-B", sampleTenant, upstreamType, taskName, false) + upstreamC := job.NewUpstreamResolved("jobC", host, "resource-C", sampleTenant, upstreamType, taskName, false) upstreams := []*job.Upstream{upstreamB, upstreamC} jobWithUpstream := job.NewWithUpstream(jobA, upstreams) @@ -773,8 +747,8 @@ func TestPostgresJobRepository(t *testing.T) { otherTenant, err := tenant.NewTenant(otherProj.Name().String(), otherNamespace.Name().String()) assert.NoError(t, err) - otherProjectJobA := job.NewJob(otherTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC}, false) - otherProjectJobB := job.NewJob(otherTenant, jobSpecB, resourceURNB, nil, false) + otherProjectJobA := job.NewJob(otherTenant, jobSpecA, "dev-external.resource.sample_a", []job.ResourceURN{"dev-external.resource.sample_c"}, false) + otherProjectJobB := job.NewJob(otherTenant, jobSpecB, "dev-external.resource.sample_b", nil, false) _, err = jobUpstreamRepo.Add(ctx, []*job.Job{otherProjectJobA, otherProjectJobB}) assert.NoError(t, err) @@ -785,11 +759,11 @@ func TestPostgresJobRepository(t *testing.T) { newTenant, _ := tenant.NewTenant(otherNamespace.ProjectName().String(), otherNamespace.Name().String()) jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_c"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURNA}, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", []job.ResourceURN{"dev.resource.sample_a"}, false) t.Run("Change Job namespace successfully", func(t *testing.T) { db := dbSetup() @@ -799,7 +773,7 @@ func TestPostgresJobRepository(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, addedJob) - upstreamAInferred := job.NewUpstreamResolved("sample-job-A", "host-1", resourceURNA, sampleTenant, "inferred", taskName, false) + upstreamAInferred := job.NewUpstreamResolved("sample-job-A", "host-1", "dev.resource.sample_a", sampleTenant, "inferred", taskName, false) jobBWithUpstream := job.NewWithUpstream(jobB, []*job.Upstream{upstreamAInferred}) err = jobRepo.ReplaceUpstreams(ctx, []*job.WithUpstream{jobBWithUpstream}) assert.NoError(t, err) @@ -831,7 +805,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, nil, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", nil, false) jobRepo := postgres.NewJobRepository(db) @@ -862,7 +836,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, nil, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", nil, false) jobRepo := postgres.NewJobRepository(db) @@ -893,11 +867,11 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNB}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_b"}, false) jobSpecX, err := job.NewSpecBuilder(jobVersion, "sample-job-X", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobX := job.NewJob(sampleTenant, jobSpecX, resourceURNX, []resource.URN{resourceURNA}, false) + jobX := job.NewJob(sampleTenant, jobSpecX, "dev.resource.sample_x", []job.ResourceURN{"dev.resource.sample_a"}, false) jobRepo := postgres.NewJobRepository(db) @@ -905,9 +879,9 @@ func TestPostgresJobRepository(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, addedJob) - upstreamBInferred := job.NewUpstreamResolved("sample-job-B", "host-1", resourceURNB, sampleTenant, "inferred", taskName, false) + upstreamBInferred := job.NewUpstreamResolved("sample-job-B", "host-1", "dev.resource.sample_b", sampleTenant, "inferred", taskName, false) jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{upstreamBInferred}) - upstreamAInferred := job.NewUpstreamResolved("sample-job-A", "host-1", resourceURNA, sampleTenant, "inferred", taskName, false) + upstreamAInferred := job.NewUpstreamResolved("sample-job-A", "host-1", "dev.resource.sample_a", sampleTenant, "inferred", taskName, false) jobXWithUpstream := job.NewWithUpstream(jobX, []*job.Upstream{upstreamAInferred}) err = jobRepo.ReplaceUpstreams(ctx, []*job.WithUpstream{jobAWithUpstream, jobXWithUpstream}) @@ -938,7 +912,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNB, resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA}) @@ -954,7 +928,7 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNB, resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA}) @@ -980,10 +954,10 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNB, resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURNC}, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", []job.ResourceURN{"dev.resource.sample_c"}, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB}) @@ -1000,10 +974,10 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNB, resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, []resource.URN{resourceURNC}, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", []job.ResourceURN{"dev.resource.sample_c"}, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB}) @@ -1024,16 +998,16 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNY, []resource.URN{resourceURNB, resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_general", []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNY, []resource.URN{resourceURNC}, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_general", []job.ResourceURN{"dev.resource.sample_c"}, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB}) assert.NoError(t, err) - actual, err := jobRepo.GetAllByResourceDestination(ctx, resourceURNY) + actual, err := jobRepo.GetAllByResourceDestination(ctx, "dev.resource.sample_general") assert.NoError(t, err) assert.NotNil(t, actual) assert.Len(t, actual, 2) @@ -1044,10 +1018,10 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNY, []resource.URN{resourceURNB, resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_general", []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNY, []resource.URN{resourceURNC}, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_general", []job.ResourceURN{"dev.resource.sample_c"}, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB}) @@ -1056,7 +1030,7 @@ func TestPostgresJobRepository(t *testing.T) { err = jobRepo.Delete(ctx, sampleTenant.ProjectName(), jobSpecB.Name(), false) assert.NoError(t, err) - actual, err := jobRepo.GetAllByResourceDestination(ctx, resourceURNY) + actual, err := jobRepo.GetAllByResourceDestination(ctx, "dev.resource.sample_general") assert.NoError(t, err) assert.Equal(t, []*job.Job{jobA}, actual) }) @@ -1069,13 +1043,13 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNY, []resource.URN{resourceURNB, resourceURNC}, false) - jobAUpstreamResolved := job.NewUpstreamResolved("sample-job-B", "", resource.ZeroURN(), sampleTenant, "inferred", taskName, false) - jobAUpstreamUnresolved := job.NewUpstreamUnresolvedInferred(resourceURNC) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_general", []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, false) + jobAUpstreamResolved := job.NewUpstreamResolved("sample-job-B", "", "", sampleTenant, "inferred", taskName, false) + jobAUpstreamUnresolved := job.NewUpstreamUnresolvedInferred("dev.resource.sample_c") jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{jobAUpstreamResolved, jobAUpstreamUnresolved}) @@ -1100,15 +1074,15 @@ func TestPostgresJobRepository(t *testing.T) { jobAUpstreamSpec, _ := job.NewSpecUpstreamBuilder().WithUpstreamNames([]job.SpecUpstreamName{"sample-job-B"}).Build() jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).WithSpecUpstream(jobAUpstreamSpec).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_c"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobSpecC, err := job.NewSpecBuilder(jobVersion, "sample-job-C", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobC := job.NewJob(sampleTenant, jobSpecC, resourceURNC, nil, false) + jobC := job.NewJob(sampleTenant, jobSpecC, "dev.resource.sample_c", nil, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB, jobC}) @@ -1117,7 +1091,7 @@ func TestPostgresJobRepository(t *testing.T) { expectedDownstream := []*job.Downstream{ job.NewDownstream(jobSpecA.Name(), proj.Name(), namespace.Name(), jobSpecA.Task().Name()), } - result, err := jobRepo.GetDownstreamByDestination(ctx, proj.Name(), resourceURNC) + result, err := jobRepo.GetDownstreamByDestination(ctx, proj.Name(), "dev.resource.sample_c") assert.NoError(t, err) assert.EqualValues(t, expectedDownstream, result) }) @@ -1129,15 +1103,15 @@ func TestPostgresJobRepository(t *testing.T) { jobSpecA, err := job.NewSpecBuilder(jobVersion, "sample-job-A", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNB, resourceURNC}, false) - jobAUpstreamResolved := job.NewUpstreamResolved("sample-job-B", "", resource.ZeroURN(), sampleTenant, "inferred", taskName, false) - jobAUpstreamUnresolved := job.NewUpstreamUnresolvedInferred(resourceURNC) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, false) + jobAUpstreamResolved := job.NewUpstreamResolved("sample-job-B", "", "", sampleTenant, "inferred", taskName, false) + jobAUpstreamUnresolved := job.NewUpstreamUnresolvedInferred("dev.resource.sample_c") jobAWithUpstream := job.NewWithUpstream(jobA, []*job.Upstream{jobAUpstreamResolved, jobAUpstreamUnresolved}) jobSpecB, err := job.NewSpecBuilder(jobVersion, "sample-job-B", jobOwner, jobSchedule, customConfig, jobTask).WithDescription(jobDescription).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB}) @@ -1159,7 +1133,7 @@ func TestPostgresJobRepository(t *testing.T) { db := dbSetup() jobRepo := postgres.NewJobRepository(db) - var sources []resource.URN + var sources []job.ResourceURN result, err := jobRepo.GetDownstreamBySources(ctx, sources) assert.NoError(t, err) @@ -1172,17 +1146,17 @@ func TestPostgresJobRepository(t *testing.T) { jobAName, _ := job.NameFrom("sample-job-A") jobSpecA, err := job.NewSpecBuilder(jobVersion, jobAName, jobOwner, jobSchedule, customConfig, jobTask).Build() assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{resourceURNB, resourceURNC}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, false) jobBName, _ := job.NameFrom("sample-job-b") jobSpecB, err := job.NewSpecBuilder(jobVersion, jobBName, jobOwner, jobSchedule, customConfig, jobTask).Build() assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURND, []resource.URN{resourceURNE}, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_d", []job.ResourceURN{"dev.resource.sample_e"}, false) jobCName, _ := job.NameFrom("sample-job-c") jobSpecC, err := job.NewSpecBuilder(jobVersion, jobCName, jobOwner, jobSchedule, customConfig, jobTask).Build() assert.NoError(t, err) - jobC := job.NewJob(sampleTenant, jobSpecC, resourceURNF, nil, false) + jobC := job.NewJob(sampleTenant, jobSpecC, "dev.resource.sample_f", nil, false) jobRepo := postgres.NewJobRepository(db) _, err = jobRepo.Add(ctx, []*job.Job{jobA, jobB, jobC}) @@ -1192,27 +1166,27 @@ func TestPostgresJobRepository(t *testing.T) { jobBAsDownstream := job.NewDownstream(jobBName, sampleTenant.ProjectName(), sampleTenant.NamespaceName(), jobTask.Name()) testCases := []struct { - sources []resource.URN + sources []job.ResourceURN expectedDownstreams []*job.Downstream }{ { - sources: []resource.URN{resourceURNB}, + sources: []job.ResourceURN{"dev.resource.sample_b"}, expectedDownstreams: []*job.Downstream{jobAAsDownstream}, }, { - sources: []resource.URN{resourceURNB, resourceURNE}, + sources: []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_e"}, expectedDownstreams: []*job.Downstream{jobAAsDownstream, jobBAsDownstream}, }, { - sources: []resource.URN{resourceURNB, resourceURNC}, + sources: []job.ResourceURN{"dev.resource.sample_b", "dev.resource.sample_c"}, expectedDownstreams: []*job.Downstream{jobAAsDownstream}, }, { - sources: []resource.URN{resourceURNE, resourceURNF}, + sources: []job.ResourceURN{"dev.resource.sample_e", "dev.resource.sample_f"}, expectedDownstreams: []*job.Downstream{jobBAsDownstream}, }, { - sources: []resource.URN{resourceURNF, resourceURNG}, + sources: []job.ResourceURN{"dev.resource.sample_f", "dev.resource.sample_g"}, expectedDownstreams: nil, }, } diff --git a/internal/store/postgres/resource/repository_test.go b/internal/store/postgres/resource/repository_test.go index be151eb662..072df7bc39 100644 --- a/internal/store/postgres/resource/repository_test.go +++ b/internal/store/postgres/resource/repository_test.go @@ -34,9 +34,6 @@ func TestPostgresResourceRepository(t *testing.T) { store := serviceResource.Bigquery kindDataset := "dataset" - resourceURN, err := serviceResource.ParseURN("bigquery://project:dataset") - assert.NoError(t, err) - t.Run("Create", func(t *testing.T) { t.Run("returns error if resource with the provided full name is already defined within project and namespace", func(t *testing.T) { pool := dbSetup() @@ -44,7 +41,7 @@ func TestPostgresResourceRepository(t *testing.T) { resourceToCreate, err := serviceResource.NewResource("project.dataset", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceToCreate.UpdateURN(resourceURN) + resourceToCreate.UpdateURN("bigquery://project:dataset") actualFirstError := repository.Create(ctx, resourceToCreate) assert.NoError(t, actualFirstError) @@ -58,9 +55,7 @@ func TestPostgresResourceRepository(t *testing.T) { resourceToCreate, err := serviceResource.NewResource("project.dataset", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceURN, err := serviceResource.ParseURN("bigquery://project:dataset") - assert.NoError(t, err) - err = resourceToCreate.UpdateURN(resourceURN) + err = resourceToCreate.UpdateURN("bigquery://project:dataset") assert.NoError(t, err) actualError := repository.Create(ctx, resourceToCreate) @@ -79,7 +74,7 @@ func TestPostgresResourceRepository(t *testing.T) { resourceToUpdate, err := serviceResource.NewResource("project.dataset", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceToUpdate.UpdateURN(resourceURN) + resourceToUpdate.UpdateURN("bigquery://project:dataset") actualError := repository.Update(ctx, resourceToUpdate) assert.ErrorContains(t, actualError, "not found for entity resource") @@ -91,7 +86,7 @@ func TestPostgresResourceRepository(t *testing.T) { resourceToCreate, err := serviceResource.NewResource("project.dataset", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceToCreate.UpdateURN(resourceURN) + resourceToCreate.UpdateURN("bigquery://project:dataset") err = repository.Create(ctx, resourceToCreate) assert.NoError(t, err) @@ -117,7 +112,7 @@ func TestPostgresResourceRepository(t *testing.T) { resourceToUpdate, err := serviceResource.NewResource("project.dataset", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceToUpdate.UpdateURN(resourceURN) + resourceToUpdate.UpdateURN("bigquery://project:dataset") actualError := repository.ChangeNamespace(ctx, resourceToUpdate, newTenant) assert.ErrorContains(t, actualError, "not found for entity resource") @@ -129,7 +124,7 @@ func TestPostgresResourceRepository(t *testing.T) { resourceToCreate, err := serviceResource.NewResource("project.dataset", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceToCreate.UpdateURN(resourceURN) + resourceToCreate.UpdateURN("bigquery://project:dataset") err = repository.Create(ctx, resourceToCreate) assert.NoError(t, err) @@ -150,18 +145,15 @@ func TestPostgresResourceRepository(t *testing.T) { pool := dbSetup() repository := repoResource.NewRepository(pool) - urn, err := serviceResource.ParseURN("bigquery://project:dataset") - assert.NoError(t, err) - resourceExisting, err := serviceResource.NewResource("project.dataset", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceExisting.UpdateURN(urn) + resourceExisting.UpdateURN("bigquery://project:dataset") otherTnnt, err := tenant.NewTenant(tnnt.ProjectName().String(), "n-optimus-2") assert.NoError(t, err) resourceToChange, err := serviceResource.NewResource("project.dataset", kindDataset, store, otherTnnt, meta, spec) assert.NoError(t, err) - resourceToChange.UpdateURN(urn) + resourceToChange.UpdateURN("bigquery://project:dataset") assert.NoError(t, repository.Create(ctx, resourceExisting), "failed create resource") assert.NoError(t, repository.Delete(ctx, resourceExisting), "failed delete resource") @@ -242,9 +234,7 @@ func TestPostgresResourceRepository(t *testing.T) { name1 := "project.dataset" resourceToCreate1, err := serviceResource.NewResource(name1, kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceURN, err := serviceResource.ParseURN("bigquery://project:dataset1") - assert.NoError(t, err) - resourceToCreate1.UpdateURN(resourceURN) + resourceToCreate1.UpdateURN("bigquery://project:dataset1") err = repository.Create(ctx, resourceToCreate1) assert.NoError(t, err) @@ -255,9 +245,7 @@ func TestPostgresResourceRepository(t *testing.T) { name2 := "project.dataset.view" resourceToCreate2, err := serviceResource.NewResource(name2, "view", store, tnnt, meta, viewSpec) assert.NoError(t, err) - resourceURN, err = serviceResource.ParseURN("bigquery://project:dataset.view") - assert.NoError(t, err) - resourceToCreate2.UpdateURN(resourceURN) + resourceToCreate2.UpdateURN("bigquery://project:dataset.view") err = repository.Create(ctx, resourceToCreate2) assert.NoError(t, err) @@ -301,14 +289,10 @@ func TestPostgresResourceRepository(t *testing.T) { existingResource1, err := serviceResource.NewResource("project.dataset1", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceURN, err := serviceResource.ParseURN("bigquery://project:dataset1") - assert.NoError(t, err) - existingResource1.UpdateURN(resourceURN) + existingResource1.UpdateURN("bigquery://project:dataset1") existingResource2, err := serviceResource.NewResource("project.dataset2", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceURN, err = serviceResource.ParseURN("bigquery://project:dataset2") - assert.NoError(t, err) - existingResource2.UpdateURN(resourceURN) + existingResource2.UpdateURN("bigquery://project:dataset2") err = repository.Create(ctx, existingResource1) assert.NoError(t, err) err = repository.Create(ctx, existingResource2) @@ -356,12 +340,9 @@ func TestPostgresResourceRepository(t *testing.T) { pool := dbSetup() repository := repoResource.NewRepository(pool) - urn, err := serviceResource.ParseURN("bigquery://project:dataset") - assert.NoError(t, err) - resourceToCreate, err := serviceResource.NewResource("project.dataset", kindDataset, store, tnnt, meta, spec) assert.NoError(t, err) - resourceToCreate.UpdateURN(urn) + resourceToCreate.UpdateURN("bigquery://project:dataset") err = repository.Create(ctx, resourceToCreate) assert.NoError(t, err) diff --git a/internal/store/postgres/resource/resource.go b/internal/store/postgres/resource/resource.go index 333456b0c5..da2ee581f9 100644 --- a/internal/store/postgres/resource/resource.go +++ b/internal/store/postgres/resource/resource.go @@ -43,7 +43,7 @@ func FromResourceToModel(r *resource.Resource) *Resource { NamespaceName: r.Tenant().NamespaceName().String(), Metadata: metadata, Spec: r.Spec(), - URN: r.URN().String(), + URN: r.URN(), Status: r.Status().String(), } } @@ -65,18 +65,7 @@ func FromModelToResource(r *Resource) (*resource.Resource, error) { output, err := resource.NewResource(r.FullName, r.Kind, store, tnnt, metadata, r.Spec) if err == nil { output = resource.FromExisting(output, resource.ReplaceStatus(resource.FromStringToStatus(r.Status))) - - var urn resource.URN - if r.URN != "" { - tempURN, err := resource.ParseURN(r.URN) - if err != nil { - return nil, err - } - - urn = tempURN - } - - output.UpdateURN(urn) + output.UpdateURN(r.URN) } return output, err } diff --git a/internal/store/postgres/scheduler/job_repository.go b/internal/store/postgres/scheduler/job_repository.go index 2f2afa24e5..b934414168 100644 --- a/internal/store/postgres/scheduler/job_repository.go +++ b/internal/store/postgres/scheduler/job_repository.go @@ -14,7 +14,6 @@ import ( "github.com/lib/pq" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/scheduler" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/errors" @@ -74,21 +73,11 @@ func (j *JobUpstreams) toJobUpstreams() (*scheduler.JobUpstream, error) { return nil, err } - var destinationURN resource.URN - if j.UpstreamResourceUrn.String != "" { - tmpURN, err := resource.ParseURN(j.UpstreamResourceUrn.String) - if err != nil { - return nil, err - } - - destinationURN = tmpURN - } - return &scheduler.JobUpstream{ JobName: j.UpstreamJobName.String, Host: j.UpstreamHost.String, TaskName: j.UpstreamTaskName.String, - DestinationURN: destinationURN, + DestinationURN: j.UpstreamResourceUrn.String, Tenant: t, Type: j.UpstreamType, External: j.UpstreamExternal.Bool, @@ -241,19 +230,11 @@ func (j *Job) toJob() (*scheduler.Job, error) { return nil, err } } - var destination resource.URN - if j.Destination != "" { - tempURN, err := resource.ParseURN(j.Destination) - if err != nil { - return nil, err - } - destination = tempURN - } schedulerJob := scheduler.Job{ ID: j.ID, Name: scheduler.JobName(j.Name), Tenant: t, - Destination: destination, + Destination: j.Destination, WindowConfig: w, Assets: j.Assets, Task: &scheduler.Task{ diff --git a/internal/store/postgres/scheduler/job_repository_test.go b/internal/store/postgres/scheduler/job_repository_test.go index 7e388eda2a..91f1c62c12 100644 --- a/internal/store/postgres/scheduler/job_repository_test.go +++ b/internal/store/postgres/scheduler/job_repository_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/scheduler" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/errors" @@ -60,7 +59,7 @@ func TestPostgresJobRepository(t *testing.T) { assert.Equal(t, "resolved", job.Upstreams.UpstreamJobs[0].State) assert.Equal(t, false, job.Upstreams.UpstreamJobs[0].External) assert.Equal(t, "bq2bq", job.Upstreams.UpstreamJobs[0].TaskName) - assert.Equal(t, "store://dev.resource.sample_b", job.Upstreams.UpstreamJobs[0].DestinationURN.String()) + assert.Equal(t, "dev.resource.sample_b", job.Upstreams.UpstreamJobs[0].DestinationURN) } } }) @@ -227,11 +226,7 @@ func addJobs(ctx context.Context, t *testing.T, pool *pgxpool.Pool) map[string]* assert.NoError(t, err) sampleTenant, err := tenant.NewTenant(proj.Name().String(), namespace.Name().String()) assert.NoError(t, err) - resourceURNA, err := resource.ParseURN("store://dev.resource.sample_a") - assert.NoError(t, err) - source, err := resource.ParseURN("store://resource-3") - assert.NoError(t, err) - jobA := job.NewJob(sampleTenant, jobSpecA, resourceURNA, []resource.URN{source}, false) + jobA := job.NewJob(sampleTenant, jobSpecA, "dev.resource.sample_a", []job.ResourceURN{"resource-3"}, false) jobSpecB, err := job.NewSpecBuilder(jobVersion, jobBName, jobOwner, jobSchedule, customConfig, jobTask). WithDescription(jobDescription). @@ -242,9 +237,7 @@ func addJobs(ctx context.Context, t *testing.T, pool *pgxpool.Pool) map[string]* WithMetadata(jobMetadata). Build() assert.NoError(t, err) - resourceURNB, err := resource.ParseURN("store://dev.resource.sample_b") - assert.NoError(t, err) - jobB := job.NewJob(sampleTenant, jobSpecB, resourceURNB, nil, false) + jobB := job.NewJob(sampleTenant, jobSpecB, "dev.resource.sample_b", nil, false) jobs := []*job.Job{jobA, jobB} @@ -254,7 +247,7 @@ func addJobs(ctx context.Context, t *testing.T, pool *pgxpool.Pool) map[string]* assert.Nil(t, err) assert.EqualValues(t, jobs, addedJobs) - upstreamB := job.NewUpstreamResolved(jobBName, "internal", resourceURNB, sampleTenant, job.UpstreamTypeStatic, taskName, false) + upstreamB := job.NewUpstreamResolved(jobBName, "internal", "dev.resource.sample_b", sampleTenant, job.UpstreamTypeStatic, taskName, false) upstreams := []*job.Upstream{upstreamB} jobWithUpstream := job.NewWithUpstream(jobA, upstreams) err = jobRepository.ReplaceUpstreams(ctx, []*job.WithUpstream{jobWithUpstream}) @@ -269,7 +262,7 @@ func addJobs(ctx context.Context, t *testing.T, pool *pgxpool.Pool) map[string]* func compareEqualJob(j *job.Job, s *scheduler.Job) bool { return j.GetName() == s.Name.String() && j.Tenant() == s.Tenant && - j.Destination().String() == s.Destination.String() && + j.Destination().String() == s.Destination && j.Spec().Task().Name().String() == s.Task.Name } diff --git a/internal/writer/logwriter.go b/internal/writer/logwriter.go index 39758d9c6d..4d694441a1 100644 --- a/internal/writer/logwriter.go +++ b/internal/writer/logwriter.go @@ -1,9 +1,6 @@ package writer import ( - "errors" - "sync" - "github.com/goto/salt/log" pb "github.com/goto/optimus/protos/gotocompany/optimus/core/v1beta1" @@ -39,30 +36,10 @@ func (l *saltLogger) Write(level LogLevel, message string) error { type BufferedLogger struct { Messages []*pb.Log - - mtx *sync.Mutex -} - -func NewSafeBufferedLogger() *BufferedLogger { - return &BufferedLogger{ - Messages: nil, - mtx: new(sync.Mutex), - } } // nolint: unparam func (b *BufferedLogger) Write(level LogLevel, message string) error { - if b == nil { - return errors.New("buffered logger is nil") - } - - if b.mtx == nil { - b.mtx = new(sync.Mutex) - } - - b.mtx.Lock() b.Messages = append(b.Messages, newLogStatusProto(level, message)) - b.mtx.Unlock() - return nil } diff --git a/plugin/plugin_service.go b/plugin/plugin_service.go index ae5a782a5c..92fbcffc20 100644 --- a/plugin/plugin_service.go +++ b/plugin/plugin_service.go @@ -9,7 +9,6 @@ import ( "github.com/goto/salt/log" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/internal/errors" upstreamidentifier "github.com/goto/optimus/plugin/upstream_identifier" "github.com/goto/optimus/plugin/upstream_identifier/evaluator" @@ -82,7 +81,7 @@ func (s PluginService) Info(_ context.Context, taskName string) (*plugin.Info, e return taskPlugin.Info(), nil } -func (s PluginService) IdentifyUpstreams(ctx context.Context, taskName string, compiledConfig, assets map[string]string) ([]resource.URN, error) { +func (s PluginService) IdentifyUpstreams(ctx context.Context, taskName string, compiledConfig, assets map[string]string) ([]string, error) { taskPlugin, err := s.pluginGetter.GetByName(taskName) if err != nil { return nil, err @@ -92,7 +91,7 @@ func (s PluginService) IdentifyUpstreams(ctx context.Context, taskName string, c if assetParsers == nil { // if plugin doesn't contain parser, then it doesn't support auto upstream generation s.l.Debug("plugin %s doesn't contain parser, auto upstream generation is not supported.", taskPlugin.Info().Name) - return []resource.URN{}, nil + return []string{}, nil } // construct all possible identifier from given parser @@ -125,7 +124,7 @@ func (s PluginService) IdentifyUpstreams(ctx context.Context, taskName string, c } // identify all upstream resource urns by all identifier from given asset - var resourceURNs []resource.URN + resourceURNs := []string{} me := errors.NewMultiError("identify upstream errors") for _, upstreamIdentifier := range upstreamIdentifiers { currentResourceURNs, err := upstreamIdentifier.IdentifyResources(ctx, assets) @@ -142,8 +141,7 @@ func (s PluginService) IdentifyUpstreams(ctx context.Context, taskName string, c if err != nil { return nil, err } - - filteredResourceURNs := make([]resource.URN, 0) + filteredResourceURNs := []string{} for _, resourceURN := range resourceURNs { if resourceURN == destinationURN { s.l.Warn("ignore destination resource %s", resourceURN) @@ -155,10 +153,10 @@ func (s PluginService) IdentifyUpstreams(ctx context.Context, taskName string, c return filteredResourceURNs, me.ToErr() } -func (s PluginService) ConstructDestinationURN(_ context.Context, taskName string, compiledConfig map[string]string) (resource.URN, error) { +func (s PluginService) ConstructDestinationURN(_ context.Context, taskName string, compiledConfig map[string]string) (string, error) { taskPlugin, err := s.pluginGetter.GetByName(taskName) if err != nil { - return resource.ZeroURN(), err + return "", err } // for now only support single template @@ -166,21 +164,16 @@ func (s PluginService) ConstructDestinationURN(_ context.Context, taskName strin if destinationURNTemplate == "" { // if plugin doesn't contain destination template, then it doesn't support auto destination generation s.l.Debug("plugin %s doesn't contain destination template, auto destination generation is not supported.", taskPlugin.Info().Name) - return resource.ZeroURN(), nil + return "", nil } convertedURNTemplate := convertToGoTemplate(destinationURNTemplate) tmpl, err := template.New("destination_urn_" + taskPlugin.Info().Name).Parse(convertedURNTemplate) if err != nil { - return resource.ZeroURN(), err - } - - rawURN, err := generateResourceURNFromTemplate(tmpl, compiledConfig) - if err != nil { - return resource.ZeroURN(), err + return "", err } - return resource.ParseURN(rawURN) + return generateResourceURNFromTemplate(tmpl, compiledConfig) } // convertToGoTemplate transforms plugin destination urn template format to go template format diff --git a/plugin/plugin_service_test.go b/plugin/plugin_service_test.go index c91edd117e..e79875b371 100644 --- a/plugin/plugin_service_test.go +++ b/plugin/plugin_service_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/plugin" upstreamidentifier "github.com/goto/optimus/plugin/upstream_identifier" "github.com/goto/optimus/plugin/upstream_identifier/evaluator" @@ -23,7 +22,8 @@ func TestNewPluginService(t *testing.T) { var logger log.Logger = nil pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -33,7 +33,8 @@ func TestNewPluginService(t *testing.T) { t.Run("should return error when pluginGetter is nil", func(t *testing.T) { logger := log.NewNoop() var pluginGetter plugin.PluginGetter = nil - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -55,7 +56,8 @@ func TestNewPluginService(t *testing.T) { logger := log.NewNoop() pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) var evaluatorFactory plugin.EvaluatorFactory = nil _, err := plugin.NewPluginService(logger, pluginGetter, upstreamIdentifierFactory, evaluatorFactory) @@ -65,7 +67,8 @@ func TestNewPluginService(t *testing.T) { logger := log.NewNoop() pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -87,7 +90,8 @@ func TestInfo(t *testing.T) { t.Run("returns error when no plugin", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -104,7 +108,8 @@ func TestInfo(t *testing.T) { t.Run("returns error when yaml mod not supported", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -122,7 +127,8 @@ func TestInfo(t *testing.T) { t.Run("returns plugin info", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -159,15 +165,11 @@ func TestIdentifyUpstreams(t *testing.T) { YamlMod: pluginYamlTestWithSelector, } - urn1, err := resource.ParseURN("bigquery://proj:datas.table1") - assert.NoError(t, err) - urn2, err := resource.ParseURN("bigquery://proj:datas.table2") - assert.NoError(t, err) - t.Run("return error when plugin is not exist on pluginGetter", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -183,7 +185,8 @@ func TestIdentifyUpstreams(t *testing.T) { t.Run("return empty resource urn if plugin doesn't have parser", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) evaluator := new(Evaluator) @@ -208,7 +211,8 @@ func TestIdentifyUpstreams(t *testing.T) { t.Run("return error when evaluator factory couldn't return file evaluator", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -225,7 +229,8 @@ func TestIdentifyUpstreams(t *testing.T) { t.Run("return error when evaluator factory couldn't return specilized evaluator", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -242,7 +247,8 @@ func TestIdentifyUpstreams(t *testing.T) { t.Run("return error when evaluator factory couldn't return evaluator due to invalid filepath type", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -261,7 +267,8 @@ func TestIdentifyUpstreams(t *testing.T) { t.Run("return error when bq2bq service account config is not provided", func(t *testing.T) { // will remove once all plugin is supported pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) evaluator := new(Evaluator) @@ -281,7 +288,8 @@ func TestIdentifyUpstreams(t *testing.T) { t.Run("return error when upstream generator can't be created", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) evaluator := new(Evaluator) @@ -301,17 +309,19 @@ func TestIdentifyUpstreams(t *testing.T) { t.Run("should success when no error encountered", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) evaluator := new(Evaluator) defer evaluator.AssertExpectations(t) - upstreamIdentifier := NewUpstreamIdentifier(t) + upstreamIdentifier := new(UpstreamIdentifier) + defer upstreamIdentifier.AssertExpectations(t) pluginGetter.On("GetByName", mock.Anything).Return(pluginTest, nil) evaluatorFactory.On("GetFileEvaluator", mock.Anything).Return(evaluator, nil) upstreamIdentifierFactory.On("GetBQUpstreamIdentifier", ctx, mock.Anything, evaluator).Return(upstreamIdentifier, nil) - upstreamIdentifier.On("IdentifyResources", ctx, assets).Return([]resource.URN{urn1}, nil) + upstreamIdentifier.On("IdentifyResources", ctx, assets).Return([]string{"bigquery://proj:datas:tabl"}, nil) pluginService, err := plugin.NewPluginService(logger, pluginGetter, upstreamIdentifierFactory, evaluatorFactory) assert.NoError(t, err) assert.NotNil(t, pluginService) @@ -324,12 +334,14 @@ func TestIdentifyUpstreams(t *testing.T) { t.Run("should generate clean dependencies without destination in it", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) evaluator := new(Evaluator) defer evaluator.AssertExpectations(t) - upstreamIdentifier := NewUpstreamIdentifier(t) + upstreamIdentifier := new(UpstreamIdentifier) + defer upstreamIdentifier.AssertExpectations(t) pluginYamlTestWithDestinationTemplate, err := yaml.NewPluginSpec("./yaml/tests/sample_plugin_with_parser_and_destination_template.yaml") assert.NoError(t, err) @@ -339,16 +351,16 @@ func TestIdentifyUpstreams(t *testing.T) { pluginGetter.On("GetByName", mock.Anything).Return(pluginTestWithDestinationTemplate, nil) evaluatorFactory.On("GetFileEvaluator", mock.Anything).Return(evaluator, nil) upstreamIdentifierFactory.On("GetBQUpstreamIdentifier", ctx, mock.Anything, evaluator).Return(upstreamIdentifier, nil) - upstreamIdentifier.On("IdentifyResources", ctx, assets).Return([]resource.URN{urn1, urn2}, nil) + upstreamIdentifier.On("IdentifyResources", ctx, assets).Return([]string{"bigquery://proj:datas:tabl", "bigquery://projectA:datasetB.tableC"}, nil) pluginService, err := plugin.NewPluginService(logger, pluginGetter, upstreamIdentifierFactory, evaluatorFactory) assert.NoError(t, err) assert.NotNil(t, pluginService) configTask := map[string]string{} configTask["BQ_SERVICE_ACCOUNT"] = "service_account_value" - configTask["PROJECT"] = "proj" - configTask["DATASET"] = "datas" - configTask["TABLE"] = "table2" + configTask["PROJECT"] = "projectA" + configTask["DATASET"] = "datasetB" + configTask["TABLE"] = "tableC" resourceURNs, err := pluginService.IdentifyUpstreams(ctx, taskName, configTask, assets) assert.NoError(t, err) assert.NotEmpty(t, resourceURNs) @@ -374,7 +386,8 @@ func TestConstructDestinationURN(t *testing.T) { t.Run("returns error if unable to find the plugin", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -390,7 +403,8 @@ func TestConstructDestinationURN(t *testing.T) { t.Run("should return empty destination if the plugin doesn't contain destination template", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -411,7 +425,8 @@ func TestConstructDestinationURN(t *testing.T) { t.Run("returns error if template is not proper", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) pluginYamlTest, err := yaml.NewPluginSpec("./yaml/tests/sample_plugin_with_unproper_destination_template.yaml") @@ -431,7 +446,8 @@ func TestConstructDestinationURN(t *testing.T) { t.Run("should properly generate a destination provided correct config inputs", func(t *testing.T) { pluginGetter := new(PluginGetter) defer pluginGetter.AssertExpectations(t) - upstreamIdentifierFactory := NewUpstreamIdentifierFactory(t) + upstreamIdentifierFactory := new(UpstreamIdentifierFactory) + defer upstreamIdentifierFactory.AssertExpectations(t) evaluatorFactory := new(EvaluatorFactory) defer evaluatorFactory.AssertExpectations(t) @@ -440,13 +456,10 @@ func TestConstructDestinationURN(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, pluginService) - expectedURN, err := resource.ParseURN("bigquery://project1:dataset1.table1") - assert.NoError(t, err) - result, err := pluginService.ConstructDestinationURN(ctx, taskName, config) assert.NoError(t, err) assert.NotEmpty(t, result) - assert.Equal(t, expectedURN, result) + assert.Equal(t, "bigquery://project1:dataset1.table1", result) }) } @@ -516,10 +529,6 @@ func (_m *UpstreamIdentifierFactory) GetBQUpstreamIdentifier(ctx context.Context _ca = append(_ca, _va...) ret := _m.Called(_ca...) - if len(ret) == 0 { - panic("no return value specified for GetBQUpstreamIdentifier") - } - var r0 upstreamidentifier.UpstreamIdentifier var r1 error if rf, ok := ret.Get(0).(func(context.Context, string, ...evaluator.Evaluator) (upstreamidentifier.UpstreamIdentifier, error)); ok { @@ -542,21 +551,6 @@ func (_m *UpstreamIdentifierFactory) GetBQUpstreamIdentifier(ctx context.Context return r0, r1 } -// NewUpstreamIdentifierFactory creates a new instance of UpstreamIdentifierFactory. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewUpstreamIdentifierFactory(t interface { - mock.TestingT - Cleanup(func()) -}, -) *UpstreamIdentifierFactory { - mock := &UpstreamIdentifierFactory{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} - // EvaluatorFactory is an autogenerated mock type for the EvaluatorFactory type type EvaluatorFactory struct { mock.Mock @@ -639,23 +633,19 @@ type UpstreamIdentifier struct { } // IdentifyResources provides a mock function with given fields: ctx, assets -func (_m *UpstreamIdentifier) IdentifyResources(ctx context.Context, assets map[string]string) ([]resource.URN, error) { +func (_m *UpstreamIdentifier) IdentifyResources(ctx context.Context, assets map[string]string) ([]string, error) { ret := _m.Called(ctx, assets) - if len(ret) == 0 { - panic("no return value specified for IdentifyResources") - } - - var r0 []resource.URN + var r0 []string var r1 error - if rf, ok := ret.Get(0).(func(context.Context, map[string]string) ([]resource.URN, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, map[string]string) ([]string, error)); ok { return rf(ctx, assets) } - if rf, ok := ret.Get(0).(func(context.Context, map[string]string) []resource.URN); ok { + if rf, ok := ret.Get(0).(func(context.Context, map[string]string) []string); ok { r0 = rf(ctx, assets) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]resource.URN) + r0 = ret.Get(0).([]string) } } @@ -667,18 +657,3 @@ func (_m *UpstreamIdentifier) IdentifyResources(ctx context.Context, assets map[ return r0, r1 } - -// NewUpstreamIdentifier creates a new instance of UpstreamIdentifier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewUpstreamIdentifier(t interface { - mock.TestingT - Cleanup(func()) -}, -) *UpstreamIdentifier { - mock := &UpstreamIdentifier{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/plugin/upstream_identifier/bq_upstream_identifier.go b/plugin/upstream_identifier/bq_upstream_identifier.go index 8f1a307137..d857b360d4 100644 --- a/plugin/upstream_identifier/bq_upstream_identifier.go +++ b/plugin/upstream_identifier/bq_upstream_identifier.go @@ -6,7 +6,6 @@ import ( "github.com/goto/salt/log" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/ext/store/bigquery" "github.com/goto/optimus/internal/errors" ) @@ -23,7 +22,7 @@ type BQUpstreamIdentifier struct { evaluatorFuncs []EvalAssetFunc } -func (g BQUpstreamIdentifier) IdentifyResources(ctx context.Context, assets map[string]string) ([]resource.URN, error) { +func (g BQUpstreamIdentifier) IdentifyResources(ctx context.Context, assets map[string]string) ([]string, error) { resourcesAccumulation := []*bigquery.ResourceURNWithUpstreams{} // generate resource urn with upstream from each evaluator @@ -46,17 +45,9 @@ func (g BQUpstreamIdentifier) IdentifyResources(ctx context.Context, assets map[ // compiled all collected resources and extract its urns flattenedResources := bigquery.ResourceURNWithUpstreamsList(resourcesAccumulation).FlattenUnique() - var resourceURNs []resource.URN - for _, r := range flattenedResources { - rawURN := r.ResourceURN.URN() - - urn, err := resource.ParseURN(rawURN) - if err != nil { - me.Append(err) - continue - } - - resourceURNs = append(resourceURNs, urn) + resourceURNs := make([]string, len(flattenedResources)) + for i, r := range flattenedResources { + resourceURNs[i] = r.ResourceURN.URN() } return resourceURNs, me.ToErr() } diff --git a/plugin/upstream_identifier/bq_upstream_identifier_test.go b/plugin/upstream_identifier/bq_upstream_identifier_test.go index cd577a6ba1..173be54df2 100644 --- a/plugin/upstream_identifier/bq_upstream_identifier_test.go +++ b/plugin/upstream_identifier/bq_upstream_identifier_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/ext/store/bigquery" upstreamidentifier "github.com/goto/optimus/plugin/upstream_identifier" ) @@ -187,14 +186,7 @@ func TestIdentifyResources(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, bqUpstreamIdentifier) - urn1, err := resource.ParseURN("bigquery://project1:dataset1.name1") - assert.NoError(t, err) - urn2, err := resource.ParseURN("bigquery://project1:dataset1.name2") - assert.NoError(t, err) - urn3, err := resource.ParseURN("bigquery://project1:dataset1.name3") - assert.NoError(t, err) - - expectedResourceURNs := []resource.URN{urn1, urn2, urn3} + expectedResourceURNs := []string{"bigquery://project1:dataset1.name1", "bigquery://project1:dataset1.name2", "bigquery://project1:dataset1.name3"} resourceURNs, err := bqUpstreamIdentifier.IdentifyResources(ctx, assets) assert.NoError(t, err) assert.NotEmpty(t, resourceURNs) diff --git a/plugin/upstream_identifier/upstream_identifier.go b/plugin/upstream_identifier/upstream_identifier.go index 54cbbc3684..4f4ad30ab7 100644 --- a/plugin/upstream_identifier/upstream_identifier.go +++ b/plugin/upstream_identifier/upstream_identifier.go @@ -6,7 +6,6 @@ import ( "github.com/goto/salt/log" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/ext/extractor" "github.com/goto/optimus/ext/store/bigquery" "github.com/goto/optimus/plugin/upstream_identifier/evaluator" @@ -25,7 +24,7 @@ type UpstreamIdentifierFactory struct { } type UpstreamIdentifier interface { - IdentifyResources(ctx context.Context, assets map[string]string) ([]resource.URN, error) + IdentifyResources(ctx context.Context, assets map[string]string) ([]string, error) } func (u *UpstreamIdentifierFactory) GetBQUpstreamIdentifier(ctx context.Context, svcAcc string, evaluators ...evaluator.Evaluator) (UpstreamIdentifier, error) { diff --git a/server/optimus.go b/server/optimus.go index 2857c2e5db..80d6a2e540 100644 --- a/server/optimus.go +++ b/server/optimus.go @@ -350,26 +350,18 @@ func (s *OptimusServer) setupHandlers() error { evaluatorFactory, _ := evaluator.NewEvaluatorFactory(s.logger) pluginService, _ := plugin.NewPluginService(s.logger, s.pluginRepo, upstreamIdentifierFactory, evaluatorFactory) - // Resource Bounded Context - requirements - resourceRepository := resource.NewRepository(s.dbPool) - backupRepository := resource.NewBackupRepository(s.dbPool) - resourceManager := rService.NewResourceManager(resourceRepository, s.logger) - secondaryResourceService := rService.NewResourceService(s.logger, resourceRepository, nil, resourceManager, s.eventHandler, nil) // note: job service can be nil - // Job Bounded Context Setup jJobRepo := jRepo.NewJobRepository(s.dbPool) jExternalUpstreamResolver, _ := jResolver.NewExternalUpstreamResolver(s.conf.ResourceManagers) jInternalUpstreamResolver := jResolver.NewInternalUpstreamResolver(jJobRepo) jUpstreamResolver := jResolver.NewUpstreamResolver(jJobRepo, jExternalUpstreamResolver, jInternalUpstreamResolver) - jJobService := jService.NewJobService( - jJobRepo, jJobRepo, jJobRepo, - pluginService, jUpstreamResolver, tenantService, - s.eventHandler, s.logger, newJobRunService, newEngine, - jobInputCompiler, secondaryResourceService, - ) + jJobService := jService.NewJobService(jJobRepo, jJobRepo, jJobRepo, pluginService, jUpstreamResolver, tenantService, s.eventHandler, s.logger, newJobRunService, newEngine) // Resource Bounded Context - primaryResourceService := rService.NewResourceService(s.logger, resourceRepository, jJobService, resourceManager, s.eventHandler, jJobService) + resourceRepository := resource.NewRepository(s.dbPool) + backupRepository := resource.NewBackupRepository(s.dbPool) + resourceManager := rService.NewResourceManager(resourceRepository, s.logger) + resourceService := rService.NewResourceService(s.logger, resourceRepository, jJobService, resourceManager, s.eventHandler, jJobService) backupService := rService.NewBackupService(backupRepository, resourceRepository, resourceManager, s.logger) // Register datastore @@ -383,7 +375,7 @@ func (s *OptimusServer) setupHandlers() error { pb.RegisterNamespaceServiceServer(s.grpcServer, tHandler.NewNamespaceHandler(s.logger, tNamespaceService)) // Resource Handler - pb.RegisterResourceServiceServer(s.grpcServer, rHandler.NewResourceHandler(s.logger, primaryResourceService)) + pb.RegisterResourceServiceServer(s.grpcServer, rHandler.NewResourceHandler(s.logger, resourceService)) pb.RegisterJobRunServiceServer(s.grpcServer, schedulerHandler.NewJobRunHandler(s.logger, newJobRunService, eventsService)) diff --git a/tests/bench/job/job_repo_test.go b/tests/bench/job/job_repo_test.go index 3c2412af72..a82057bcb3 100644 --- a/tests/bench/job/job_repo_test.go +++ b/tests/bench/job/job_repo_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/assert" serviceJob "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" serviceTenant "github.com/goto/optimus/core/tenant" repoJob "github.com/goto/optimus/internal/store/postgres/job" repoTenant "github.com/goto/optimus/internal/store/postgres/tenant" @@ -122,9 +121,7 @@ func BenchmarkJobRepository(b *testing.B) { inferredUpstreamName, err := serviceJob.NameFrom(name) assert.NoError(b, err) - inferredUpstreamDestination, err := resource.ParseURN(fmt.Sprintf("store://dev.resource.sample_inferred_upstream_%d", i)) - assert.NoError(b, err) - + inferredUpstreamDestination := serviceJob.ResourceURN(fmt.Sprintf("dev.resource.sample_inferred_upstream_%d", i)) jobTreatedAsInferredUpstream := setup.NewDummyJobBuilder(). OverrideName(inferredUpstreamName). OverrideDestinationURN(inferredUpstreamDestination). @@ -137,7 +134,7 @@ func BenchmarkJobRepository(b *testing.B) { currentJob := setup.NewDummyJobBuilder(). OverrideName(currentJobName). OverrideSpecUpstreamNames([]serviceJob.SpecUpstreamName{serviceJob.SpecUpstreamNameFrom(staticUpstreamName.String())}). - OverrideSourceURNs([]resource.URN{inferredUpstreamDestination}). + OverrideSourceURNs([]serviceJob.ResourceURN{inferredUpstreamDestination}). Build(tnnt) storedJobs, err := repo.Add(ctx, []*serviceJob.Job{ @@ -224,9 +221,7 @@ func BenchmarkJobRepository(b *testing.B) { jobName, err := serviceJob.NameFrom(name) assert.NoError(b, err) - destination, err := resource.ParseURN(fmt.Sprintf("store://dev.resource.sample_%d", i)) - assert.NoError(b, err) - + destination := serviceJob.ResourceURN(fmt.Sprintf("dev.resource.sample_%d", i)) jobs[i] = setup.NewDummyJobBuilder(). OverrideName(jobName). OverrideDestinationURN(destination). @@ -257,9 +252,7 @@ func BenchmarkJobRepository(b *testing.B) { jobName, err := serviceJob.NameFrom(name) assert.NoError(b, err) - destination, err := resource.ParseURN(fmt.Sprintf("store://dev.resource.sample_%d", i)) - assert.NoError(b, err) - + destination := serviceJob.ResourceURN(fmt.Sprintf("dev.resource.sample_%d", i)) jobs[i] = setup.NewDummyJobBuilder(). OverrideName(jobName). OverrideDestinationURN(destination). @@ -276,9 +269,7 @@ func BenchmarkJobRepository(b *testing.B) { upstreams := make([]*serviceJob.Upstream, maxNumberOfUpstreams) for j := 0; j < maxNumberOfUpstreams; j++ { - resourceURN, err := resource.ParseURN(fmt.Sprintf("store://dev.resource.resource_%d_%d", i, j)) - assert.NoError(b, err) - + resourceURN := serviceJob.ResourceURN(fmt.Sprintf("dev.resource.resource_%d_%d", i, j)) upstream := serviceJob.NewUpstreamResolved(jobName, "http://optimus.io", resourceURN, tnnt, serviceJob.UpstreamTypeInferred, "bq2bq", false) upstreams[j] = upstream } @@ -367,10 +358,8 @@ func BenchmarkJobRepository(b *testing.B) { upstreams := make([]*serviceJob.Upstream, maxNumberOfUpstreams) for i := 0; i < maxNumberOfUpstreams; i++ { - resourceURN, err := resource.ParseURN(fmt.Sprintf("store://dev.resource.sample_%d", i)) - assert.NoError(b, err) - - upstreams[i] = serviceJob.NewUpstreamUnresolvedInferred(resourceURN) + resourceURN := fmt.Sprintf("dev.resource.sample_%d", i) + upstreams[i] = serviceJob.NewUpstreamUnresolvedInferred(serviceJob.ResourceURN(resourceURN)) } withUpstream := serviceJob.NewWithUpstream(currentJob, upstreams) @@ -392,8 +381,7 @@ func BenchmarkJobRepository(b *testing.B) { rootJobName, err := serviceJob.NameFrom("root_job") assert.NoError(b, err) - rootJobDestination, err := resource.ParseURN("store://root_job_destination") - assert.NoError(b, err) + rootJobDestination := serviceJob.ResourceURN("root_job_destination") rootJob := setup.NewDummyJobBuilder(). OverrideName(rootJobName). OverrideDestinationURN(rootJobDestination). @@ -405,13 +393,12 @@ func BenchmarkJobRepository(b *testing.B) { for i := 0; i < maxNumberOfDownstreams; i++ { currentJobName, err := serviceJob.NameFrom(fmt.Sprintf("downstream_job_%d", i)) assert.NoError(b, err) - currentDestinationURN, err := resource.ParseURN(fmt.Sprintf("store://dev.resource.sample_%d", i)) - assert.NoError(b, err) + currentDestinationURN := serviceJob.ResourceURN(fmt.Sprintf("dev.resource.sample_%d", i)) currentJob := setup.NewDummyJobBuilder(). OverrideName(currentJobName). OverrideDestinationURN(currentDestinationURN). - OverrideSourceURNs([]resource.URN{rootJobDestination}). + OverrideSourceURNs([]serviceJob.ResourceURN{rootJobDestination}). Build(tnnt) _, err = repo.Add(ctx, []*serviceJob.Job{currentJob}) @@ -433,8 +420,7 @@ func BenchmarkJobRepository(b *testing.B) { rootJobName, err := serviceJob.NameFrom("root_job") assert.NoError(b, err) - rootJobDestination, err := resource.ParseURN("store://root_job_destination") - assert.NoError(b, err) + rootJobDestination := serviceJob.ResourceURN("root_job_destination") rootJob := setup.NewDummyJobBuilder(). OverrideName(rootJobName). OverrideDestinationURN(rootJobDestination). @@ -446,8 +432,7 @@ func BenchmarkJobRepository(b *testing.B) { for i := 0; i < maxNumberOfDownstreams; i++ { currentJobName, err := serviceJob.NameFrom(fmt.Sprintf("downstream_job_%d", i)) assert.NoError(b, err) - currentDestinationURN, err := resource.ParseURN(fmt.Sprintf("store://dev.resource.sample_%d", i)) - assert.NoError(b, err) + currentDestinationURN := serviceJob.ResourceURN(fmt.Sprintf("dev.resource.sample_%d", i)) currentJob := setup.NewDummyJobBuilder(). OverrideName(currentJobName). diff --git a/tests/bench/resource/resource_repo_test.go b/tests/bench/resource/resource_repo_test.go index 70e0191c16..8a3f04a4ee 100644 --- a/tests/bench/resource/resource_repo_test.go +++ b/tests/bench/resource/resource_repo_test.go @@ -19,10 +19,7 @@ import ( ) func BenchmarkResourceRepository(b *testing.B) { - const ( - maxNumberOfResources = 64 - store = "store" - ) + const maxNumberOfResources = 64 projectName := "project_test" transporterKafkaBrokerKey := "KAFKA_BROKERS" @@ -82,9 +79,7 @@ func BenchmarkResourceRepository(b *testing.B) { resourceToCreate, err := serviceResource.NewResource(fullName, bigquery.KindDataset, serviceResource.Bigquery, tnnt, meta, spec) assert.NoError(b, err) - name := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) - urn, err := serviceResource.NewURN(store, name) - assert.NoError(b, err) + urn := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) err = resourceToCreate.UpdateURN(urn) assert.NoError(b, err) @@ -102,9 +97,7 @@ func BenchmarkResourceRepository(b *testing.B) { resourceToCreate, err := serviceResource.NewResource(fullName, bigquery.KindDataset, serviceResource.Bigquery, tnnt, meta, spec) assert.NoError(b, err) - name := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) - urn, err := serviceResource.NewURN(store, name) - assert.NoError(b, err) + urn := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) err = resourceToCreate.UpdateURN(urn) assert.NoError(b, err) @@ -129,9 +122,7 @@ func BenchmarkResourceRepository(b *testing.B) { resourceToUpdate, err := serviceResource.NewResource(fullName, bigquery.KindDataset, serviceResource.Bigquery, tnnt, updatedMeta, spec) assert.NoError(b, err) - name := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) - urn, err := serviceResource.NewURN(store, name) - assert.NoError(b, err) + urn := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) err = resourceToUpdate.UpdateURN(urn) assert.NoError(b, err) @@ -158,9 +149,7 @@ func BenchmarkResourceRepository(b *testing.B) { resourceToCreate, err := serviceResource.NewResource(fullName, bigquery.KindDataset, serviceResource.Bigquery, tnnt, meta, spec) assert.NoError(b, err) - name := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) - urn, err := serviceResource.NewURN(store, name) - assert.NoError(b, err) + urn := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) err = resourceToCreate.UpdateURN(urn) assert.NoError(b, err) @@ -191,9 +180,7 @@ func BenchmarkResourceRepository(b *testing.B) { resourceToCreate, err := serviceResource.NewResource(fullName, bigquery.KindDataset, serviceResource.Bigquery, tnnt, meta, spec) assert.NoError(b, err) - name := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) - urn, err := serviceResource.NewURN(store, name) - assert.NoError(b, err) + urn := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) err = resourceToCreate.UpdateURN(urn) assert.NoError(b, err) @@ -219,9 +206,7 @@ func BenchmarkResourceRepository(b *testing.B) { resourceToCreate, err := serviceResource.NewResource(fullName, bigquery.KindDataset, serviceResource.Bigquery, tnnt, meta, spec) assert.NoError(b, err) - name := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) - urn, err := serviceResource.NewURN(store, name) - assert.NoError(b, err) + urn := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) err = resourceToCreate.UpdateURN(urn) assert.NoError(b, err) @@ -249,9 +234,7 @@ func BenchmarkResourceRepository(b *testing.B) { resourceToCreate, err := serviceResource.NewResource(fullName, bigquery.KindDataset, serviceResource.Bigquery, tnnt, meta, spec) assert.NoError(b, err) - name := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) - urn, err := serviceResource.NewURN(store, name) - assert.NoError(b, err) + urn := fmt.Sprintf("%s:%s.%s", projectName, namespaceName, fullName) err = resourceToCreate.UpdateURN(urn) assert.NoError(b, err) diff --git a/tests/setup/job.go b/tests/setup/job.go index b6cdcae5da..a60ef7d3ba 100644 --- a/tests/setup/job.go +++ b/tests/setup/job.go @@ -2,7 +2,6 @@ package setup import ( "github.com/goto/optimus/core/job" - "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" "github.com/goto/optimus/internal/lib/window" "github.com/goto/optimus/internal/models" @@ -38,8 +37,8 @@ type DummyJobBuilder struct { name job.Name - destinationURN resource.URN - sourceURNs []resource.URN + destinationURN job.ResourceURN + sourceURNs []job.ResourceURN specUpstreamNames []job.SpecUpstreamName specHTTPUpstreams []*job.SpecHTTPUpstream @@ -103,16 +102,6 @@ func NewDummyJobBuilder() *DummyJobBuilder { panic(err) } - dummyDestination, err := resource.ParseURN("store://sample_job_destination") - if err != nil { - panic(err) - } - - dummySource, err := resource.ParseURN("store://source_of_sample_job") - if err != nil { - panic(err) - } - return &DummyJobBuilder{ version: version, owner: "dev_test", @@ -135,8 +124,8 @@ func NewDummyJobBuilder() *DummyJobBuilder { resourceLimitConfig: job.NewMetadataResourceConfig("128m", "128Mi"), scheduler: map[string]string{"scheduler_config_key": "value"}, name: name, - destinationURN: dummyDestination, - sourceURNs: []resource.URN{dummySource}, + destinationURN: job.ResourceURN("sample_job_destination"), + sourceURNs: []job.ResourceURN{"source_of_sample_job"}, specUpstreamNames: []job.SpecUpstreamName{job.SpecUpstreamNameFrom("smpale_job_upstream")}, specHTTPUpstreams: []*job.SpecHTTPUpstream{specHTTPUpstream}, } @@ -256,13 +245,13 @@ func (d *DummyJobBuilder) OverrideName(name job.Name) *DummyJobBuilder { return &output } -func (d *DummyJobBuilder) OverrideDestinationURN(destinationURN resource.URN) *DummyJobBuilder { +func (d *DummyJobBuilder) OverrideDestinationURN(destinationURN job.ResourceURN) *DummyJobBuilder { output := *d output.destinationURN = destinationURN return &output } -func (d *DummyJobBuilder) OverrideSourceURNs(sourceURNs []resource.URN) *DummyJobBuilder { +func (d *DummyJobBuilder) OverrideSourceURNs(sourceURNs []job.ResourceURN) *DummyJobBuilder { output := *d output.sourceURNs = sourceURNs return &output