Skip to content

Commit

Permalink
Rename spec* to unit - first pass
Browse files Browse the repository at this point in the history
  • Loading branch information
jwillp committed Sep 4, 2024
1 parent 2e8328d commit 2716396
Show file tree
Hide file tree
Showing 24 changed files with 682 additions and 612 deletions.
20 changes: 17 additions & 3 deletions pkg/specter/artifactproc.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
// Copyright 2024 Morébec
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package specter

import (
Expand Down Expand Up @@ -81,9 +95,9 @@ func (n ProcessorArtifactRegistry) FindAll() ([]ArtifactRegistryEntry, error) {

type ArtifactProcessingContext struct {
context.Context
Specifications SpecificationGroup
Artifacts []Artifact
Logger Logger
Units UnitGroup
Artifacts []Artifact
Logger Logger

ArtifactRegistry ProcessorArtifactRegistry
processorName string
Expand Down
2 changes: 1 addition & 1 deletion pkg/specter/artifactproc_fileartifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ const DefaultWriteMode WriteMode = WriteOnceMode

var _ Artifact = (*FileArtifact)(nil)

// FileArtifact is a data structure that can be used by a SpecificationProcessor to generate file artifacts
// FileArtifact is a data structure that can be used by a UnitProcessor to generate file artifacts
// that can be written by the FileArtifactProcessor.
type FileArtifact struct {
Path string
Expand Down
8 changes: 4 additions & 4 deletions pkg/specter/assembly.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,15 @@ func WithSourceLoaders(loaders ...SourceLoader) PipelineOption {
}
}

// WithLoaders configures the SpecificationLoader of a Pipeline instance.
func WithLoaders(loaders ...SpecificationLoader) PipelineOption {
// WithLoaders configures the UnitLoader of a Pipeline instance.
func WithLoaders(loaders ...UnitLoader) PipelineOption {
return func(s *Pipeline) {
s.Loaders = append(s.Loaders, loaders...)
}
}

// WithProcessors configures the SpecProcess of a Pipeline instance.
func WithProcessors(processors ...SpecificationProcessor) PipelineOption {
// WithProcessors configures the UnitProcess of a Pipeline instance.
func WithProcessors(processors ...UnitProcessor) PipelineOption {
return func(s *Pipeline) {
s.Processors = append(s.Processors, processors...)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/specter/assembly_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func TestWithSourceLoaders(t *testing.T) {
}

func TestWithLoaders(t *testing.T) {
loader := &specterutils.HCLGenericSpecLoader{}
loader := &specterutils.HCLGenericUnitLoader{}
s := NewPipeline(WithLoaders(loader))
require.Contains(t, s.Loaders, loader)
}
Expand Down
16 changes: 15 additions & 1 deletion pkg/specter/common.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,22 @@
// Copyright 2024 Morébec
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package specter

type ArtifactID string

// Artifact represents a result or output generated by a SpecificationProcessor.
// Artifact represents a result or output generated by a UnitProcessor.
// An artifact is a unit of data or information produced as part of the processing workflow.
// It can be a transient, in-memory object, or it might represent more permanent entities such as
// files on disk, records in a database, deployment units, or other forms of data artifacts.
Expand Down
72 changes: 36 additions & 36 deletions pkg/specter/pipeline.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,15 @@ const RunThrough RunMode = "run-through"
const defaultRunMode = PreviewMode

const SourceLoadingFailedErrorCode = "specter.source_loading_failed"
const SpecificationLoadingFailedErrorCode = "specter.specification_loading_failed"
const SpecificationProcessingFailedErrorCode = "specter.specification_processing_failed"
const UnitLoadingFailedErrorCode = "specter.unit_loading_failed"
const UnitProcessingFailedErrorCode = "specter.unit_processing_failed"
const ArtifactProcessingFailedErrorCode = "specter.artifact_processing_failed"

// Pipeline is the service responsible to run a specter pipeline.
type Pipeline struct {
SourceLoaders []SourceLoader
Loaders []SpecificationLoader
Processors []SpecificationProcessor
Loaders []UnitLoader
Processors []UnitProcessor
ArtifactProcessors []ArtifactProcessor
ArtifactRegistry ArtifactRegistry
Logger Logger
Expand All @@ -53,7 +53,7 @@ type PipelineResult struct {

SourceLocations []string
Sources []Source
Specifications []Specification
Units []Unit
Artifacts []Artifact
RunMode RunMode
}
Expand Down Expand Up @@ -92,18 +92,18 @@ func (p Pipeline) Run(ctx context.Context, sourceLocations []string, runMode Run
return result, e
}

// Load Specifications
result.Specifications, err = p.loadSpecifications(ctx, result.Sources)
// Load Units
result.Units, err = p.loadUnits(ctx, result.Sources)
if err != nil {
e := errors.WrapWithMessage(err, SpecificationLoadingFailedErrorCode, "failed loading specifications")
e := errors.WrapWithMessage(err, UnitLoadingFailedErrorCode, "failed loading units")
p.Logger.Error(e.Error())
return result, e
}

// Process Specifications
result.Artifacts, err = p.processSpecifications(ctx, result.Specifications)
// Process Units
result.Artifacts, err = p.processUnits(ctx, result.Units)
if err != nil {
e := errors.WrapWithMessage(err, SpecificationProcessingFailedErrorCode, "failed processing specifications")
e := errors.WrapWithMessage(err, UnitProcessingFailedErrorCode, "failed processing units")
p.Logger.Error(e.Error())
return result, e
}
Expand All @@ -114,7 +114,7 @@ func (p Pipeline) Run(ctx context.Context, sourceLocations []string, runMode Run
}

// Process Artifact
if err = p.processArtifacts(ctx, result.Specifications, result.Artifacts); err != nil {
if err = p.processArtifacts(ctx, result.Units, result.Artifacts); err != nil {
e := errors.WrapWithMessage(err, ArtifactProcessingFailedErrorCode, "failed processing artifacts")
p.Logger.Error(e.Error())
return result, e
Expand All @@ -131,7 +131,7 @@ func (p Pipeline) logResult(run PipelineResult) {
p.Logger.Info(fmt.Sprintf("Run time: %s", run.ExecutionTime()))
p.Logger.Info(fmt.Sprintf("Number of source locations: %d", len(run.SourceLocations)))
p.Logger.Info(fmt.Sprintf("Number of sources: %d", len(run.Sources)))
p.Logger.Info(fmt.Sprintf("Number of specifications: %d", len(run.Specifications)))
p.Logger.Info(fmt.Sprintf("Number of units: %d", len(run.Units)))
p.Logger.Info(fmt.Sprintf("Number of artifacts: %d", len(run.Artifacts)))
}

Expand Down Expand Up @@ -171,12 +171,12 @@ func (p Pipeline) loadSources(ctx context.Context, sourceLocations []string) ([]
return sources, errors.GroupOrNil(errs)
}

// loadSpecifications performs the loading of Specifications.
func (p Pipeline) loadSpecifications(ctx context.Context, sources []Source) ([]Specification, error) {
p.Logger.Info("\nLoading specifications ...")
// loadUnits performs the loading of Units.
func (p Pipeline) loadUnits(ctx context.Context, sources []Source) ([]Unit, error) {
p.Logger.Info("\nLoading units ...")

// Load specifications
var specifications []Specification
// Load units
var units []Unit
var sourcesNotLoaded []Source
errs := errors.NewGroup(errors.InternalErrorCode)

Expand All @@ -190,14 +190,14 @@ func (p Pipeline) loadSpecifications(ctx context.Context, sources []Source) ([]S
continue
}

loadedSpecs, err := l.Load(src)
loadedUnits, err := l.Load(src)
if err != nil {
p.Logger.Error(err.Error())
errs = errs.Append(err)
continue
}

specifications = append(specifications, loadedSpecs...)
units = append(units, loadedUnits...)
wasLoaded = true
}

Expand All @@ -211,24 +211,24 @@ func (p Pipeline) loadSpecifications(ctx context.Context, sources []Source) ([]S
p.Logger.Warning(fmt.Sprintf("%q could not be loaded.", src))
}

p.Logger.Warning("%d specifications were not loaded.")
p.Logger.Warning("%d units were not loaded.")
}

p.Logger.Info(fmt.Sprintf("%d specifications loaded.", len(specifications)))
p.Logger.Info(fmt.Sprintf("%d units loaded.", len(units)))

return specifications, errors.GroupOrNil(errs)
return units, errors.GroupOrNil(errs)
}

// processSpecifications sends the specifications to processors.
func (p Pipeline) processSpecifications(ctx context.Context, specs []Specification) ([]Artifact, error) {
// processUnits sends the units to processors.
func (p Pipeline) processUnits(ctx context.Context, units []Unit) ([]Artifact, error) {
pctx := ProcessingContext{
Context: ctx,
Specifications: specs,
Artifacts: nil,
Logger: p.Logger,
Context: ctx,
Units: units,
Artifacts: nil,
Logger: p.Logger,
}

p.Logger.Info("\nProcessing specifications ...")
p.Logger.Info("\nProcessing units ...")
for _, processor := range p.Processors {
if err := ctx.Err(); err != nil {
return nil, err
Expand All @@ -245,12 +245,12 @@ func (p Pipeline) processSpecifications(ctx context.Context, specs []Specificati
p.Logger.Info(fmt.Sprintf("-> %s", o.ID()))
}

p.Logger.Success("Specifications processed successfully.")
p.Logger.Success("Units processed successfully.")
return pctx.Artifacts, nil
}

// processArtifacts sends a list of ProcessingArtifacts to the registered ArtifactProcessors.
func (p Pipeline) processArtifacts(ctx context.Context, specifications []Specification, artifacts []Artifact) error {
func (p Pipeline) processArtifacts(ctx context.Context, units []Unit, artifacts []Artifact) error {
if p.ArtifactRegistry == nil {
p.ArtifactRegistry = &InMemoryArtifactRegistry{}
}
Expand All @@ -273,10 +273,10 @@ func (p Pipeline) processArtifacts(ctx context.Context, specifications []Specifi

processorName := processor.Name()
artifactCtx := ArtifactProcessingContext{
Context: ctx,
Specifications: specifications,
Artifacts: artifacts,
Logger: p.Logger,
Context: ctx,
Units: units,
Artifacts: artifacts,
Logger: p.Logger,
ArtifactRegistry: ProcessorArtifactRegistry{
processorName: processorName,
registry: p.ArtifactRegistry,
Expand Down
38 changes: 19 additions & 19 deletions pkg/specter/pipeline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestRunResult_ExecutionTime(t *testing.T) {
require.Equal(t, r.ExecutionTime(), time.Hour*1)
}

func TestSpecter_Run(t *testing.T) {
func TestUnitter_Run(t *testing.T) {
testDay := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)

type given struct {
Expand Down Expand Up @@ -71,12 +71,12 @@ func TestSpecter_Run(t *testing.T) {
},
then: then{
expectedRunResult: PipelineResult{
RunMode: PreviewMode,
Sources: nil,
Specifications: nil,
Artifacts: nil,
StartedAt: testDay,
EndedAt: testDay,
RunMode: PreviewMode,
Sources: nil,
Units: nil,
Artifacts: nil,
StartedAt: testDay,
EndedAt: testDay,
},
expectedError: assert.NoError,
},
Expand All @@ -97,12 +97,12 @@ func TestSpecter_Run(t *testing.T) {
},
then: then{
expectedRunResult: PipelineResult{
RunMode: PreviewMode,
Sources: nil,
Specifications: nil,
Artifacts: nil,
StartedAt: testDay,
EndedAt: testDay,
RunMode: PreviewMode,
Sources: nil,
Units: nil,
Artifacts: nil,
StartedAt: testDay,
EndedAt: testDay,
},
expectedError: assert.NoError,
},
Expand All @@ -123,12 +123,12 @@ func TestSpecter_Run(t *testing.T) {
},
then: then{
expectedRunResult: PipelineResult{
RunMode: PreviewMode,
Sources: nil,
Specifications: nil,
Artifacts: nil,
StartedAt: testDay,
EndedAt: testDay,
RunMode: PreviewMode,
Sources: nil,
Units: nil,
Artifacts: nil,
StartedAt: testDay,
EndedAt: testDay,
},
expectedError: assert.NoError,
},
Expand Down
Loading

0 comments on commit 2716396

Please sign in to comment.