Skip to content

Commit

Permalink
fix comment
Browse files Browse the repository at this point in the history
  • Loading branch information
ryota-cloud committed Mar 4, 2025
1 parent a8d7980 commit b31d0f6
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 11 deletions.
22 changes: 11 additions & 11 deletions metadata-ingestion/src/datahub/ingestion/source/vertexai.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def get_report(self) -> SourceReport:

def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
"""
Main Function to fetch and yields work units for various VertexAI resources.
Main Function to fetch and yields mcps for various VertexAI resources.
- Models and Model Versions from the Model Registry
- Training Jobs
"""
Expand All @@ -165,15 +165,15 @@ def _gen_project_workunits(self) -> Iterable[MetadataWorkUnit]:

def _get_ml_models_mcps(self) -> Iterable[MetadataChangeProposalWrapper]:
"""
Fetch List of Models in Model Registry and generate a corresponding work unit.
Fetch List of Models in Model Registry and generate a corresponding mcp.
"""
registered_models = self.client.Model.list()
for model in registered_models:
# create work unit for Model Group (= Model in VertexAI)
# create mcp for Model Group (= Model in VertexAI)
yield from self._gen_ml_group_mcps(model)
model_versions = model.versioning_registry.list_versions()
for model_version in model_versions:
# create work unit for Model (= Model Version in VertexAI)
# create mcp for Model (= Model Version in VertexAI)
logger.info(
f"Ingesting a model (name: {model.display_name} id:{model.name})"
)
Expand All @@ -183,11 +183,11 @@ def _get_ml_models_mcps(self) -> Iterable[MetadataChangeProposalWrapper]:

def _get_training_jobs_mcps(self) -> Iterable[MetadataChangeProposalWrapper]:
"""
Fetches training jobs from Vertex AI and generates corresponding work units.
Fetches training jobs from Vertex AI and generates corresponding mcps.
This method retrieves various types of training jobs from Vertex AI, including
CustomJob, CustomTrainingJob, CustomContainerTrainingJob, CustomPythonPackageTrainingJob,
AutoMLTabularTrainingJob, AutoMLTextTrainingJob, AutoMLImageTrainingJob, AutoMLVideoTrainingJob,
and AutoMLForecastingTrainingJob. For each job, it generates work units containing metadata
and AutoMLForecastingTrainingJob. For each job, it generates mcps containing metadata
about the job, its inputs, and its outputs.
"""
class_names = [
Expand Down Expand Up @@ -235,7 +235,7 @@ def _gen_training_job_mcps(
self, job_meta: TrainingJobMetadata
) -> Iterable[MetadataChangeProposalWrapper]:
"""
Generate a work unit for VertexAI Training Job
Generate a mcp for VertexAI Training Job
"""
job = job_meta.job
job_id = self._make_vertexai_job_name(entity_id=job.name)
Expand Down Expand Up @@ -293,7 +293,7 @@ def _gen_ml_group_mcps(
model: Model,
) -> Iterable[MetadataChangeProposalWrapper]:
"""
Generate an MLModelGroup work unit for a VertexAI Model.
Generate an MLModelGroup mcp for a VertexAI Model.
"""
ml_model_group_urn = self._make_ml_model_group_urn(model)

Expand Down Expand Up @@ -518,7 +518,7 @@ def _gen_ml_model_endpoint_mcps(
training_job_urn: Optional[str] = None,
) -> Iterable[MetadataChangeProposalWrapper]:
"""
Generate an MLModel and Endpoint work unit for an VertexAI Model Version.
Generate an MLModel and Endpoint mcp for an VertexAI Model Version.
"""

endpoint: Optional[Endpoint] = self._search_endpoint(model)
Expand All @@ -539,11 +539,11 @@ def _gen_ml_model_mcps(
endpoint_urn: Optional[str] = None,
) -> Iterable[MetadataChangeProposalWrapper]:
"""
Generate an MLModel workunit for an VertexAI Model Version.
Generate an MLModel mcp for an VertexAI Model Version.
Every Model Version is a DataHub MLModel entity associated with an MLModelGroup
corresponding to a registered Model in VertexAI Model Registry.
"""
logging.info(f"starting model work unit for model {model.name}")
logging.info(f"generating model mcp for {model.name}")

model_group_urn = self._make_ml_model_group_urn(model)
model_name = self._make_vertexai_model_name(entity_id=model.name)
Expand Down
1 change: 1 addition & 0 deletions metadata-ingestion/tests/unit/test_vertexai_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,7 @@ def test_get_endpoint_mcps(
elif isinstance(aspect, SubTypesClass):
assert aspect.typeNames == ["Endpoint"]


def test_get_training_jobs_mcps(
source: VertexAISource,
) -> None:
Expand Down

0 comments on commit b31d0f6

Please sign in to comment.