Merge branch 'master' into feature/refactor-getLastIngestionRun

.github/.codecov.yml

@@ -1,21 +1,19 @@
 comment:
-  layout: "header, files, footer" # remove "new" from "header" and "footer"
-  hide_project_coverage: true # set to false
+  layout: "condensed_header, condensed_files, condensed_footer"
+  hide_project_coverage: true
   require_changes: false  # if true: only post the comment if coverage changes
 
 codecov:
   #due to ci-optimization, reports for modules that have not changed may be quite old
   max_report_age: off
 
+github_checks:
+  #Hide annotations that show up in github PR reviews. There still is a red bar next to lines not covered
+  annotations: false
+
 flag_management:
   default_rules: # the rules that will be followed for any flag added, generally
     carryforward: true
-    statuses:
-      - type: project
-        target: auto
-        threshold: 0%  #Not enforcing project coverage yet.
-      - type: patch
-        target: 90%
   individual_flags: # exceptions to the default rules above, stated flag by flag
     - name: frontend
       paths:
@@ -55,11 +53,8 @@ flag_management:
         - "metadata-ingestion-modules/prefect-plugin/**"
 coverage:
   status:
-    project:
-      default:
-        target: 0% # no threshold enforcement yet
-        only_pulls: true
+    project: false
     patch:
       default:
-        target: 90% # for new code added in the patch
-        only_pulls: true
+        target: 75% # for new code added in the patch
+        only_pulls: true

README.md

@@ -18,27 +18,38 @@ export const Logo = (props) => {
 <!--
 HOSTED_DOCS_ONLY-->
 <p align="center">
+<a href="https://datahubproject.io">
 <img alt="DataHub" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-logo-color-mark.svg" height="150" />
+</a>
 </p>
 <!-- -->
 
 # DataHub: The Data Discovery Platform for the Modern Data Stack
-## Built with ❤️ by <img src="https://datahubproject.io/img/acryl-logo-light-mark.png" width="25"/> [Acryl Data](https://acryldata.io) and <img src="https://datahubproject.io/img/LI-In-Bug.png" width="25"/> [LinkedIn](https://engineering.linkedin.com)
-[![Version](https://img.shields.io/github/v/release/datahub-project/datahub?include_prereleases)](https://github.com/datahub-project/datahub/releases/latest)
-[![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub)
-[![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush)
-[![Docker Pulls](https://img.shields.io/docker/pulls/acryldata/datahub-gms.svg)](https://hub.docker.com/r/acryldata/datahub-gms)
-[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme)
-[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md)
-[![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr)
-[![License](https://img.shields.io/github/license/datahub-project/datahub)](https://github.com/datahub-project/datahub/blob/master/LICENSE)
-[![YouTube](https://img.shields.io/youtube/channel/subscribers/UC3qFQC5IiwR5fvWEqi_tJ5w?style=social)](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w)
-[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://medium.com/datahub-project)
-[![Follow](https://img.shields.io/twitter/follow/datahubproject?label=Follow&style=social)](https://twitter.com/datahubproject)
-### 🏠 Hosted DataHub Docs (Courtesy of Acryl Data): [datahubproject.io](https://datahubproject.io/docs)
+
+### Built with ❤️ by <img src="https://datahubproject.io/img/acryl-logo-light-mark.png" width="20"/> [Acryl Data](https://acryldata.io) and <img src="https://datahubproject.io/img/LI-In-Bug.png" width="20"/> [LinkedIn](https://engineering.linkedin.com)
+
+<div>
+  <a target="_blank" href="https://github.com/datahub-project/datahub/blob/master/LICENSE">
+    <img alt="Apache 2.0 License" src="https://img.shields.io/badge/License-Apache_2.0-blue.svg?label=license&labelColor=133554&color=1890ff" /></a>
+  <a target="_blank" href="https://pypi.org/project/acryl-datahub/">
+    <img alt="PyPI" src="https://img.shields.io/pypi/dm/acryl-datahub?label=downloads&labelColor=133554&color=1890ff" /></a>
+  <a target="_blank" href="https://github.com/datahub-project/datahub/pulse">
+    <img alt="GitHub commit activity" src="https://img.shields.io/github/commit-activity/m/datahub-project/datahub?label=commits&labelColor=133554&color=1890ff" /></a>
+  <br />
+  <a target="_blank" href="https://pages.acryl.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme">
+    <img alt="Slack" src="https://img.shields.io/badge/slack-join_community-red.svg?logo=slack&labelColor=133554&color=1890ff" /></a>
+  <a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w">
+    <img alt="YouTube" src="https://img.shields.io/youtube/channel/subscribers/UC3qFQC5IiwR5fvWEqi_tJ5w?style=flat&logo=youtube&label=subscribers&labelColor=133554&color=1890ff"/></a>
+  <a href="https://blog.datahubproject.io/">
+    <img alt="Medium" src="https://img.shields.io/badge/blog-DataHub-red.svg?style=flat&logo=medium&logoColor=white&labelColor=133554&color=1890ff" /></a>
+  <a href="https://x.com/datahubproject">
+    <img alt="X (formerly Twitter) Follow" src="https://img.shields.io/badge/follow-datahubproject-red.svg?style=flat&logo=x&labelColor=133554&color=1890ff" /></a>
+</div>
 
 ---
 
+### 🏠 Docs: [datahubproject.io](https://datahubproject.io/docs)
+
 [Quickstart](https://datahubproject.io/docs/quickstart) |
 [Features](https://datahubproject.io/docs/) |
 [Roadmap](https://feature-requests.datahubproject.io/roadmap) |
@@ -47,6 +58,7 @@ HOSTED_DOCS_ONLY-->
 [Town Hall](https://datahubproject.io/docs/townhalls)
 
 ---
+
 > 📣 DataHub Town Hall is the 4th Thursday at 9am US PT of every month - [add it to your calendar!](https://rsvp.datahubproject.io/)
 >
 > - Town-hall Zoom link: [zoom.datahubproject.io](https://zoom.datahubproject.io)
@@ -70,11 +82,11 @@ Check out DataHub's [Features](docs/features.md) & [Roadmap](https://feature-req
 
 ## Demo and Screenshots
 
-There's a [hosted demo environment](https://demo.datahubproject.io/) courtesy of [Acryl Data](https://acryldata.io) where you can explore DataHub without installing it locally
+There's a [hosted demo environment](https://demo.datahubproject.io/) courtesy of [Acryl Data](https://acryldata.io) where you can explore DataHub without installing it locally.
 
 ## Quickstart
 
-Please follow the [DataHub Quickstart Guide](https://datahubproject.io/docs/quickstart) to get a copy of DataHub up & running locally using [Docker](https://docker.com). As the guide assumes some basic knowledge of Docker, we'd recommend you to go through the "Hello World" example of [A Docker Tutorial for Beginners](https://docker-curriculum.com) if Docker is completely foreign to you.
+Please follow the [DataHub Quickstart Guide](https://datahubproject.io/docs/quickstart) to run DataHub locally using [Docker](https://docker.com).
 
 ## Development
 
@@ -106,7 +118,7 @@ We welcome contributions from the community. Please refer to our [Contributing G
 
 ## Community
 
-Join our [Slack workspace](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
+Join our [Slack workspace](https://pages.acryl.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
 
 ## Security
 
@@ -159,8 +171,6 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Wolt](https://wolt.com)
 - [Zynga](https://www.zynga.com)
 
-
-
 ## Select Articles & Talks
 
 - [DataHub Blog](https://blog.datahubproject.io/)

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/analytics/resolver/GetMetadataAnalyticsResolver.java

@@ -78,7 +78,7 @@ private List<AnalyticsChart> getCharts(MetadataAnalyticsInput input, OperationCo
 
     SearchResult searchResult =
         _entityClient.searchAcrossEntities(
-            opContext, entities, query, filter, 0, 0, Collections.emptyList(), null);
+            opContext, entities, query, filter, 0, 0, Collections.emptyList());
 
     List<AggregationMetadata> aggregationMetadataList =
         searchResult.getMetadata().getAggregations();

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/container/ContainerEntitiesResolver.java

@@ -91,8 +91,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
                                         new CriterionArray(ImmutableList.of(filterCriterion))))),
                     start,
                     count,
-                    Collections.emptyList(),
-                    null));
+                    Collections.emptyList()));
 
           } catch (Exception e) {
             throw new RuntimeException(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java

@@ -185,7 +185,6 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
                         finalFilter,
                         start,
                         count,
-                        null,
                         null));
             results
                 .getSearchResults()

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java

@@ -95,8 +95,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
                                 new ConjunctiveCriterion().setAnd(criteria))),
                     start,
                     count,
-                    Collections.emptyList(),
-                    null));
+                    Collections.emptyList()));
 
           } catch (Exception e) {
             throw new RuntimeException(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/lineage/UpdateLineageResolver.java

@@ -222,16 +222,15 @@ private void checkLineageEdgePrivileges(
     if (!isAuthorized(context, upstreamUrn, editLineagePrivileges)) {
       throw new AuthorizationException(
           String.format(
-              "Unauthorized to edit %s lineage. Please contact your DataHub administrator.",
-              upstreamUrn.getEntityType()));
+              "Unauthorized to edit %s lineage for %s", upstreamUrn, upstreamUrn.getEntityType()));
     }
 
     Urn downstreamUrn = UrnUtils.getUrn(lineageEdge.getDownstreamUrn());
     if (!isAuthorized(context, downstreamUrn, editLineagePrivileges)) {
       throw new AuthorizationException(
           String.format(
-              "Unauthorized to edit %s lineage. Please contact your DataHub administrator.",
-              downstreamUrn.getEntityType()));
+              "Unauthorized to edit %s lineage for %s",
+              downstreamUrn, downstreamUrn.getEntityType()));
     }
   }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/GetGrantedPrivilegesResolver.java

@@ -57,6 +57,6 @@ public CompletableFuture<Privileges> get(final DataFetchingEnvironment environme
   }
 
   private boolean isAuthorized(final QueryContext context, final String actor) {
-    return actor.equals(context.getActorUrn());
+    return PolicyAuthUtils.canManagePolicies(context) || actor.equals(context.getActorUrn());
   }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/GetQuickFiltersResolver.java

@@ -108,8 +108,7 @@ private SearchResult getSearchResults(
             : null,
         0,
         0,
-        Collections.emptyList(),
-        null);
+        Collections.emptyList());
   }
 
   /**

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossEntitiesResolver.java

@@ -16,6 +16,7 @@
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.query.SearchFlags;
 import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.query.filter.SortCriterion;
 import com.linkedin.metadata.service.ViewService;
 import com.linkedin.view.DataHubViewInfo;
 import graphql.schema.DataFetcher;
@@ -80,6 +81,7 @@ public CompletableFuture<ScrollResults> get(DataFetchingEnvironment environment)
           } else {
             searchFlags = null;
           }
+          List<SortCriterion> sortCriteria = SearchUtils.getSortCriteria(input.getSortInput());
 
           try {
             log.debug(
@@ -108,6 +110,7 @@ public CompletableFuture<ScrollResults> get(DataFetchingEnvironment environment)
                         : baseFilter,
                     scrollId,
                     keepAlive,
+                    sortCriteria,
                     count));
           } catch (Exception e) {
             log.error(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java

@@ -146,8 +146,7 @@ private List<String> getStructuredPropertyFacets(final QueryContext context) {
               createStructuredPropertyFilter(),
               0,
               100,
-              Collections.emptyList(),
-              null);
+              Collections.emptyList());
       return result.getEntities().stream()
           .map(entity -> String.format("structuredProperties.%s", entity.getEntity().getId()))
           .collect(Collectors.toList());

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java

@@ -378,6 +378,8 @@ public static CompletableFuture<ScrollResults> scrollAcrossEntities(
       Integer inputCount,
       String scrollId,
       String inputKeepAlive,
+      List<SortCriterion> sortCriteria,
+      List<String> facets,
       String className) {
 
     final List<EntityType> entityTypes =
@@ -431,7 +433,15 @@ public static CompletableFuture<ScrollResults> scrollAcrossEntities(
           try {
             final ScrollResult scrollResult =
                 _entityClient.scrollAcrossEntities(
-                    context, finalEntityNames, query, finalFilters, scrollId, keepAlive, count);
+                    context,
+                    finalEntityNames,
+                    query,
+                    finalFilters,
+                    scrollId,
+                    keepAlive,
+                    sortCriteria,
+                    count,
+                    facets);
             return UrnScrollResultsMapper.map(inputContext, scrollResult);
           } catch (Exception e) {
             log.warn(
@@ -518,14 +528,7 @@ public static CompletableFuture<SearchResults> searchAcrossEntities(
           try {
             final SearchResult searchResult =
                 _entityClient.searchAcrossEntities(
-                    context,
-                    finalEntityNames,
-                    query,
-                    finalFilters,
-                    start,
-                    count,
-                    sortCriteria,
-                    null);
+                    context, finalEntityNames, query, finalFilters, start, count, sortCriteria);
             return UrnSearchResultsMapper.map(inputContext, searchResult);
           } catch (Exception e) {
             log.warn(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/siblings/SiblingsSearchResolver.java

@@ -19,6 +19,7 @@
 import com.linkedin.metadata.utils.CriterionUtils;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -61,6 +62,8 @@ public CompletableFuture<ScrollResults> get(DataFetchingEnvironment environment)
         input.getCount(),
         input.getScrollId(),
         input.getKeepAlive(),
+        List.of(),
+        List.of(),
         this.getClass().getSimpleName());
   }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataplatform/DataPlatformType.java

@@ -2,24 +2,38 @@
 
 import static com.linkedin.metadata.Constants.*;
 
+import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.urn.Urn;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
+import com.linkedin.datahub.graphql.generated.AutoCompleteResults;
 import com.linkedin.datahub.graphql.generated.DataPlatform;
 import com.linkedin.datahub.graphql.generated.Entity;
+import com.linkedin.datahub.graphql.generated.FacetFilterInput;
+import com.linkedin.datahub.graphql.generated.SearchResults;
+import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.datahub.graphql.types.EntityType;
+import com.linkedin.datahub.graphql.types.SearchableEntityType;
 import com.linkedin.datahub.graphql.types.dataplatform.mappers.DataPlatformMapper;
+import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper;
+import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper;
 import com.linkedin.entity.EntityResponse;
 import com.linkedin.entity.client.EntityClient;
+import com.linkedin.metadata.query.AutoCompleteResult;
+import com.linkedin.metadata.query.filter.Filter;
+import com.linkedin.metadata.search.SearchResult;
 import graphql.execution.DataFetcherResult;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
-public class DataPlatformType implements EntityType<DataPlatform, String> {
+public class DataPlatformType
+    implements SearchableEntityType<DataPlatform, String>, EntityType<DataPlatform, String> {
 
   private final EntityClient _entityClient;
 
@@ -75,4 +89,39 @@ public com.linkedin.datahub.graphql.generated.EntityType type() {
   public Function<Entity, String> getKeyProvider() {
     return Entity::getUrn;
   }
+
+  @Override
+  public SearchResults search(
+      @Nonnull String query,
+      @Nullable List<FacetFilterInput> filters,
+      int start,
+      int count,
+      @Nonnull final QueryContext context)
+      throws Exception {
+    final Map<String, String> facetFilters =
+        ResolverUtils.buildFacetFilters(filters, ImmutableSet.of());
+    final SearchResult searchResult =
+        _entityClient.search(
+            context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)),
+            DATA_PLATFORM_ENTITY_NAME,
+            query,
+            facetFilters,
+            start,
+            count);
+    return UrnSearchResultsMapper.map(context, searchResult);
+  }
+
+  @Override
+  public AutoCompleteResults autoComplete(
+      @Nonnull String query,
+      @Nullable String field,
+      @Nullable Filter filters,
+      int limit,
+      @Nonnull final QueryContext context)
+      throws Exception {
+    final AutoCompleteResult result =
+        _entityClient.autoComplete(
+            context.getOperationContext(), DATA_PLATFORM_ENTITY_NAME, query, filters, limit);
+    return AutoCompleteResultsMapper.map(context, result);
+  }
 }

datahub-graphql-core/src/main/resources/search.graphql

@@ -347,6 +347,11 @@ input ScrollAcrossEntitiesInput {
   Flags controlling search options
   """
   searchFlags: SearchFlags
+
+  """
+  Optional - Information on how to sort this search result
+  """
+  sortInput: SearchSortInput
 }