Skip to content

Commit

Permalink
Merge branch 'master' into mlflow-connector-latest
Browse files Browse the repository at this point in the history
  • Loading branch information
yoonhyejin authored Mar 4, 2025
2 parents e23d5f3 + 5576b3c commit e62f3df
Show file tree
Hide file tree
Showing 394 changed files with 14,821 additions and 3,888 deletions.
23 changes: 9 additions & 14 deletions .github/.codecov.yml
Original file line number Diff line number Diff line change
@@ -1,21 +1,19 @@
comment:
layout: "header, files, footer" # remove "new" from "header" and "footer"
hide_project_coverage: true # set to false
layout: "condensed_header, condensed_files, condensed_footer"
hide_project_coverage: true
require_changes: false # if true: only post the comment if coverage changes

codecov:
#due to ci-optimization, reports for modules that have not changed may be quite old
max_report_age: off

github_checks:
#Hide annotations that show up in github PR reviews. There still is a red bar next to lines not covered
annotations: false

flag_management:
default_rules: # the rules that will be followed for any flag added, generally
carryforward: true
statuses:
- type: project
target: auto
threshold: 0% #Not enforcing project coverage yet.
- type: patch
target: 90%
individual_flags: # exceptions to the default rules above, stated flag by flag
- name: frontend
paths:
Expand Down Expand Up @@ -55,11 +53,8 @@ flag_management:
- "metadata-ingestion-modules/prefect-plugin/**"
coverage:
status:
project:
default:
target: 0% # no threshold enforcement yet
only_pulls: true
project: false
patch:
default:
target: 90% # for new code added in the patch
only_pulls: true
target: 75% # for new code added in the patch
only_pulls: true
46 changes: 28 additions & 18 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,38 @@ export const Logo = (props) => {
<!--
HOSTED_DOCS_ONLY-->
<p align="center">
<a href="https://datahubproject.io">
<img alt="DataHub" src="https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/datahub-logo-color-mark.svg" height="150" />
</a>
</p>
<!-- -->

# DataHub: The Data Discovery Platform for the Modern Data Stack
## Built with ❤️ by <img src="https://datahubproject.io/img/acryl-logo-light-mark.png" width="25"/> [Acryl Data](https://acryldata.io) and <img src="https://datahubproject.io/img/LI-In-Bug.png" width="25"/> [LinkedIn](https://engineering.linkedin.com)
[![Version](https://img.shields.io/github/v/release/datahub-project/datahub?include_prereleases)](https://github.com/datahub-project/datahub/releases/latest)
[![PyPI version](https://badge.fury.io/py/acryl-datahub.svg)](https://badge.fury.io/py/acryl-datahub)
[![build & test](https://github.com/datahub-project/datahub/workflows/build%20&%20test/badge.svg?branch=master&event=push)](https://github.com/datahub-project/datahub/actions?query=workflow%3A%22build+%26+test%22+branch%3Amaster+event%3Apush)
[![Docker Pulls](https://img.shields.io/docker/pulls/acryldata/datahub-gms.svg)](https://hub.docker.com/r/acryldata/datahub-gms)
[![Slack](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme)
[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md)
[![GitHub commit activity](https://img.shields.io/github/commit-activity/m/datahub-project/datahub)](https://github.com/datahub-project/datahub/pulls?q=is%3Apr)
[![License](https://img.shields.io/github/license/datahub-project/datahub)](https://github.com/datahub-project/datahub/blob/master/LICENSE)
[![YouTube](https://img.shields.io/youtube/channel/subscribers/UC3qFQC5IiwR5fvWEqi_tJ5w?style=social)](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w)
[![Medium](https://img.shields.io/badge/Medium-12100E?style=for-the-badge&logo=medium&logoColor=white)](https://medium.com/datahub-project)
[![Follow](https://img.shields.io/twitter/follow/datahubproject?label=Follow&style=social)](https://twitter.com/datahubproject)
### 🏠 Hosted DataHub Docs (Courtesy of Acryl Data): [datahubproject.io](https://datahubproject.io/docs)

### Built with ❤️ by <img src="https://datahubproject.io/img/acryl-logo-light-mark.png" width="20"/> [Acryl Data](https://acryldata.io) and <img src="https://datahubproject.io/img/LI-In-Bug.png" width="20"/> [LinkedIn](https://engineering.linkedin.com)

<div>
<a target="_blank" href="https://github.com/datahub-project/datahub/blob/master/LICENSE">
<img alt="Apache 2.0 License" src="https://img.shields.io/badge/License-Apache_2.0-blue.svg?label=license&labelColor=133554&color=1890ff" /></a>
<a target="_blank" href="https://pypi.org/project/acryl-datahub/">
<img alt="PyPI" src="https://img.shields.io/pypi/dm/acryl-datahub?label=downloads&labelColor=133554&color=1890ff" /></a>
<a target="_blank" href="https://github.com/datahub-project/datahub/pulse">
<img alt="GitHub commit activity" src="https://img.shields.io/github/commit-activity/m/datahub-project/datahub?label=commits&labelColor=133554&color=1890ff" /></a>
<br />
<a target="_blank" href="https://pages.acryl.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme">
<img alt="Slack" src="https://img.shields.io/badge/slack-join_community-red.svg?logo=slack&labelColor=133554&color=1890ff" /></a>
<a href="https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w">
<img alt="YouTube" src="https://img.shields.io/youtube/channel/subscribers/UC3qFQC5IiwR5fvWEqi_tJ5w?style=flat&logo=youtube&label=subscribers&labelColor=133554&color=1890ff"/></a>
<a href="https://blog.datahubproject.io/">
<img alt="Medium" src="https://img.shields.io/badge/blog-DataHub-red.svg?style=flat&logo=medium&logoColor=white&labelColor=133554&color=1890ff" /></a>
<a href="https://x.com/datahubproject">
<img alt="X (formerly Twitter) Follow" src="https://img.shields.io/badge/follow-datahubproject-red.svg?style=flat&logo=x&labelColor=133554&color=1890ff" /></a>
</div>

---

### 🏠 Docs: [datahubproject.io](https://datahubproject.io/docs)

[Quickstart](https://datahubproject.io/docs/quickstart) |
[Features](https://datahubproject.io/docs/) |
[Roadmap](https://feature-requests.datahubproject.io/roadmap) |
Expand All @@ -47,6 +58,7 @@ HOSTED_DOCS_ONLY-->
[Town Hall](https://datahubproject.io/docs/townhalls)

---

> 📣 DataHub Town Hall is the 4th Thursday at 9am US PT of every month - [add it to your calendar!](https://rsvp.datahubproject.io/)
>
> - Town-hall Zoom link: [zoom.datahubproject.io](https://zoom.datahubproject.io)
Expand All @@ -70,11 +82,11 @@ Check out DataHub's [Features](docs/features.md) & [Roadmap](https://feature-req

## Demo and Screenshots

There's a [hosted demo environment](https://demo.datahubproject.io/) courtesy of [Acryl Data](https://acryldata.io) where you can explore DataHub without installing it locally
There's a [hosted demo environment](https://demo.datahubproject.io/) courtesy of [Acryl Data](https://acryldata.io) where you can explore DataHub without installing it locally.

## Quickstart

Please follow the [DataHub Quickstart Guide](https://datahubproject.io/docs/quickstart) to get a copy of DataHub up & running locally using [Docker](https://docker.com). As the guide assumes some basic knowledge of Docker, we'd recommend you to go through the "Hello World" example of [A Docker Tutorial for Beginners](https://docker-curriculum.com) if Docker is completely foreign to you.
Please follow the [DataHub Quickstart Guide](https://datahubproject.io/docs/quickstart) to run DataHub locally using [Docker](https://docker.com).

## Development

Expand Down Expand Up @@ -106,7 +118,7 @@ We welcome contributions from the community. Please refer to our [Contributing G

## Community

Join our [Slack workspace](https://datahubproject.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.
Join our [Slack workspace](https://pages.acryl.io/slack?utm_source=github&utm_medium=readme&utm_campaign=github_readme) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.

## Security

Expand Down Expand Up @@ -159,8 +171,6 @@ Here are the companies that have officially adopted DataHub. Please feel free to
- [Wolt](https://wolt.com)
- [Zynga](https://www.zynga.com)



## Select Articles & Talks

- [DataHub Blog](https://blog.datahubproject.io/)
Expand Down
40 changes: 40 additions & 0 deletions datahub-frontend/app/auth/GuestAuthenticationConfigs.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package auth;

public class GuestAuthenticationConfigs {
public static final String GUEST_ENABLED_CONFIG_PATH = "auth.guest.enabled";
public static final String GUEST_USER_CONFIG_PATH = "auth.guest.user";
public static final String GUEST_PATH_CONFIG_PATH = "auth.guest.path";
public static final String DEFAULT_GUEST_USER_NAME = "guest";
public static final String DEFAULT_GUEST_PATH = "/public";

private Boolean isEnabled = false;
private String guestUser =
DEFAULT_GUEST_USER_NAME; // Default if not defined but guest auth is enabled.
private String guestPath =
DEFAULT_GUEST_PATH; // The path for initial access to login as guest and bypass login page.

public GuestAuthenticationConfigs(final com.typesafe.config.Config configs) {
if (configs.hasPath(GUEST_ENABLED_CONFIG_PATH)
&& configs.getBoolean(GUEST_ENABLED_CONFIG_PATH)) {
isEnabled = true;
}
if (configs.hasPath(GUEST_USER_CONFIG_PATH)) {
guestUser = configs.getString(GUEST_USER_CONFIG_PATH);
}
if (configs.hasPath(GUEST_PATH_CONFIG_PATH)) {
guestPath = configs.getString(GUEST_PATH_CONFIG_PATH);
}
}

public boolean isGuestEnabled() {
return isEnabled;
}

public String getGuestUser() {
return guestUser;
}

public String getGuestPath() {
return guestPath;
}
}
21 changes: 21 additions & 0 deletions datahub-frontend/app/controllers/AuthenticationController.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import auth.AuthUtils;
import auth.CookieConfigs;
import auth.GuestAuthenticationConfigs;
import auth.JAASConfigs;
import auth.NativeAuthenticationConfigs;
import auth.sso.SsoManager;
Expand Down Expand Up @@ -58,6 +59,8 @@ public class AuthenticationController extends Controller {
private final CookieConfigs cookieConfigs;
private final JAASConfigs jaasConfigs;
private final NativeAuthenticationConfigs nativeAuthenticationConfigs;
private final GuestAuthenticationConfigs guestAuthenticationConfigs;

private final boolean verbose;

@Inject private org.pac4j.core.config.Config ssoConfig;
Expand All @@ -73,6 +76,7 @@ public AuthenticationController(@Nonnull Config configs) {
cookieConfigs = new CookieConfigs(configs);
jaasConfigs = new JAASConfigs(configs);
nativeAuthenticationConfigs = new NativeAuthenticationConfigs(configs);
guestAuthenticationConfigs = new GuestAuthenticationConfigs(configs);
verbose = configs.hasPath(AUTH_VERBOSE_LOGGING) && configs.getBoolean(AUTH_VERBOSE_LOGGING);
}

Expand Down Expand Up @@ -110,6 +114,23 @@ public Result authenticate(Http.Request request) {
return Results.redirect(redirectPath);
}

if (guestAuthenticationConfigs.isGuestEnabled()
&& guestAuthenticationConfigs.getGuestPath().equals(redirectPath)) {
final String accessToken =
authClient.generateSessionTokenForUser(guestAuthenticationConfigs.getGuestUser());
redirectPath =
"/"; // We requested guest login by accessing {guestPath} URL. It is not really a target.
CorpuserUrn guestUserUrn = new CorpuserUrn(guestAuthenticationConfigs.getGuestUser());
return Results.redirect(redirectPath)
.withSession(createSessionMap(guestUserUrn.toString(), accessToken))
.withCookies(
createActorCookie(
guestUserUrn.toString(),
cookieConfigs.getTtlInHours(),
cookieConfigs.getAuthCookieSameSite(),
cookieConfigs.getAuthCookieSecure()));
}

// 1. If SSO is enabled, redirect to IdP if not authenticated.
if (ssoManager.isSsoEnabled()) {
return redirectToIdentityProvider(request, redirectPath)
Expand Down
5 changes: 5 additions & 0 deletions datahub-frontend/conf/application.conf
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,11 @@ auth.oidc.grantType = ${?AUTH_OIDC_GRANT_TYPE}
#
auth.jaas.enabled = ${?AUTH_JAAS_ENABLED}
auth.native.enabled = ${?AUTH_NATIVE_ENABLED}
auth.guest.enabled = ${?GUEST_AUTHENTICATION_ENABLED}
# The name of the guest user id
auth.guest.user = ${?GUEST_AUTHENTICATION_USER}
# The path to bypass login page and get logged in as guest
auth.guest.path = ${?GUEST_AUTHENTICATION_PATH}

# Enforces the usage of a valid email for user sign up
auth.native.signUp.enforceValidEmail = true
Expand Down
61 changes: 61 additions & 0 deletions datahub-frontend/test/security/GuestAuthenticationConfigsTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package security;

import static org.junit.jupiter.api.Assertions.*;

import auth.GuestAuthenticationConfigs;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junitpioneer.jupiter.ClearEnvironmentVariable;
import org.junitpioneer.jupiter.SetEnvironmentVariable;

@TestInstance(TestInstance.Lifecycle.PER_METHOD)
@SetEnvironmentVariable(key = "DATAHUB_SECRET", value = "test")
@SetEnvironmentVariable(key = "KAFKA_BOOTSTRAP_SERVER", value = "")
@SetEnvironmentVariable(key = "DATAHUB_ANALYTICS_ENABLED", value = "false")
@SetEnvironmentVariable(key = "AUTH_OIDC_ENABLED", value = "true")
@SetEnvironmentVariable(key = "AUTH_OIDC_JIT_PROVISIONING_ENABLED", value = "false")
@SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_ID", value = "testclient")
@SetEnvironmentVariable(key = "AUTH_OIDC_CLIENT_SECRET", value = "testsecret")
@SetEnvironmentVariable(key = "AUTH_VERBOSE_LOGGING", value = "true")
class GuestAuthenticationConfigsTest {

@BeforeEach
@ClearEnvironmentVariable(key = "GUEST_AUTHENTICATION_ENABLED")
@ClearEnvironmentVariable(key = "GUEST_AUTHENTICATION_USER")
@ClearEnvironmentVariable(key = "GUEST_AUTHENTICATION_PATH")
public void clearConfigCache() {
ConfigFactory.invalidateCaches();
}

@Test
public void testGuestConfigDisabled() {
Config config = ConfigFactory.load();
GuestAuthenticationConfigs guestAuthConfig = new GuestAuthenticationConfigs(config);
assertFalse(guestAuthConfig.isGuestEnabled());
}

@Test
@SetEnvironmentVariable(key = "GUEST_AUTHENTICATION_ENABLED", value = "true")
public void testGuestConfigEnabled() {
Config config = ConfigFactory.load();
GuestAuthenticationConfigs guestAuthConfig = new GuestAuthenticationConfigs(config);
assertTrue(guestAuthConfig.isGuestEnabled());
assertEquals("guest", guestAuthConfig.getGuestUser());
assertEquals("/public", guestAuthConfig.getGuestPath());
}

@Test
@SetEnvironmentVariable(key = "GUEST_AUTHENTICATION_ENABLED", value = "true")
@SetEnvironmentVariable(key = "GUEST_AUTHENTICATION_USER", value = "publicUser")
@SetEnvironmentVariable(key = "GUEST_AUTHENTICATION_PATH", value = "/publicPath")
public void testGuestConfigWithUserEnabled() {
Config config = ConfigFactory.load();
GuestAuthenticationConfigs guestAuthConfig = new GuestAuthenticationConfigs(config);
assertTrue(guestAuthConfig.isGuestEnabled());
assertEquals("publicUser", guestAuthConfig.getGuestUser());
assertEquals("/publicPath", guestAuthConfig.getGuestPath());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1026,7 +1026,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
new ScrollAcrossEntitiesResolver(this.entityClient, this.viewService))
.dataFetcher(
"searchAcrossLineage",
new SearchAcrossLineageResolver(this.entityClient, this.entityRegistry))
new SearchAcrossLineageResolver(
this.entityClient, this.entityRegistry, this.viewService))
.dataFetcher(
"scrollAcrossLineage", new ScrollAcrossLineageResolver(this.entityClient))
.dataFetcher(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ private List<AnalyticsChart> getCharts(MetadataAnalyticsInput input, OperationCo

SearchResult searchResult =
_entityClient.searchAcrossEntities(
opContext, entities, query, filter, 0, 0, Collections.emptyList(), null);
opContext, entities, query, filter, 0, 0, Collections.emptyList());

List<AggregationMetadata> aggregationMetadataList =
searchResult.getMetadata().getAggregations();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
new CriterionArray(ImmutableList.of(filterCriterion))))),
start,
count,
Collections.emptyList(),
null));
Collections.emptyList()));

} catch (Exception e) {
throw new RuntimeException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,6 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
finalFilter,
start,
count,
null,
null));
results
.getSearchResults()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
new ConjunctiveCriterion().setAnd(criteria))),
start,
count,
Collections.emptyList(),
null));
Collections.emptyList()));

} catch (Exception e) {
throw new RuntimeException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -222,16 +222,15 @@ private void checkLineageEdgePrivileges(
if (!isAuthorized(context, upstreamUrn, editLineagePrivileges)) {
throw new AuthorizationException(
String.format(
"Unauthorized to edit %s lineage. Please contact your DataHub administrator.",
upstreamUrn.getEntityType()));
"Unauthorized to edit %s lineage for %s", upstreamUrn, upstreamUrn.getEntityType()));
}

Urn downstreamUrn = UrnUtils.getUrn(lineageEdge.getDownstreamUrn());
if (!isAuthorized(context, downstreamUrn, editLineagePrivileges)) {
throw new AuthorizationException(
String.format(
"Unauthorized to edit %s lineage. Please contact your DataHub administrator.",
downstreamUrn.getEntityType()));
"Unauthorized to edit %s lineage for %s",
downstreamUrn, downstreamUrn.getEntityType()));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,6 @@ public CompletableFuture<Privileges> get(final DataFetchingEnvironment environme
}

private boolean isAuthorized(final QueryContext context, final String actor) {
return actor.equals(context.getActorUrn());
return PolicyAuthUtils.canManagePolicies(context) || actor.equals(context.getActorUrn());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,7 @@ private SearchResult getSearchResults(
: null,
0,
0,
Collections.emptyList(),
null);
Collections.emptyList());
}

/**
Expand Down
Loading

0 comments on commit e62f3df

Please sign in to comment.