Fix and refactor browser page provenance issue (#1757)

The name of a provenance triple is changed from url to the import name.
Browser fetches all provenance and url, so the logic needs to be updated
to fetch the url.

This PR also updates to use V1 API for triples fetch. This is only used
in browser page.

server/routes/api/browser.py

@@ -20,21 +20,34 @@
 import services.datacommons as dc
 from flask import Response
 from flask import request
-from flask import current_app
 import routes.api.place as place_api
-import logging
 
 bp = flask.Blueprint('api.browser', __name__, url_prefix='/api/browser')
 
 NO_MMETHOD_KEY = 'no_mmethod'
 NO_OBSPERIOD_KEY = 'no_obsPeriod'
 
 
-@cache.memoize(timeout=3600 * 24)  # Cache for one day.
-@bp.route('/triples/<path:dcid>')
-def triple_api(dcid):
+@bp.route('/triples/<path:direction>/<path:dcid>')
+def triples(direction, dcid):
     """Returns all the triples given a node dcid."""
-    return json.dumps(dc.get_triples([dcid]).get(dcid, []))
+    if direction != "in" and direction != "out":
+        return "Invalid direction provided, please use 'in' or 'out'", 400
+    return dc.triples(dcid, direction).get("triples")
+
+
+@bp.route('/provenance')
+def provenance():
+    """Returns all the provenance information."""
+    resp = dc.triples("Provenance", "in")
+    prov_list = resp.get("triples", {}).get("typeOf", {}).get("nodes", [])
+    dcids = map(lambda item: item["dcid"], prov_list)
+    resp = dc.property_values(dcids, "url", "out")
+    result = {}
+    for dcid, urls in resp.items():
+        if len(urls) > 0:
+            result[dcid] = urls[0]
+    return result
 
 
 @cache.memoize(timeout=3600 * 24)  # Cache for one day.

server/services/datacommons.py

@@ -38,7 +38,6 @@
     'search': '/search',
     'get_property_labels': '/node/property-labels',
     'get_property_values': '/node/property-values',
-    'get_triples': '/node/triples',
     'get_places_in': '/node/places-in',
     'get_place_ranking': '/node/ranking-locations',
     'get_stat_set_series': '/v1/stat/set/series',
@@ -161,12 +160,21 @@ def series_within(parent_place, child_type, stat_vars, all):
         })
 
 
+def triples(node, direction):
+    """Retrieves the triples for a node.
+    Args:
+        node: Node DCID.
+        direction: Predicate direction, either be 'in' or 'out'.
+    """
+    return get(f'/v1/triples/{direction}/{node}')
+
+
 def property_values(nodes, prop, direction):
     """Retrieves the property values for a list of nodes.
     Args:
         nodes: A list of node DCIDs.
         prop: The property label toquery for.
-        dir: Direction of the property, either be 'in' or 'out'.
+        direction: Direction of the property, either be 'in' or 'out'.
     """
     resp = post(f'/v1/bulk/property/values/{direction}', {
         'nodes': sorted(nodes),
@@ -395,17 +403,6 @@ def get_property_values(dcids,
     return results
 
 
-def get_triples(dcids, limit=0):
-    """
-    Get the triples in the raw format as the REST response.
-
-    This is used by the flask server to retrieve node triples.
-    Limit of 0 does not apply a limit and use all available triples from cache.
-    """
-    url = API_ROOT + API_ENDPOINTS['get_triples']
-    return send_request(url, req_json={'dcids': dcids, 'limit': limit})
-
-
 def get_places_in(dcids, place_type):
     # Convert the dcids field and format the request to GetPlacesIn
     url = API_ROOT + API_ENDPOINTS['get_places_in']

static/js/browser/browser_page.tsx

@@ -176,31 +176,22 @@ export class BrowserPage extends React.Component<
       : this.props.dcid;
   }
 
-  private isProvenanceEntity(prov): boolean {
-    return (
-      prov["subjectTypes"] &&
-      prov["subjectTypes"][0] === "Provenance" &&
-      !!prov["subjectName"]
-    );
-  }
-
   private fetchData(): void {
     const provenancePromise = axios
-      .get("/api/browser/triples/Provenance")
+      .get("/api/browser/provenance")
       .then((resp) => resp.data);
     const labelsPromise = axios
       .get("/api/browser/proplabels/" + this.getArcDcid())
       .then((resp) => resp.data);
     Promise.all([labelsPromise, provenancePromise])
-      .then(([labelsData, ProvenanceData]) => {
+      .then(([labelsData, provenanceData]) => {
         const provDomain = {};
-        for (const prov of ProvenanceData) {
-          if (this.isProvenanceEntity(prov)) {
-            try {
-              provDomain[prov["subjectId"]] = new URL(prov["subjectName"]).host;
-            } catch (err) {
-              console.log("Invalid url in prov: " + prov["subjectName"]);
-            }
+        for (const provId in provenanceData) {
+          const url = provenanceData[provId];
+          try {
+            provDomain[provId] = new URL(url).host;
+          } catch (err) {
+            console.log("Invalid url in prov: " + url);
           }
         }
         this.setState({

static/js/browser/out_arc_section.tsx

@@ -220,38 +220,34 @@ export class OutArcSection extends React.Component<
   private fetchDataFromTriples(): void {
     loadSpinner(LOADING_CONTAINER_ID);
     axios
-      .get("/api/browser/triples/" + this.props.dcid)
+      .get("/api/browser/triples/out/" + this.props.dcid)
       .then((resp) => {
         const triplesData = resp.data;
-        const outArcs = triplesData.filter(
-          (t) => t.subjectId === this.props.dcid
-        );
         const outArcsByPredProv: OutArcData = {};
-        for (const outArc of outArcs) {
-          const predicate = outArc.predicate;
-          if (IGNORED_OUT_ARC_PROPERTIES.has(predicate)) {
+        for (const pred in triplesData) {
+          if (IGNORED_OUT_ARC_PROPERTIES.has(pred)) {
             continue;
           }
-          if (!outArcsByPredProv[predicate]) {
-            outArcsByPredProv[predicate] = {};
-          }
-          const outArcsOfPredicate = outArcsByPredProv[predicate];
-          const provId = outArc.provenanceId;
-          if (!(provId in outArcsOfPredicate)) {
-            outArcsOfPredicate[provId] = [];
-          }
-          let valueText = "";
-          let valueDcid: string;
-          if (outArc.objectId) {
-            valueText = outArc.objectName ? outArc.objectName : outArc.objectId;
-            valueDcid = outArc.objectId;
-          } else {
-            valueText = outArc.objectValue;
+          const predData = {};
+          for (const node of triplesData[pred]["nodes"]) {
+            const provId = node["provenanceId"];
+            if (!(provId in predData)) {
+              predData[provId] = [];
+            }
+            let valueText = "";
+            let valueDcid: string;
+            if (node["dcid"]) {
+              valueText = node["name"] ? node["name"] : node["dcid"];
+              valueDcid = node["dcid"];
+            } else {
+              valueText = node["value"];
+            }
+            predData[provId].push({
+              dcid: valueDcid,
+              text: valueText,
+            });
           }
-          outArcsOfPredicate[provId].push({
-            dcid: valueDcid,
-            text: valueText,
-          });
+          outArcsByPredProv[pred] = predData;
         }
         removeSpinner(LOADING_CONTAINER_ID);
         this.setState({