Better download error handling

telemetry-dispatch-setup/action.yml

@@ -5,6 +5,8 @@ description: |
   current job, so that this metadata can be associated with spans during the final
   parsing of job metadata.
 
+  Obtains GitHub Actions job list and matches current job using runner name and attempt number.
+
   This action should be called at the beginning of child workflows, generally as the first
   step in any job other than computing the matrix.
 
@@ -15,13 +17,29 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - uses: rapidsai/shared-actions/telemetry-impls/load-then-clone@main
-    # overrides loaded value
+    - uses: rapidsai/shared-actions/telemetry-impls/load-then-clone@better-download-error-handling
+    - name: Creating folder for job-created telemetry artifacts
+      shell: bash
+      run: mkdir -p telemetry-artifacts
+    - uses: ./shared-actions/telemetry-impls/github-actions-job-info
+    - shell: bash
+      run:
+        echo JOB_ID="$(cat job_info.json | jq -r '.id')" >> ${GITHUB_ENV};
+    # overrides loaded value.
     - name: Set OTEL_SERVICE_NAME from job
       uses: ./shared-actions/telemetry-impls/set-otel-service-name
-    - name: Store attributes to use as metadata when creating spans
-      # This also sets OTEL_RESOURCE_ATTRIBUTES, for any subsequent steps
-      # in the calling workflow that might use it.
-      uses: ./shared-actions/telemetry-impls/stash-job-attributes
-      with:
-        extra_attributes: ${{ inputs.extra_attributes }}
+    - name: Add attribute metadata beyond the stashed basic stuff
+      shell: bash
+      run:
+        attributes="${OTEL_RESOURCE_ATTRIBUTES}";
+        labels="$(jq -r '.labels | join(" ")' job_info.json)";
+        if [ "${labels}" != "" ]; then
+          attributes="${attributes},rapids.labels=${labels}";
+        fi;
+        if [ "${{ inputs.extra_attributes }}" != "" ]; then
+          attributes="${attributes},${{ inputs.extra_attributes }}";
+        fi;
+        attributes=$(echo "${attributes}" | sed 's/^,//');
+        attributes=$(echo "${attributes}" | sed 's/,$//');
+        attributes=$(echo "${attributes}" | sed -r "s/(git.job_url=[^,]+)/\1\/job\/${JOB_ID}/");
+        echo OTEL_RESOURCE_ATTRIBUTES="${attributes}" >> ${GITHUB_ENV};

telemetry-dispatch-stash-job-artifacts/action.yml

@@ -0,0 +1,16 @@
+name: dispatch-stash-job-artifacts
+description: |
+  Clones a particular branch/ref of a shared-actions repo, then
+  call the stash-artifacts implementation script, which writes
+  some environment variables so that downstream jobs can refer to them.
+
+  Inputs here are all assumed to be env vars set outside of this script.
+  Set them in your main repo's workflows (export to ${GITHUB_ENV}!!)
+
+runs:
+  using: 'composite'
+  steps:
+    - uses: rapidsai/shared-actions/telemetry-impls/load-then-clone@better-download-error-handling
+    # Stash current job's OTEL_RESOURCE_ATTRIBUTES and any files in the telemetry-artifacts directory
+    - name: Stash job artifacts
+      uses: ./shared-actions/telemetry-impls/stash-job-artifacts

telemetry-dispatch-summarize/action.yml

@@ -3,18 +3,13 @@ description: |
   This action is run in a final job on the top-level workflow, after all other
   jobs are completed. This action downloads the JSON records of all jobs from
   the current run. It then associates metadata records that were uploaded with
-  the telemetry-dispatch-stash-job-attributes action with jobs. This is
+  the telemetry-dispatch-stash-job-artifacts action with jobs. This is
   effectively label metadata. Finally, this action creates OpenTelemetry spans
   with the timing and label metadata, and sends it to the configured Tempo
   endpoint (or forwarder).
 
 runs:
   using: 'composite'
   steps:
-    - uses: rapidsai/shared-actions/telemetry-impls/load-then-clone@main
-      if: ${{ github.run_attempt == '1' }}
+    - uses: rapidsai/shared-actions/telemetry-impls/load-then-clone@better-download-error-handling
     - uses: ./shared-actions/telemetry-impls/summarize
-      if: ${{ github.run_attempt == '1' }}
-    - shell: bash
-      run: echo "Skipping telemetry summary on rerun jobs."
-      if: ${{ github.run_attempt != '1' }}

telemetry-impls/load-base-env-vars/action.yml

@@ -9,10 +9,6 @@ description: |
 runs:
   using: 'composite'
   steps:
-    - name: Download base environment variables file
-      uses: actions/download-artifact@v4
-      with:
-        name: telemetry-tools-env-vars
     - name: Set environment variables from file into GITHUB_ENV
       shell: bash
       # Only set the env var if it is not already set

telemetry-impls/load-then-clone/action.yml

@@ -21,6 +21,7 @@ runs:
       uses: actions/download-artifact@v4
       with:
         name: telemetry-tools-env-vars
+        path: telemetry-artifacts
     # We can't use ./telemetry-implementation/load-base-env-vars here
     # because at this point we have not cloned the repo.
     - name: Set environment variables from file into GITHUB_ENV
@@ -38,10 +39,10 @@ runs:
           else
             echo "Load base env info: ignoring new value for "${env_var_name}" in loading base env vars. It is already set to "${!env_var_name}"." >&2;
           fi
-        done <telemetry-env-vars
+        done <telemetry-artifacts/telemetry-env-vars
     - name: Clone shared-actions repo with loaded env vars
       uses: actions/checkout@v4
       with:
-        repository: ${{ env.SHARED_ACTIONS_REPO }}
-        ref: ${{ env.SHARED_ACTIONS_REF }}
+        repository: ${{ env.SHARED_ACTIONS_REPO || 'rapidsai/shared-actions' }}
+        ref: ${{ env.SHARED_ACTIONS_REF || 'main' }}
         path: ./shared-actions

telemetry-impls/stash-job-artifacts/action.yml

@@ -0,0 +1,23 @@
+
+name: stash-job-artifacts
+description: |
+    Saves and uploads a file with telemetry attributes that should be attached to spans from this run.
+
+    We stash only the attributes here because we retrieve the rest of the timing
+    info later. We get info for all jobs at once, so we wait to retrieve that info
+    at the very end of the top-level job.
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Write attributes to file, one per line
+      shell: bash
+      run:
+        IFS=, read -ra values <<< "$OTEL_RESOURCE_ATTRIBUTES";
+        printf "%s\n" "${values[@]}" > telemetry-artifacts/attrs;
+
+    - name: Upload attr file and any other files
+      uses: actions/upload-artifact@v4
+      with:
+        name: telemetry-tools-artifacts-${{ env.JOB_ID }}
+        path: telemetry-artifacts

telemetry-impls/summarize/action.yml

@@ -30,11 +30,12 @@ runs:
       with:
         path: telemetry-artifacts
         pattern: telemetry-tools-*
-        merge-multiple: true
+        merge-multiple: false
 
     - name: Run parse and send trace/spans to endpoint
       shell: bash
       run: |
+        ls -lR telemetry-artifacts
         timeout 5m python3 ./shared-actions/telemetry-impls/summarize/send_trace.py
     - name: Clean up attributes artifacts from all jobs
       uses: ./shared-actions/telemetry-impls/clean-up-artifacts