Merge pull request #62 from CCBR/datashare

refactor: save reports to datashare

.gitignore

@@ -11,6 +11,7 @@
 
 # data
 /data/
+/datashare/
 
 # R
 .Rproj.user

bin/disk_usage.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+today=`date +"%Y-%m-%d %H:%M:%S"`
+
+df /data/CCBR |\
+  awk -v today="$today" 'NR==1{$(NF+1)="datetime"} NR>1{$(NF+1)=today}1' |\
+  sed -E 's/Mounted on/Mounted_on/' |\
+  sed -E 's/ +/\t/g' |\
+  tail -n 1 \
+  >> results/disk_usage.tsv

bin/render.R

@@ -1,5 +1,5 @@
 #!/usr/bin/env Rscript
 rmarkdown::render("report.Rmd",
-  output_file = "docs/report.html",
+  output_file = "datashare/report.html",
   params = list(input_dir = "data")
 )

bin/render.sh

@@ -4,6 +4,7 @@
 module load singularity
 SINGULARITY_CACHEDIR=/data/CCBR_Pipeliner/SIFS
 
+# render report
 echo "cd /mnt && \
     Rscript bin/render.R \
     " |\

bin/render_report_biowulf.sh

@@ -6,19 +6,23 @@ SINGULARITY_CACHEDIR=/data/CCBR_Pipeliner/SIFS
 
 today=$(date +'%Y-%m-%d')
 year=$(date +'%Y')
-mkdir -p docs/$year
-html_filename="docs/${year}/spacesavers2-report_${today}.html"
+mkdir -p datashare/$year
+html_filename="datashare/${year}/spacesavers2-report_${today}.html"
 recipient_email="kelly.sovacool@nih.gov,vishal.koparde@nih.gov"
 
+url=https://hpc.nih.gov/~CCBR_Pipeliner/spacesavers2/${year}/spacesavers2-report_${today}.html
+
+# update disk usage
+bash bin/disk_usage.sh
+# render report and send via email
 echo "cd /mnt && \
     Rscript bin/render.R && \
-    cp docs/report.html $html_filename && \
+    cp datashare/report.html $html_filename && \
     python src/send_email.py \
         $html_filename \
+        $url \
         $recipient_email \
     " |\
     singularity exec -C -B $PWD:/mnt,/data/CCBR_Pipeliner/userdata/spacesavers2/:/mnt/data docker://nciccbr/spacesavers2:0.1.1 bash
 
-git add docs
-git commit -m 'chore: render report 🤖'
-git push
+cp -r datashare/* /data/CCBR_Pipeliner/datashare/spacesavers2/

report.Rmd

@@ -13,14 +13,12 @@ params:
   input_dir: '/data/CCBR_Pipeliner/userdata/spacesavers2/'
   n_top_users: 10
 knit: (function(inputFile, encoding) {
-  rmarkdown::render(inputFile, encoding = encoding, output_dir = "docs/") })
+  rmarkdown::render(inputFile, encoding = encoding, output_dir = "datashare/") })
 ---
 ```{r setup}
 knitr::opts_chunk$set(message = FALSE, warning = FALSE)
 ```
 
-View this report on the web: `r glue::glue("<https://ccbr.github.io/spacesavers2/{lubridate::year(lubridate::today())}/spacesavers2-report_{lubridate::today()}.html>")`
-
 Notice a bug or want to make a suggestion for this report? [Open an issue](https://github.com/CCBR/spacesavers2/issues) on GitHub.
 
 ```{r load}
@@ -252,17 +250,6 @@ summary_metrics <- summary_dat_recent %>%
 
 ## Total disk usage
 
-```{sh df, eval=dir.exists('/data/CCBR'), echo = FALSE}
-today=`date +"%Y-%m-%d %H:%M:%S"`
-df /data/CCBR |\
-  awk -v today="$today" 'NR==1{$(NF+1)="datetime"} NR>1{$(NF+1)=today}1' |\
-  sed -E 's/Mounted on/Mounted_on/' |\
-  sed -E 's/ +/\t/g' |\
-  # remove header and append to keep track over time
-  tail -n 1 |\
-  >> results/disk_usage.tsv
-```
-
 ```{r disk_usage_latest}
 disk_usage <- read_tsv(here("results", "disk_usage.tsv")) %>% 
       mutate(used_tib = from_bytes(to_bytes(Used, "KiB"), "TiB"),

src/send_email.py

@@ -3,11 +3,11 @@
 """ Email the html report
 
 Usage:
-    python src/send_email.py <report_html> <recipient_emails>
+    python src/send_email.py <report_html> <report_url> <recipient_emails>
 
 Example:
-    python src/send_email.py docs/report.html kelly.sovacool@nih.gov
-    python src/send_email.py docs/2024/report_2024-01-17.html kelly.sovacool@nih.gov,vishal.koparde@nih.gov
+    python src/send_email.py docs/report.html https://hpc.nih.gov/~CCBR_Pipeliner/spacesavers2/report.html kelly.sovacool@nih.gov
+    python src/send_email.py docs/2024/spacesavers2-report_2024-01-17.html https://hpc.nih.gov/~CCBR_Pipeliner/spacesavers2/2024/spacesavers2-report_2024-01-17.html kelly.sovacool@nih.gov,vishal.koparde@nih.gov
 
 """
 
@@ -44,11 +44,15 @@ def send_email(
 
 
 if __name__ == "__main__":
+    # TODO switch to click for argument parsing if this starts to get any more complicated
     html_filename = sys.argv[1] if len(sys.argv) > 1 else ''
-    recipient_addr = sys.argv[2] if len(sys.argv) > 2 else 'kelly.sovacool@nih.gov'
+    url = sys.argv[2] if len(sys.argv) > 2 else ''
+    recipient_addr = sys.argv[3] if len(sys.argv) > 3 else 'kelly.sovacool@nih.gov'
+
+    download_text = f"Download the attached report or from {url}\n" if url else ''
     send_email(
         subject=f"🚀 spacesavers2 report",
         recipient=recipient_addr,
-        plain_text = f"Download the attached report or view it at https://ccbr.github.io/spacesavers2/{html_filename.strip('docs/')}\n\nThis is an automated email.",
+        plain_text = f"{download_text}\n\nThis is an automated email.",
         html_attach = html_filename
     )