PGBouncer: Some tests

test-pool.sh

@@ -0,0 +1,270 @@
+#!/bin/bash
+######################################################################################
+# Bash Script for Bexhoma Test Runs - Compare PostgreSQL with and without PGBouncer
+######################################################################################
+#
+# This scripts starts a sequence of experiments with varying parameters.
+# Each experiment waits until previous tests have been completed.
+# Logs are written to a log folder.
+# At the end, logs are cleaned and the summaries are extracted and stored in separate files.
+#
+# Author: Patrick K. Erdelt
+# Email: patrick.erdelt@bht-berlin.de
+# Date: 2024-10-01
+# Version: 1.0
+######################################################################################
+
+
+BEXHOMA_NODE_SUT="cl-worker11"
+BEXHOMA_NODE_LOAD="cl-worker19"
+BEXHOMA_NODE_BENCHMARK="cl-worker19"
+LOG_DIR="./logs_tests"
+
+mkdir -p $LOG_DIR
+
+# Define the wait_process function
+wait_process() {
+    local process_name=$1
+
+    # Wait until the process with the name passed as an argument has terminated
+    while ps aux | grep "[p]ython $process_name.py" > /dev/null; do
+        # Process is still running, wait for 5 seconds
+        echo "$(date +"%Y-%m-%d %H:%M:%S"): Waiting for process python $process_name.py to terminate..."
+        sleep 60
+    done
+
+    echo "$(date +"%Y-%m-%d %H:%M:%S"): Process python $process_name.py has terminated."
+}
+
+# Example usage
+#wait_process "tpch"
+
+
+# Wait for all previous jobs to complete
+wait_process "tpch"
+wait_process "tpcds"
+wait_process "hammerdb"
+wait_process "benchbase"
+wait_process "ycsb"
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+####################################################
+################## YCSB PGBouncer ##################
+####################################################
+
+
+
+
+
+
+
+### YCSB Loader Test for Scaling the Driver (TestCases.md)
+nohup python ycsb.py -ms 1 -tr \
+  -sf 16 \
+  -sfo 1 \
+  --workload c \
+  -dbms PGBouncer PostgreSQL \
+  -rnn $BEXHOMA_NODE_SUT -rnl $BEXHOMA_NODE_LOAD -rnb $BEXHOMA_NODE_BENCHMARK \
+  -tb 131072 \
+  -nlp 4,16,32 \
+  -nlt 64 \
+  -nlf 1 \
+  -nbp 1 \
+  -nbt 64 \
+  -nbf 1 \
+  -ne 1 \
+  -nc 1 \
+  -m -mc \
+  run </dev/null &>$LOG_DIR/test_ycsb_testcase_pgbouncer_1.log &
+
+
+wait_process "ycsb"
+
+
+### YCSB Execution Test for Scaling the Driver (TestCases.md)
+nohup python ycsb.py -ms 1 -tr \
+  -sf 16 \
+  -sfo 16 \
+  --workload c \
+  -dbms PGBouncer PostgreSQL \
+  -rnn $BEXHOMA_NODE_SUT -rnl $BEXHOMA_NODE_LOAD -rnb $BEXHOMA_NODE_BENCHMARK \
+  -tb 131072 \
+  -nlp 8 \
+  -nlt 64 \
+  -nlf 1 \
+  -nbp 4,16,32 \
+  -nbt 64 \
+  -nbf 1 \
+  -ne 1 \
+  -nc 1 \
+  -m -mc \
+  run </dev/null &>$LOG_DIR/test_ycsb_testcase_pgbouncer_2.log &
+
+wait_process "ycsb"
+
+
+### YCSB Execution Test for Scaling the Driver (TestCases.md)
+nohup python ycsb.py -ms 1 -tr \
+  -sf 16 \
+  -sfo 16 \
+  --workload c \
+  -dbms PGBouncer PostgreSQL \
+  -rnn $BEXHOMA_NODE_SUT -rnl $BEXHOMA_NODE_LOAD -rnb $BEXHOMA_NODE_BENCHMARK \
+  -tb 131072 \
+  -nlp 8 \
+  -nlt 64 \
+  -nlf 1 \
+  -nbp 4,16,32 \
+  -nbt 64 \
+  -nbf 1 \
+  -ne 1 \
+  -nc 1 \
+  -rst shared -rss 50Gi \
+  -m -mc \
+  run </dev/null &>$LOG_DIR/test_ycsb_testcase_pgbouncer_3.log &
+
+wait_process "ycsb"
+
+
+
+
+
+### YCSB Execution Test for Scaling the Driver (TestCases.md)
+nohup python ycsb.py -ms 1 -tr -db \
+  -sf 16 \
+  -sfo 16 \
+  --workload c \
+  -dbms PGBouncer PostgreSQL \
+  -rnn $BEXHOMA_NODE_SUT -rnl $BEXHOMA_NODE_LOAD -rnb $BEXHOMA_NODE_BENCHMARK \
+  -tb 131072 \
+  -nlp 8 \
+  -nlt 64 \
+  -nlf 1 \
+  -nbp 8 \
+  -nbt 64 \
+  -nbf 1 \
+  -ne 1 \
+  -nc 1 \
+  -rst shared -rss 50Gi \
+  -m -mc \
+  run </dev/null &>$LOG_DIR/test_ycsb_testcase_pgbouncer_tmp.log &
+
+wait_process "ycsb"
+
+
+
+nohup python ycsb.py -ms 1 -tr -db \
+  -sf 16 \
+  -sfo 16 \
+  --workload c \
+  -dbms PGBouncer PostgreSQL \
+  -rnn $BEXHOMA_NODE_SUT -rnl $BEXHOMA_NODE_LOAD -rnb $BEXHOMA_NODE_BENCHMARK \
+  -tb 131072 \
+  -nlp 8 \
+  -nlt 64 \
+  -nlf 1 \
+  -nbp 8 \
+  -nbt 64 \
+  -nbf 1 \
+  -ne 1 \
+  -nc 1 \
+  -rst shared -rss 50Gi \
+  run </dev/null &>$LOG_DIR/test_ycsb_testcase_pgbouncer_tmp2.log &
+
+wait_process "ycsb"
+
+
+
+
+nohup python ycsb.py -ms 1 -tr -db \
+  -sf 16 \
+  -sfo 16 \
+  --workload c \
+  -dbms PGBouncer PostgreSQL \
+  -rnn $BEXHOMA_NODE_SUT -rnl $BEXHOMA_NODE_LOAD -rnb $BEXHOMA_NODE_BENCHMARK \
+  -tb 131072 \
+  -nlp 8 \
+  -nlt 64 \
+  -nlf 1 \
+  -nbp 1,2,4,8 \
+  -nbt 64 \
+  -nbf 1 \
+  -ne 1 \
+  -nc 1 \
+  -rst shared -rss 50Gi \
+  run </dev/null &>$LOG_DIR/test_ycsb_testcase_pgbouncer_tmp3.log &
+
+wait_process "ycsb"
+
+
+
+
+
+
+
+
+
+### YCSB Loader Test for Scaling the Driver (TestCases.md)
+nohup python ycsb.py -ms 1 -tr \
+  -sf 1 \
+  -sfo 1 \
+  --workload c \
+  -dbms PGBouncer PostgreSQL \
+  -rnn $BEXHOMA_NODE_SUT -rnl $BEXHOMA_NODE_LOAD -rnb $BEXHOMA_NODE_BENCHMARK \
+  -tb 131072 \
+  -nlp 8 \
+  -nlt 64 \
+  -nlf 1 \
+  -nbp 4,16,32 \
+  -nbt 64 \
+  -nbf 1 \
+  -ne 1 \
+  -nc 1 \
+  -rst shared -rss 30Gi \
+  run </dev/null &>$LOG_DIR/test_ycsb_testcase_pgbouncer_tmp.log &
+
+wait_process "ycsb"
+
+
+
+
+
+
+
+
+
+###########################################
+############## Clean Folder ###############
+###########################################
+
+
+
+export MYDIR=$(pwd)
+cd $LOG_DIR
+# remove connection errors from logs
+grep -rl "Warning: Use tokens from the TokenRequest API or manually created secret-based tokens instead of auto-generated secret-based tokens." . | xargs sed -i '/Warning: Use tokens from the TokenRequest API or manually created secret-based tokens instead of auto-generated secret-based tokens./d'
+cd $MYDIR
+
+# Loop over each text file in the source directory
+for file in "$LOG_DIR"/*.log; do
+    # Get the filename without the path and extension
+    echo "Cleaning $file"
+    filename=$(basename "$file" .log)
+    # Extract lines starting from "## Show Summary" and save as <filename>_summary.txt in the destination directory
+    awk '/## Show Summary/ {show=1} show {print}' "$file" > "$LOG_DIR/${filename}_summary.txt"
+done
+
+echo "Extraction complete! Files are saved in $LOG_DIR."
+