diff --git a/scripts/import_from_relay.sh b/scripts/import_from_relay.sh index c552589..bf819a2 100755 --- a/scripts/import_from_relay.sh +++ b/scripts/import_from_relay.sh @@ -24,30 +24,62 @@ record_created_at() { done } +previous_p_value="" +last_p_value="" filter_events() { - jq -cr 'select( - (.tags | length > 1) - or - (.tags[0][0] != "p" or .tags[0][1] != "0497384b57b43c107a778870462901bf68e0e8583b32e2816563543c059784a4") - )' + while read -r event; do + current_p_value=$(echo "$event" | jq -r '.tags[0][1]') + + # Skip if the current p value is the same as the last p value stored + if [[ "$current_p_value" == "$last_p_value" ]]; then + continue # Probably spam, skip this event + fi + + # Update the last seen p value + previous_p_value="$last_p_value" + last_p_value="$current_p_value" + + # Apply the filtering logic + echo "$event" | jq -cr 'select( + (.tags | length > 1) or + ( + .tags[0][0] != "p" or + .tags[0][1] != "0497384b57b43c107a778870462901bf68e0e8583b32e2816563543c059784a4" or + .tags[0][1] != "4bc7982c4ee4078b2ada5340ae673f18d3b6a664b1f97e8d6799e6074cb5c39d" + ) + )' + done } # Infinite loop +decrease_secs=2 while true; do if [ -f "$created_at_file" ]; then - initial_date=$(cat "$created_at_file") - until_option="--until $initial_date" + current_date=$(cat "$created_at_file") + until_option="--until $current_date" else until_option="" fi - # Tee before jq filter to record all dates, even skipped entries + echo "Iteration starting at: $current_date" + nak req -k 3 $until_option --paginate --paginate-interval ${wait_seconds}s "$relay_url" \ - | filter_events \ - | tee >(record_created_at) \ - | nc localhost 3001 + | filter_events \ + | tee >(record_created_at) \ + | nc localhost 3001 - echo "Command failed or completed, waiting for $wait_seconds seconds before retrying..." >&2 - sleep $wait_seconds + + previous_date="$current_date" + current_date=$(cat "$created_at_file") + + if [ "$current_date" == "$previous_date" ] && [ "$last_p_value" == "$previous_p_value" ]; then + echo "Reducing time by $decrease_secs seconds" + + current_date=$((current_date - decrease_secs)) + echo "$current_date" > "$created_at_file" + decrease_secs=$((decrease_secs * 2)) + else + decrease_secs=2 + fi done