Stronger import script

scripts/import_from_relay.sh

@@ -24,30 +24,62 @@ record_created_at() {
   done
 }
 
+previous_p_value=""
+last_p_value=""
 filter_events() {
-  jq -cr 'select(
-    (.tags | length > 1)
-    or
-    (.tags[0][0] != "p" or .tags[0][1] != "0497384b57b43c107a778870462901bf68e0e8583b32e2816563543c059784a4")
-  )'
+  while read -r event; do
+    current_p_value=$(echo "$event" | jq -r '.tags[0][1]')
+
+    # Skip if the current p value is the same as the last p value stored
+    if [[ "$current_p_value" == "$last_p_value" ]]; then
+      continue  # Probably spam, skip this event
+    fi
+
+    # Update the last seen p value
+    previous_p_value="$last_p_value"
+    last_p_value="$current_p_value"
+
+    # Apply the filtering logic
+    echo "$event" | jq -cr 'select(
+      (.tags | length > 1) or
+      (
+        .tags[0][0] != "p" or
+        .tags[0][1] != "0497384b57b43c107a778870462901bf68e0e8583b32e2816563543c059784a4" or
+        .tags[0][1] != "4bc7982c4ee4078b2ada5340ae673f18d3b6a664b1f97e8d6799e6074cb5c39d"
+      )
+    )'
+  done
 }
 
 
 # Infinite loop
+decrease_secs=2
 while true; do
   if [ -f "$created_at_file" ]; then
-    initial_date=$(cat "$created_at_file")
-    until_option="--until $initial_date"
+    current_date=$(cat "$created_at_file")
+    until_option="--until $current_date"
   else
     until_option=""
   fi
 
-  # Tee before jq filter to record all dates, even skipped entries
+  echo "Iteration starting at: $current_date"
+
   nak req -k 3 $until_option --paginate --paginate-interval ${wait_seconds}s "$relay_url" \
-  | filter_events \
-  | tee >(record_created_at) \
-  | nc localhost 3001
+    | filter_events \
+    | tee >(record_created_at) \
+    | nc localhost 3001
 
-  echo "Command failed or completed, waiting for $wait_seconds seconds before retrying..." >&2
-  sleep $wait_seconds
+
+  previous_date="$current_date"
+  current_date=$(cat "$created_at_file")
+
+  if [ "$current_date" == "$previous_date" ] && [ "$last_p_value" == "$previous_p_value" ]; then
+    echo "Reducing time by $decrease_secs seconds"
+
+    current_date=$((current_date - decrease_secs))
+    echo "$current_date" > "$created_at_file"
+    decrease_secs=$((decrease_secs * 2))
+  else
+    decrease_secs=2
+  fi
 done