Merge branch 'main' into contributors

Author: DonnieBLT

website/admin.py

@@ -1,6 +1,7 @@
 from urllib.parse import urlparse
 
 from django.contrib import admin
+from django.contrib.admin import SimpleListFilter
 from django.contrib.auth.admin import UserAdmin
 from django.contrib.auth.models import User
 from django.template.defaultfilters import truncatechars
@@ -400,6 +401,31 @@ def unblock_user_agent(modeladmin, request, queryset):
 unblock_user_agent.short_description = "Unblock selected UserAgent"
 
 
+# Custom filter for IP address ranges
+class IPAddressRangeFilter(SimpleListFilter):
+    title = "IP Address Range"
+    parameter_name = "ip_range"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("internal", "Internal (127.0.0.1)"),
+            ("local", "Local (192.168.x.x)"),
+            ("vpn", "VPN (10.x.x.x)"),
+            ("ipv6", "IPv6"),
+        )
+
+    def queryset(self, request, queryset):
+        if self.value() == "internal":
+            return queryset.filter(address__startswith="127.0.0.1")
+        if self.value() == "local":
+            return queryset.filter(address__startswith="192.168.")
+        if self.value() == "vpn":
+            return queryset.filter(address__startswith="10.")
+        if self.value() == "ipv6":
+            return queryset.filter(address__contains=":")
+        return queryset
+
+
 class IPAdmin(admin.ModelAdmin):
     list_display = (
         "id",
@@ -414,6 +440,10 @@ class IPAdmin(admin.ModelAdmin):
         "referer",
     )
 
+    search_fields = ["address", "user", "agent", "path", "method", "referer"]
+    list_filter = ["method", "created", IPAddressRangeFilter]
+    date_hierarchy = "created"
+
     actions = [block_ip, unblock_ip, block_user_agent, unblock_user_agent]
 
 
@@ -554,6 +584,7 @@ class GitHubIssueAdmin(admin.ModelAdmin):
         "is_merged",
         "user_profile",
         "sent_by_user",
+        "repo",
     ]
     search_fields = [
         "title",

website/management/commands/fetch_gsoc_prs.py

@@ -41,12 +41,18 @@ def add_arguments(self, parser):
             default=None,
             help="Comma-separated list of repositories to process (e.g., 'OWASP-BLT/BLT,OWASP-BLT/BLT-Flutter')",
         )
+        parser.add_argument(
+            "--reset",
+            action="store_true",
+            help="Reset the last_pr_page_processed counter and start from the beginning",
+        )
 
     def handle(self, *args, **options):
         days = options["days"]
         limit = options["limit"]
         verbose = True  # Always use verbose mode for debugging
         repos_arg = options["repos"]
+        reset = options["reset"]
 
         self.stdout.write(f"Fetching closed PRs from the past {days} days for GSoC repositories")
 
@@ -72,6 +78,7 @@ def handle(self, *args, **options):
 
         total_prs_fetched = 0
         total_prs_added = 0
+        total_prs_updated = 0
 
         for repo_full_name in all_repos:
             try:
@@ -80,16 +87,23 @@ def handle(self, *args, **options):
                 # Check if the repository exists in our database
                 repo = self.get_or_create_repo(owner, repo_name)
 
-                # Fetch closed PRs from the past year
-                prs = self.fetch_closed_prs(owner, repo_name, days, verbose)
+                # Reset the last_pr_page_processed if requested
+                if reset:
+                    repo.last_pr_page_processed = 0
+                    repo.save()
+                    self.stdout.write(f"Reset last_pr_page_processed for {repo_full_name}")
 
-                # Save PRs to the database
-                added_count = self.save_prs_to_db(repo, prs, verbose)
+                # Fetch closed PRs from the past specified days
+                prs_fetched, prs_added, prs_updated = self.fetch_and_save_prs(repo, owner, repo_name, days, verbose)
 
-                total_prs_fetched += len(prs)
-                total_prs_added += added_count
+                total_prs_fetched += prs_fetched
+                total_prs_added += prs_added
+                total_prs_updated += prs_updated
 
-                self.stdout.write(f"Processed {repo_full_name}: Fetched {len(prs)} PRs, Added {added_count} new PRs")
+                self.stdout.write(
+                    f"Processed {repo_full_name}: Fetched {prs_fetched} PRs, "
+                    f"Added {prs_added} new PRs, Updated {prs_updated} existing PRs"
+                )
 
             except Exception as e:
                 logger.error(f"Error processing repository {repo_full_name}: {str(e)}", exc_info=True)
@@ -98,7 +112,9 @@ def handle(self, *args, **options):
         self.stdout.write(
             self.style.SUCCESS(
                 f"Completed fetching PRs for GSoC repositories. "
-                f"Total fetched: {total_prs_fetched}, Total added: {total_prs_added}"
+                f"Total fetched: {total_prs_fetched}, "
+                f"Total added: {total_prs_added}, "
+                f"Total updated: {total_prs_updated}"
             )
         )
 
@@ -140,35 +156,43 @@ def get_or_create_repo(self, owner, repo_name):
                     watchers=repo_data.get("watchers_count", 0),
                     primary_language=repo_data.get("language"),
                     is_owasp_repo=owner.upper() == "OWASP",
+                    last_pr_page_processed=0,
                 )
                 repo.save()
 
         return repo
 
-    def fetch_closed_prs(self, owner, repo_name, days, verbose=False):
+    def fetch_and_save_prs(self, repo, owner, repo_name, days, verbose=False):
         """
-        Fetch closed pull requests from GitHub API.
-        Only fetches PRs from the past specified number of days.
+        Fetch closed pull requests from GitHub API and save them to the database.
+        Returns a tuple of (total_prs_fetched, total_prs_added, total_prs_updated).
         """
-        prs = []
-        page = 1
-        per_page = 100
+        total_prs_fetched = 0
+        total_prs_added = 0
+        total_prs_updated = 0
 
         # Calculate date for filtering
         since_date = timezone.now() - timedelta(days=days)
         since_date_str = since_date.strftime("%Y-%m-%dT%H:%M:%SZ")
 
         self.stdout.write(f"Fetching PRs since {since_date_str} for {owner}/{repo_name}")
         self.stdout.write(f"Current date: {timezone.now().strftime('%Y-%m-%dT%H:%M:%SZ')}, Looking back {days} days")
+        self.stdout.write(f"Starting from page {repo.last_pr_page_processed + 1}")
 
+        # Set up headers for GitHub API
         headers = {"Accept": "application/vnd.github.v3+json"}
         if settings.GITHUB_TOKEN:
             headers["Authorization"] = f"token {settings.GITHUB_TOKEN}"
             self.stdout.write("Using GitHub token for authentication")
         else:
             self.stdout.write("No GitHub token found, using unauthenticated requests (rate limits may apply)")
 
-        while True:
+        # Start from the last processed page + 1
+        page = repo.last_pr_page_processed + 1
+        per_page = 100
+        reached_end = False
+
+        while not reached_end:
             url = (
                 f"https://api.github.com/repos/{owner}/{repo_name}/pulls"
                 f"?state=closed&per_page={per_page}&page={page}&sort=updated&direction=desc"
@@ -185,6 +209,7 @@ def fetch_closed_prs(self, owner, repo_name, days, verbose=False):
                 data = response.json()
                 if not data:
                     self.stdout.write(f"No more PRs found for {owner}/{repo_name} on page {page}")
+                    reached_end = True
                     break
 
                 self.stdout.write(f"Fetched {len(data)} PRs from page {page}")
@@ -193,11 +218,21 @@ def fetch_closed_prs(self, owner, repo_name, days, verbose=False):
                 merged_count = sum(1 for pr in data if pr.get("merged_at") is not None)
                 self.stdout.write(f"Found {merged_count} merged PRs on page {page}")
 
-                prs.extend(data)
+                # Process this page of PRs
+                prs_added, prs_updated = self.save_prs_to_db(repo, data, verbose)
+                total_prs_fetched += len(data)
+                total_prs_added += prs_added
+                total_prs_updated += prs_updated
+
+                # Update the repository's last processed page
+                repo.last_pr_page_processed = page
+                repo.last_pr_fetch_date = timezone.now()
+                repo.save()
 
                 # Check if we've reached the last page
                 if len(data) < per_page:
                     self.stdout.write(f"Reached last page ({page}) for {owner}/{repo_name}")
+                    reached_end = True
                     break
 
                 page += 1
@@ -208,27 +243,36 @@ def fetch_closed_prs(self, owner, repo_name, days, verbose=False):
                 break
 
         if verbose:
-            self.stdout.write(f"Fetched {len(prs)} PRs for {owner}/{repo_name}")
-            merged_prs = sum(1 for pr in prs if pr.get("merged_at") is not None)
-            self.stdout.write(f"Of which {merged_prs} are merged PRs")
+            self.stdout.write(f"Fetched {total_prs_fetched} PRs for {owner}/{repo_name}")
+            merged_prs = sum(
+                1
+                for pr in GitHubIssue.objects.filter(
+                    repo=repo, type="pull_request", is_merged=True, created_at__gte=since_date
+                )
+            )
+            self.stdout.write(f"Total merged PRs in database: {merged_prs}")
 
-        return prs
+        return total_prs_fetched, total_prs_added, total_prs_updated
 
     @transaction.atomic
     def save_prs_to_db(self, repo, prs, verbose=False):
         """
         Save pull requests to the database.
-        Returns the number of new PRs added.
+        Returns the number of new PRs added and updated.
         """
         added_count = 0
+        updated_count = 0
         skipped_count = 0
+        skipped_not_merged = 0
+
+        self.stdout.write(f"Processing {len(prs)} PRs for {repo.name}")
 
         for pr in prs:
-            # Check if PR already exists in the database
-            if GitHubIssue.objects.filter(issue_id=pr["id"]).exists():
-                skipped_count += 1
+            # Skip PRs that aren't merged
+            if not pr.get("merged_at"):
+                skipped_not_merged += 1
                 if verbose:
-                    self.stdout.write(f"PR {pr['number']} already exists in the database")
+                    self.stdout.write(f"PR {pr['number']} is not merged, skipping")
                 continue
 
             # Parse dates
@@ -249,29 +293,44 @@ def save_prs_to_db(self, repo, prs, verbose=False):
             user_profile = None
             if pr["user"] and pr["user"]["html_url"]:
                 user_profile = UserProfile.objects.filter(github_url=pr["user"]["html_url"]).first()
+                if not user_profile and verbose:
+                    self.stdout.write(f"No user profile found for {pr['user']['html_url']}")
+
+            # Prepare the data for the GitHubIssue
+            issue_data = {
+                "title": pr["title"],
+                "body": pr["body"] or "",
+                "state": pr["state"],
+                "type": "pull_request",
+                "created_at": created_at,
+                "updated_at": updated_at,
+                "closed_at": closed_at,
+                "merged_at": merged_at,
+                "is_merged": is_merged,
+                "url": pr["html_url"],
+                "repo": repo,
+                "user_profile": user_profile,
+            }
+
+            # Try to get the existing issue or create a new one
+            try:
+                github_issue, created = GitHubIssue.objects.update_or_create(issue_id=pr["id"], defaults=issue_data)
+
+                if created:
+                    added_count += 1
+                    if verbose:
+                        self.stdout.write(f"Added PR #{pr['number']}: {pr['title']}")
+                else:
+                    updated_count += 1
+                    if verbose:
+                        self.stdout.write(f"Updated PR #{pr['number']}: {pr['title']}")
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"Error saving PR #{pr['number']}: {str(e)}"))
+                skipped_count += 1
 
-            # Create the GitHubIssue
-            github_issue = GitHubIssue(
-                issue_id=pr["id"],
-                title=pr["title"],
-                body=pr["body"] or "",
-                state=pr["state"],
-                type="pull_request",
-                created_at=created_at,
-                updated_at=updated_at,
-                closed_at=closed_at,
-                merged_at=merged_at,
-                is_merged=is_merged,
-                url=pr["html_url"],
-                repo=repo,
-                user_profile=user_profile,
-            )
-            github_issue.save()
-
-            added_count += 1
-
-            if verbose:
-                self.stdout.write(f"Added PR #{pr['number']}: {pr['title']}")
+        self.stdout.write(f"Skipped {skipped_count} PRs due to errors")
+        self.stdout.write(f"Skipped {skipped_not_merged} PRs that are not merged")
+        self.stdout.write(f"Added {added_count} new PRs to the database")
+        self.stdout.write(f"Updated {updated_count} existing PRs in the database")
 
-        self.stdout.write(f"Skipped {skipped_count} PRs that already exist in the database")
-        return added_count
+        return added_count, updated_count

website/migrations/0226_repo_last_pr_fetch_date_repo_last_pr_page_processed.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1.6 on 2025-03-10 21:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("website", "0225_hackathon_hackathonsponsor_hackathonprize_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="repo",
+            name="last_pr_fetch_date",
+            field=models.DateTimeField(blank=True, help_text="When PRs were last fetched", null=True),
+        ),
+        migrations.AddField(
+            model_name="repo",
+            name="last_pr_page_processed",
+            field=models.IntegerField(default=0, help_text="Last page of PRs processed from GitHub API"),
+        ),
+    ]

website/models.py

@@ -1488,6 +1488,8 @@ class Repo(models.Model):
     ai_summary = models.TextField(null=True, blank=True)
     created_at = models.DateTimeField(auto_now_add=True)
     updated_at = models.DateTimeField(auto_now=True)
+    last_pr_page_processed = models.IntegerField(default=0, help_text="Last page of PRs processed from GitHub API")
+    last_pr_fetch_date = models.DateTimeField(null=True, blank=True, help_text="When PRs were last fetched")
 
     def save(self, *args, **kwargs):
         if not self.slug:

website/templates/gsoc.html

@@ -209,10 +209,8 @@ <h3 class="text-xl font-semibold text-red-700 mb-2">OWASP BLT Project Ideas</h3>
             </div>
             <!-- Top Contributors Leaderboard -->
             <div class="bg-white rounded-xl shadow-lg hover:shadow-xl transition-all duration-300 p-8 mb-12 border-l-4 border-red-500">
-                <h2 class="text-3xl font-bold text-gray-900 mb-6">Top Contributors for GSoC 2025 Projects</h2>
-                <p class="text-gray-700 mb-4">
-                    For transparency, we are implementing a Top contributors List  for students this year. This demonstrates our commitment to working students who are contunually contributing while also signaling our intent to other projects and organizations.
-                </p>
+                <h2 class="text-3xl font-bold text-gray-900 mb-6">Top OWASP GSoC 2025 Contributors and Projects</h2>
+                <p class="text-gray-700 mb-4">A glimpse of the top contributors and projects for OWASP GSoC 2025.</p>
                 <div class="grid grid-cols-1 md:grid-cols-2 gap-8">
                     {% for project, data in projects.items %}
                         <div class="bg-gray-50 rounded-lg p-6">