Add SSVC trees, resource URL and max_advisories

CHANGELOG.rst

@@ -5,6 +5,8 @@ next release
 ---------------------
 
 - WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories.
+- WARNING: We will deprecate improver pipelines for calculating package version rank, grouping advisories for packages and calculating risk scores in the next release, we are doing it at advisory import time instead of as separate pipelines, this will improve the performance and consistency of the data.
+- Calculate package verion rank, group advisories for packages and package risk score and advisory risk score during import of advisories.
 - Add attribute ``pipeline_id`` to AdvisoryV2 to track the pipeline that created the advisory, also rename existing ``datasource_id`` and AVIDs. 
 
 Version v38.6.0

vulnerabilities/improvers/__init__.py

@@ -11,7 +11,6 @@
 from vulnerabilities.improvers import vulnerability_status
 from vulnerabilities.pipelines import add_cvss31_to_CVEs
 from vulnerabilities.pipelines import compute_package_risk
-from vulnerabilities.pipelines import compute_package_version_rank
 from vulnerabilities.pipelines import enhance_with_exploitdb
 from vulnerabilities.pipelines import enhance_with_kev
 from vulnerabilities.pipelines import enhance_with_metasploit
@@ -32,7 +31,6 @@
     enhance_with_metasploit as enhance_with_metasploit_v2,
 )
 from vulnerabilities.pipelines.v2_improvers import flag_ghost_packages as flag_ghost_packages_v2
-from vulnerabilities.pipelines.v2_improvers import group_advisories_for_packages
 from vulnerabilities.pipelines.v2_improvers import reference_collect_commits
 from vulnerabilities.pipelines.v2_improvers import relate_severities
 from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2
@@ -61,7 +59,6 @@
         enhance_with_metasploit.MetasploitImproverPipeline,
         enhance_with_exploitdb.ExploitDBImproverPipeline,
         compute_package_risk.ComputePackageRiskPipeline,
-        compute_package_version_rank.ComputeVersionRankPipeline,
         add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline,
         remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline,
         populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
@@ -75,7 +72,6 @@
         collect_ssvc_trees.CollectSSVCPipeline,
         relate_severities.RelateSeveritiesPipeline,
         archive_urls.ArchiveImproverPipeline,
-        group_advisories_for_packages.GroupAdvisoriesForPackages,
         compute_advisory_todo_v2.ComputeToDo,
         reference_collect_commits.CollectReferencesFixCommitsPipeline,
         enhance_with_github_poc.GithubPocsImproverPipeline,

vulnerabilities/migrations/0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more.py

@@ -0,0 +1,48 @@
+# Generated by Django 5.2.11 on 2026-05-26 08:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0132_migrate_advisoryv2_datasource_ids"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="advisory_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="An advisory is a unique vulnerability identifier in some database, such as PYSEC-2020-2233",
+                max_length=200,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="avid",
+            field=models.CharField(
+                help_text="Unique ID for the datasource used for this advisory .e.g.: pysec_importer_v2/PYSEC-2020-2233",
+                max_length=250,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="datasource_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the datasource used for this advisory .e.g.: nginx",
+                max_length=50,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="pipeline_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2",
+                max_length=50,
+            ),
+        ),
+    ]

vulnerabilities/migrations/0134_alter_advisoryset_unique_together.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.2.11 on 2026-05-28 13:58
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0133_alter_advisoryv2_advisory_id_alter_advisoryv2_avid_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="advisoryset",
+            unique_together={("package", "relation_type", "primary_advisory")},
+        ),
+    ]

vulnerabilities/migrations/0135_impactedpackage_is_latest.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.2.11 on 2026-05-30 17:49
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0134_alter_advisoryset_unique_together"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="impactedpackage",
+            name="is_latest",
+            field=models.BooleanField(
+                db_index=True,
+                default=False,
+                help_text="Indicates whether this is the latest impact for the advisory.",
+            ),
+        ),
+    ]

vulnerabilities/migrations/0136_populate_impactedpackage_is_latest.py

@@ -0,0 +1,22 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0135_impactedpackage_is_latest"),
+    ]
+
+
+    def mark_latest_impacted_packages(apps, schema_editor):
+        AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2")
+        ImpactedPackage = apps.get_model("vulnerabilities", "ImpactedPackage")
+
+        print("\nMarking impacted packages for latest V2 advisories as is_latest=True.")
+        latest_advisory_ids = AdvisoryV2.objects.filter(is_latest=True).values_list("id", flat=True)
+        ImpactedPackage.objects.filter(advisory_id__in=latest_advisory_ids).update(is_latest=True)
+
+
+    operations = [
+        migrations.RunPython(code=mark_latest_impacted_packages, reverse_code=migrations.RunPython.noop),
+    ]

vulnerabilities/models.py

@@ -19,6 +19,7 @@
 from itertools import groupby
 from operator import attrgetter
 from traceback import format_exc as traceback_format_exc
+from typing import Dict
 from typing import List
 from typing import NamedTuple
 from typing import Optional
@@ -2933,53 +2934,63 @@ def latest_for_avids(self, avids):
         return self.filter(avid__in=avids).latest_per_avid()
 
     def latest_affecting_advisories_for_purl(self, purl):
-        adv_ids = ImpactedPackageAffecting.objects.filter(package__package_url=purl).values_list(
-            "impacted_package__advisory_id",
-            flat=True,
-        )
+        adv_ids = ImpactedPackage.objects.filter(
+            affecting_packages__package_url=purl, is_latest=True
+        ).values_list("advisory_id", flat=True)
+
         return self.filter(id__in=Subquery(adv_ids)).latest_per_avid()
 
     def latest_affecting_advisories_for_purls(self, purls):
-        adv_ids = ImpactedPackageAffecting.objects.filter(
-            package__package_url__in=purls
+        adv_ids = ImpactedPackage.objects.filter(
+            affecting_packages__package_url__in=purls, is_latest=True
         ).values_list(
-            "impacted_package__advisory_id",
+            "advisory_id",
             flat=True,
         )
         return self.filter(id__in=Subquery(adv_ids)).latest_per_avid()
 
-    def latest_affecting_advisories_for_packages(self, purls):
-        adv_ids = ImpactedPackageAffecting.objects.filter(package__in=purls).values_list(
-            "impacted_package__advisory_id",
+    def latest_affecting_advisories_for_packages(self, packages):
+        adv_ids = ImpactedPackage.objects.filter(
+            affecting_packages__in=packages, is_latest=True
+        ).values_list(
+            "advisory_id",
             flat=True,
         )
         return self.filter(id__in=Subquery(adv_ids)).latest_per_avid()
 
     def latest_fixed_by_advisories_for_purl(self, purl):
-        adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list(
-            "impacted_package__advisory_id",
+        adv_ids = ImpactedPackage.objects.filter(
+            fixed_by_packages__package_url=purl, is_latest=True
+        ).values_list(
+            "advisory_id",
             flat=True,
         )
         return self.filter(id__in=Subquery(adv_ids)).latest_per_avid()
 
     def latest_fixed_by_advisories_for_purls(self, purls):
-        adv_ids = ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list(
-            "impacted_package__advisory_id",
+        adv_ids = ImpactedPackage.objects.filter(
+            fixed_by_packages__package_url__in=purls, is_latest=True
+        ).values_list(
+            "advisory_id",
             flat=True,
         )
 
         return self.filter(id__in=Subquery(adv_ids)).latest_per_avid()
 
     def latest_advisories_for_purls(self, purls):
         adv_ids = (
-            ImpactedPackageAffecting.objects.filter(package__package_url__in=purls)
+            ImpactedPackage.objects.filter(
+                affecting_packages__package_url__in=purls, is_latest=True
+            )
             .values_list(
-                "impacted_package__advisory_id",
+                "advisory_id",
                 flat=True,
             )
             .union(
-                ImpactedPackageFixedBy.objects.filter(package__package_url__in=purls).values_list(
-                    "impacted_package__advisory_id",
+                ImpactedPackage.objects.filter(
+                    fixed_by_packages__package_url__in=purls, is_latest=True
+                ).values_list(
+                    "advisory_id",
                     flat=True,
                 )
             )
@@ -2990,14 +3001,16 @@ def latest_advisories_for_purls(self, purls):
 
     def latest_advisories_for_purl(self, purl):
         adv_ids = (
-            ImpactedPackageAffecting.objects.filter(package__package_url=purl)
+            ImpactedPackage.objects.filter(affecting_packages__package_url=purl, is_latest=True)
             .values_list(
-                "impacted_package__advisory_id",
+                "advisory_id",
                 flat=True,
             )
             .union(
-                ImpactedPackageFixedBy.objects.filter(package__package_url=purl).values_list(
-                    "impacted_package__advisory_id",
+                ImpactedPackage.objects.filter(
+                    fixed_by_packages__package_url=purl, is_latest=True
+                ).values_list(
+                    "advisory_id",
                     flat=True,
                 )
             )
@@ -3033,6 +3046,9 @@ class AdvisorySet(models.Model):
 
     created_at = models.DateTimeField(auto_now_add=True)
 
+    class Meta:
+        unique_together = ("package", "relation_type", "primary_advisory")
+
 
 class AdvisorySetMember(models.Model):
 
@@ -3054,15 +3070,15 @@ class AdvisoryV2(models.Model):
 
     # This is similar to a type or a namespace
     datasource_id = models.CharField(
-        max_length=200,
+        max_length=50,
         blank=False,
         null=False,
         db_index=True,
         help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx",
     )
 
     pipeline_id = models.CharField(
-        max_length=200,
+        max_length=50,
         blank=False,
         null=False,
         db_index=True,
@@ -3071,7 +3087,7 @@ class AdvisoryV2(models.Model):
 
     # This is similar to a name
     advisory_id = models.CharField(
-        max_length=500,
+        max_length=200,
         blank=False,
         null=False,
         unique=False,
@@ -3081,7 +3097,7 @@ class AdvisoryV2(models.Model):
     )
 
     avid = models.CharField(
-        max_length=500,
+        max_length=250,
         blank=False,
         null=False,
         help_text="Unique ID for the datasource used for this advisory ."
@@ -3348,6 +3364,14 @@ class ImpactedPackage(models.Model):
         help_text="Timestamp of the last successful vers range unfurl.",
     )
 
+    is_latest = models.BooleanField(
+        default=False,
+        blank=False,
+        null=False,
+        db_index=True,
+        help_text="Indicates whether this is the latest impact for the advisory.",
+    )
+
     def to_dict(self):
         from vulnerabilities.utils import purl_to_dict
 
@@ -3419,7 +3443,7 @@ def search(self, query: str = None):
         except ValueError:
             # otherwise use query as a plain string
             qs = qs.filter(package_url__icontains=query)
-        return qs.order_by("package_url").order_by("-version_rank")
+        return qs.order_by("type", "namespace", "name", "-version_rank")
 
     def with_vulnerability_counts(self):
         return self.annotate(
@@ -3528,6 +3552,24 @@ def with_is_vulnerable(self):
             )
         )
 
+    def all_vulnerable(self):
+        latest_impacts = ImpactedPackageAffecting.objects.filter(
+            package_id=OuterRef("pk"),
+            impacted_package__is_latest=True,
+        )
+
+        query = PackageV2.objects.filter(Exists(latest_impacts))
+        return query
+
+    def all_vulnerable_purls(self):
+        return self.all_vulnerable().values_list("package_url", flat=True)
+
+    def filter_plain_purls(self, plain_purls=[]):
+        return PackageV2.objects.filter(plain_package_url__in=plain_purls)
+
+    def filter_purls(self, purls=[]):
+        return PackageV2.objects.filter(package_url__in=purls)
+
     def from_purl(self, purl: Union[PackageURL, str]):
         """
         Return a new Package given a ``purl`` PackageURL object or PURL string.
@@ -3872,6 +3914,7 @@ class GroupedAdvisory(NamedTuple):
     weighted_severity: Optional[float]
     exploitability: Optional[float]
     risk_score: Optional[float]
+    ssvc_trees: List[Dict]
 
 
 class AdvisoryPOC(models.Model):