diff --git a/insights/__init__.py b/insights/__init__.py new file mode 100644 index 000000000..20854f2ad --- /dev/null +++ b/insights/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# diff --git a/insights/apps.py b/insights/apps.py new file mode 100644 index 000000000..7200f28cd --- /dev/null +++ b/insights/apps.py @@ -0,0 +1,14 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.apps import AppConfig + + +class InsightsConfig(AppConfig): + name = "insights" diff --git a/insights/charts/__init__.py b/insights/charts/__init__.py new file mode 100644 index 000000000..56fcd1c9b --- /dev/null +++ b/insights/charts/__init__.py @@ -0,0 +1,89 @@ +from functools import partial + +from insights.models import ChartDefinition + +from .importer_panel import _get_snapshot_data +from .importer_panel import build_importer_exploit_columns +from .importer_panel import build_importer_package_columns +from .importer_panel import collect_importers +from .package_panel import collect_cwes +from .package_panel import collect_ecosystem_distribution +from .package_panel import collect_packages +from .package_panel import format_ecosystem_distribution +from .package_panel import format_top_cwes +from .package_panel import format_top_packages +from .severity_panel import collect_severities +from .severity_panel import get_severity_snapshot_data + +CHARTS = [ + ChartDefinition( + id="pkg-dist-donut", + title="Ecosystem Distribution", + panel="package_panel", + chart_type="donut", + formatter_fn=format_ecosystem_distribution, + collect_fn=collect_ecosystem_distribution, + ), + ChartDefinition( + id="pkg-name-donut", + title="Top 10 Packages", + panel="package_panel", + chart_type="donut", + formatter_fn=format_top_packages, + collect_fn=collect_packages, + is_per_package=True, + ), + ChartDefinition( + id="pkg-cwe-bar", + title="Top 10 CWE Distribution", + panel="package_panel", + chart_type="colored_bar", + formatter_fn=format_top_cwes, + collect_fn=collect_cwes, + is_per_package=False, + has_search=True, + ), + ChartDefinition( + id="severity-scatter-plot", + title="Severity Distribution across Packages", + panel="severity_panel", + chart_type="scatter", + formatter_fn=get_severity_snapshot_data, + collect_fn=collect_severities, + has_search=True, + ), + ChartDefinition( + id="importer-empty-pkg-bar", + title="Affected Package Coverage across Importers", + panel="importer_panel", + chart_type="importer_bar", + formatter_fn=partial(_get_snapshot_data, build_columns_fn=build_importer_package_columns), + collect_fn=collect_importers, + ), + ChartDefinition( + id="importer-exploit-bar", + title="Exploit Coverage across Importers", + panel="importer_panel", + chart_type="importer_bar", + formatter_fn=partial(_get_snapshot_data, build_columns_fn=build_importer_exploit_columns), + collect_fn=collect_importers, + ), +] + +PANELS = [ + "overview_panel", + "package_panel", + "severity_panel", + "importer_panel", + "data_quality_panel", +] +PANEL_LABELS = { + "overview_panel": "Overview", + "package_panel": "Package Analytics", + "severity_panel": "Severity Analytics", + "importer_panel": "Importer Analytics", + "data_quality_panel": "Data Quality Analytics", +} +PANEL_LAYOUTS = { + "package_panel": "split_top", +} diff --git a/insights/charts/importer_panel.py b/insights/charts/importer_panel.py new file mode 100644 index 000000000..de0c7968a --- /dev/null +++ b/insights/charts/importer_panel.py @@ -0,0 +1,210 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from typing import Any +from typing import Dict + +from django.db.models import Count +from django.db.models import Exists +from django.db.models import OuterRef +from django.db.models import Q + +from insights.models import ImporterInsight +from insights.utils import format_importer_name +from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import AdvisoryV2 + +# Ignore Phantom Importers that don't collect Affected Packages +IGNORED_IMPORTERS = { + "epss_importer_v2", + "epss", + "vulnrichment_importer_v2", + "suse_importer_v2", + "suse_score", +} + + +def packages_queryset(): + """Return a query set of package statistics by importer.""" + return ( + AdvisoryV2.objects.filter(is_latest=True) + .exclude(datasource_id__in=IGNORED_IMPORTERS) + .values("datasource_id") + .annotate( + total_advisories=Count("avid", distinct=True), + advisories_with_packages=Count( + "avid", filter=Q(impacted_packages__affecting_packages__isnull=False), distinct=True + ), + advisories_with_ghost_packages=Count( + "avid", + filter=Q(impacted_packages__affecting_packages__is_ghost=True), + distinct=True, + ), + ) + .order_by("-total_advisories") + .iterator() + ) + + +def exploits_queryset(): + """Return a query set of exploit statistics by importer.""" + kev_exploits = AdvisoryExploit.objects.filter(advisory=OuterRef("pk"), data_source="KEV") + metasploit_exploits = AdvisoryExploit.objects.filter( + advisory=OuterRef("pk"), data_source="Metasploit" + ) + exploitdb_exploits = AdvisoryExploit.objects.filter( + advisory=OuterRef("pk"), data_source="Exploit-DB" + ) + + return ( + AdvisoryV2.objects.filter(is_latest=True) + .exclude(datasource_id__in=IGNORED_IMPORTERS) + .annotate( + has_kev=Exists(kev_exploits), + has_metasploit=Exists(metasploit_exploits), + has_exploitdb=Exists(exploitdb_exploits), + ) + .values("datasource_id") + .annotate( + advisories_with_kev=Count("avid", filter=Q(has_kev=True), distinct=True), + advisories_with_metasploit=Count( + "avid", filter=Q(has_kev=False, has_metasploit=True), distinct=True + ), + advisories_with_exploitdb=Count( + "avid", + filter=Q(has_kev=False, has_metasploit=False, has_exploitdb=True), + distinct=True, + ), + ) + .iterator() + ) + + +def iter_importer_insights(): + """Yield ImporterInsight objects by merging package and exploit statistics.""" + exploit_stats_by_importer = {} + for record in exploits_queryset(): + exploit_stats_by_importer[record["datasource_id"]] = record + + for pkg_record in packages_queryset(): + datasource_id = pkg_record["datasource_id"] + exploit_record = exploit_stats_by_importer.get(datasource_id, {}) + + yield ImporterInsight( + importer=datasource_id, + total_advisories=pkg_record["total_advisories"], + advisories_with_packages=pkg_record["advisories_with_packages"], + advisories_with_ghost_packages=pkg_record["advisories_with_ghost_packages"], + advisories_with_kev=exploit_record.get("advisories_with_kev", 0), + advisories_with_metasploit=exploit_record.get("advisories_with_metasploit", 0), + advisories_with_exploitdb=exploit_record.get("advisories_with_exploitdb", 0), + ) + + +def collect_importers(pipeline): + """Calculates and stores importer insights.""" + + # Two charts use this function, so we avoid rerunning the query twice + if pipeline.importer_insights: + return + + for insight in iter_importer_insights(): + pipeline.importer_insights.append(insight) + + +def _get_snapshot_data(snapshot: Any, build_columns_fn) -> Dict[str, Any]: + """Helper to format importer statistics using a provided column builder.""" + all_importers = list(snapshot.importer_insights.order_by("-total_advisories")) + data = {} + + if all_importers: + # Top 5 by default for global view + data["global"] = build_columns_fn(all_importers[:5]) + + # Individual data for each importer + for importer_insight in all_importers: + formatted_name = format_importer_name(importer_insight.importer) + data[formatted_name] = build_columns_fn([importer_insight]) + + return data + + +def build_importer_package_columns(importers) -> Dict[str, Any]: + """Helper to build mappings for Package Coverage chart as expected by Billboard.JS""" + importer_names = [] + total_advisories = [] + advisories_with_packages = [] + advisories_without_packages = [] + advisories_with_ghost_packages = [] + advisories_without_ghost_packages = [] + + for importer_insight in importers: + importer_names.append(format_importer_name(importer_insight.importer)) + total_advisories.append(importer_insight.total_advisories) + advisories_with_packages.append(importer_insight.advisories_with_packages) + advisories_without_packages.append( + importer_insight.total_advisories - importer_insight.advisories_with_packages + ) + advisories_with_ghost_packages.append(importer_insight.advisories_with_ghost_packages) + advisories_without_ghost_packages.append( + importer_insight.advisories_with_packages + - importer_insight.advisories_with_ghost_packages + ) + + return { + "x_categories": importer_names, + "columns": [ + ["total_advisories"] + total_advisories, + ["advisories_without_packages"] + advisories_without_packages, + ["advisories_with_packages"] + advisories_with_packages, + ["advisories_without_ghost_packages"] + advisories_without_ghost_packages, + ["advisories_with_ghost_packages"] + advisories_with_ghost_packages, + ], + "groups": [ + ["advisories_without_packages", "advisories_with_packages"], + ["advisories_without_ghost_packages", "advisories_with_ghost_packages"], + ], + } + + +def build_importer_exploit_columns(importers) -> Dict[str, Any]: + """Helper to build mappings for Exploit Coverage chart as expected by Billboard.JS""" + importer_names = [] + total_advisories = [] + advisories_with_kev = [] + advisories_with_metasploit = [] + advisories_with_exploitdb = [] + advisories_without_exploits = [] + + for importer_insight in importers: + importer_names.append(format_importer_name(importer_insight.importer)) + total_advisories.append(importer_insight.total_advisories) + advisories_with_kev.append(importer_insight.advisories_with_kev) + advisories_with_metasploit.append(importer_insight.advisories_with_metasploit) + advisories_with_exploitdb.append(importer_insight.advisories_with_exploitdb) + + total_with_exploits = ( + importer_insight.advisories_with_kev + + importer_insight.advisories_with_metasploit + + importer_insight.advisories_with_exploitdb + ) + advisories_without_exploits.append(importer_insight.total_advisories - total_with_exploits) + + return { + "x_categories": importer_names, + "columns": [ + ["total_advisories"] + total_advisories, + ["advisories_without_exploits"] + advisories_without_exploits, + ["advisories_with_exploitdb"] + advisories_with_exploitdb, + ["advisories_with_metasploit"] + advisories_with_metasploit, + ["advisories_with_kev"] + advisories_with_kev, + ], + "groups": [ + ["advisories_with_exploitdb", "advisories_with_metasploit", "advisories_with_kev"] + ], + } diff --git a/insights/charts/package_panel.py b/insights/charts/package_panel.py new file mode 100644 index 000000000..1b0187a0f --- /dev/null +++ b/insights/charts/package_panel.py @@ -0,0 +1,183 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from collections import Counter +from collections import defaultdict +from typing import Any +from typing import Dict + +from django.db.models import Count +from django.db.models import F +from django.db.models import Q + +from insights.models import PackageCWEInsight +from insights.models import PackageInsight +from insights.models import PackageNameInsight +from insights.utils import get_cwe_label +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 + +# Generic OWASP category CWEs used by NVD, not actual weakness IDs. +IGNORED_CWE_IDS = [937, 1035] + + +# Package Distribution +def ecosystem_distribution_queryset(package_types): + """Return a query set of total package counts per ecosystem.""" + return ( + PackageV2.objects.filter(type__in=package_types) + .values("type") + .annotate(total_package_count=Count("id")) + .iterator() + ) + + +def iter_ecosystem_distribution_insights(package_types): + """Yield total package type counts per ecosystem.""" + for package_stat in ecosystem_distribution_queryset(package_types): + yield package_stat["type"], package_stat["total_package_count"] + + +def collect_ecosystem_distribution(pipeline: Any) -> None: + """Collect total package type counts per ecosystem.""" + for package_type, total_packages in iter_ecosystem_distribution_insights(pipeline.packages): + if package_type in pipeline.package_insights: + pipeline.package_insights[package_type].total_packages = total_packages + + +def format_ecosystem_distribution(snapshot: Any) -> Dict[str, Any]: + """Format the Ecosystem Distribution Chart as expected by Billboard.JS""" + stats = list(snapshot.package_insights.exclude(package="global").order_by("-total_packages")) + + columns = [[stat.package, stat.total_packages] for stat in stats[:10]] + + others_count = sum(stat.total_packages for stat in stats[10:]) + others_list = [[stat.package, stat.total_packages] for stat in stats[10:20]] + if others_count > 0: + columns.append(["Others", others_count]) + + return {"global": {"columns": columns, "others_list": others_list}} + + +# Top 10 Packages +def top_packages_queryset(package_type: str): + """Return a query set of the top 10 packages by number of distinct advisories.""" + return ( + PackageV2.objects.filter(type=package_type) + .values("name") + .annotate( + count=Count( + "affected_in_impacts__advisory", + filter=Q(affected_in_impacts__advisory__is_latest=True), + ) + ) + .filter(count__gt=0) + .order_by("-count")[:10] + .iterator() + ) + + +def iter_packages_insights(package_type: str, insight_obj: Any): + """Yield PackageNameInsight objects for the top 10 packages of a given type.""" + for stat in top_packages_queryset(package_type): + yield PackageNameInsight( + package_insight=insight_obj, name=stat["name"], count=stat["count"] + ) + + +def collect_packages(pipeline: Any, package_type: str) -> None: + """Collect the top 10 packages for a package type""" + insight_obj = pipeline.package_insights[package_type] + for insight in iter_packages_insights(package_type, insight_obj): + pipeline.package_names.append(insight) + + +def build_name_chart_columns(name_counts: dict) -> Dict[str, Any]: + """Helper to build package name donut charts.""" + return {"columns": [[name, count] for name, count in name_counts.items()]} + + +def format_top_packages(snapshot: Any) -> Dict[str, Any]: + """ + Format name data for the frontend donut chart as expected by Billboard.JS + Returns Top 10 package names for each package type and global. + """ + data = {} + global_counts = defaultdict(int) + + for package_insight in ( + snapshot.package_insights.exclude(package="global").prefetch_related("names").all() + ): + name_counts = {ns.name: ns.count for ns in package_insight.names.all()} + data[package_insight.package] = build_name_chart_columns(name_counts) + for name, count in name_counts.items(): + global_counts[name] += count + + top_global = dict(Counter(global_counts).most_common(10)) + data["global"] = build_name_chart_columns(top_global) + return data + + +# Top CWE Distribution +def compute_top_cwes(packages=None) -> dict: + """Computes the top 10 CWEs for the given queryset of packages. If None, computes globally.""" + qs = AdvisoryV2.objects.filter(weaknesses__isnull=False) + if packages is not None: + qs = qs.filter(impacted_packages__affecting_packages__in=packages) + + top_10 = ( + qs.values(cwe=F("weaknesses__cwe_id")) + .exclude(cwe__in=IGNORED_CWE_IDS) + .annotate(count=Count("avid", distinct=True)) + .order_by("-count")[:10] + ) + return {stat["cwe"]: stat["count"] for stat in top_10 if stat["cwe"]} + + +def iter_cwes_insights(insight_obj: Any): + """Yield PackageCWEInsight objects for the global top 10 CWE distribution.""" + for cwe_id, count in compute_top_cwes().items(): + yield PackageCWEInsight(package_insight=insight_obj, cwe_id=cwe_id, count=count) + + +def collect_cwes(pipeline: Any) -> None: + """Collect the global top 10 CWE distribution.""" + if "global" not in pipeline.package_insights: + pipeline.package_insights["global"] = PackageInsight(package="global") + + # Collect top 10 CWEs globally + insight_obj = pipeline.package_insights["global"] + for insight in iter_cwes_insights(insight_obj): + pipeline.package_cwes.append(insight) + + +def build_cwe_chart_columns(cwe_counts: dict) -> Dict[str, Any]: + """Helper to CWE chart as expected by Billboard.JS""" + # Sort CWEs by count in descending order + sorted_cwes = sorted(cwe_counts.items(), key=lambda item: item[1], reverse=True) + return { + "columns": [ + ["x"] + [cwe_id for cwe_id, count in sorted_cwes], + ["Advisories"] + [count for cwe_id, count in sorted_cwes], + ], + "full_labels": [get_cwe_label(cwe_id) for cwe_id, count in sorted_cwes], + } + + +def format_top_cwes(snapshot: Any) -> Dict[str, Any]: + """Format CWE distribution data.""" + data = {} + try: + global_insight = snapshot.package_insights.get(package="global") + cwe_counts = {cwe.cwe_id: cwe.count for cwe in global_insight.cwes.all()} + if cwe_counts: + data["global"] = build_cwe_chart_columns(cwe_counts) + except PackageInsight.DoesNotExist: + pass + + return data diff --git a/insights/charts/severity_panel.py b/insights/charts/severity_panel.py new file mode 100644 index 000000000..51430bb30 --- /dev/null +++ b/insights/charts/severity_panel.py @@ -0,0 +1,74 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from typing import Any +from typing import Dict + +from django.db.models import Count + +from insights.models import SeverityInsight +from vulnerabilities.models import AdvisoryV2 + +CVSS_SCORE_LABELS = ["0-1", "1-2", "2-3", "3-4", "4-5", "5-6", "6-7", "7-8", "8-9", "9-10"] + + +def aggregate_severity_buckets(queryset: Any, count_field: str) -> list[int]: + """Helper to aggregate a queryset of severity values into exactly 10 CVSS buckets.""" + buckets = [0] * 10 + + for severity in queryset: + try: + score = float(severity.get("severities__value")) + if 0 <= score <= 10: + buckets[min(9, int(score))] += severity[count_field] + except (ValueError, TypeError): + continue + + return buckets + + +def iter_severity_insights(): + """Yield SeverityInsight objects.""" + advisory_severities = ( + AdvisoryV2.objects.filter( + severities__isnull=False, severities__scoring_system__icontains="cvss" + ) + .values("severities__value") + .annotate(advisory_count=Count("avid", distinct=True)) + .iterator() + ) + + severity_buckets = aggregate_severity_buckets(advisory_severities, "advisory_count") + yield SeverityInsight(buckets=severity_buckets) + + +def collect_severities(pipeline: Any) -> None: + """Pre-compute the global severity distribution.""" + for insight in iter_severity_insights(): + pipeline.severity_insight = insight + + +def format_severity_chart_data(buckets: list[int]) -> Dict[str, Any]: + """Helper to build mappings for Severity Distribution chart as expected by Billboard.JS""" + return { + "columns": [ + ["x"] + CVSS_SCORE_LABELS, + ["Advisories"] + buckets, + ] + } + + +def get_severity_snapshot_data(snapshot: Any) -> Dict[str, Any]: + """Return the global severity distribution for the frontend scatter plot.""" + if hasattr(snapshot, "severity_insight"): + insight = snapshot.severity_insight + global_buckets = insight.buckets + else: + global_buckets = [0] * 10 + + return {"global": format_severity_chart_data(global_buckets)} diff --git a/insights/insights_snapshot_pipeline.py b/insights/insights_snapshot_pipeline.py new file mode 100644 index 000000000..06084c062 --- /dev/null +++ b/insights/insights_snapshot_pipeline.py @@ -0,0 +1,85 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from aboutcode.pipeline import LoopProgress +from django.db import transaction + +from insights.charts import CHARTS +from insights.models import DailySnapshot +from insights.models import ImporterInsight +from insights.models import PackageCWEInsight +from insights.models import PackageInsight +from insights.models import PackageNameInsight +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class InsightsSnapshotPipeline(VulnerableCodePipeline): + """Pipeline to compute aggregated statistics for the Insights Dashboard.""" + + pipeline_id = "insights_snapshot" + + @classmethod + def steps(cls): + return ( + cls.compute_chart_analytics, + cls.save_snapshot, + ) + + def compute_chart_analytics(self): + """Run chart collect_fns to compute analytics.""" + + # List all package types + self.packages = list(PackageV2.objects.order_by().values_list("type", flat=True).distinct()) + + self.package_insights = {pkg: PackageInsight(package=pkg) for pkg in self.packages} + self.package_names = [] + self.package_cwes = [] + self.importer_insights = [] + self.severity_insight = None + active_charts = [chart_def for chart_def in CHARTS if chart_def.collect_fn] + + # Count steps for progress bar + total_steps = sum( + len(self.packages) if chart_def.is_per_package else 1 for chart_def in active_charts + ) + + progress = LoopProgress(total_iterations=total_steps, logger=self.log, progress_step=1) + progress_iter = iter(progress.iter(range(total_steps))) + + for chart_def in active_charts: + self.log(f"Running collect_fn for {chart_def.id}") + + if chart_def.is_per_package: + for pkg in self.packages: + next(progress_iter, None) + chart_def.collect_fn(self, pkg) + else: + next(progress_iter, None) + chart_def.collect_fn(self) + + @transaction.atomic + def save_snapshot(self): + self.log("Saving snapshot") + snapshot = DailySnapshot.objects.create() + + for insight in self.package_insights.values(): + insight.snapshot_id = snapshot.id + + for insight in self.importer_insights: + insight.snapshot_id = snapshot.id + + if self.severity_insight: + self.severity_insight.snapshot_id = snapshot.id + self.severity_insight.save() + + PackageInsight.objects.bulk_create(self.package_insights.values(), batch_size=5000) + PackageNameInsight.objects.bulk_create(self.package_names, batch_size=5000) + PackageCWEInsight.objects.bulk_create(self.package_cwes, batch_size=5000) + ImporterInsight.objects.bulk_create(self.importer_insights, batch_size=5000) + self.log("Snapshot saved successfully.") diff --git a/insights/migrations/0001_initial.py b/insights/migrations/0001_initial.py new file mode 100644 index 000000000..26b349636 --- /dev/null +++ b/insights/migrations/0001_initial.py @@ -0,0 +1,174 @@ +# Generated by Django 5.2.11 on 2026-07-23 21:56 + +import django.contrib.postgres.fields +import django.db.models.deletion +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="DailySnapshot", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "ordering": ["-created_at"], + "get_latest_by": "created_at", + }, + ), + migrations.CreateModel( + name="PackageInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("package", models.CharField(max_length=50)), + ("total_packages", models.IntegerField(default=0)), + ( + "snapshot", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="package_insights", + to="insights.dailysnapshot", + ), + ), + ], + ), + migrations.CreateModel( + name="PackageCWEInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("cwe_id", models.CharField(max_length=50)), + ("count", models.IntegerField()), + ( + "package_insight", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="cwes", + to="insights.packageinsight", + ), + ), + ], + ), + migrations.CreateModel( + name="PackageNameInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("name", models.CharField(max_length=255)), + ("count", models.IntegerField()), + ( + "package_insight", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="names", + to="insights.packageinsight", + ), + ), + ], + ), + migrations.CreateModel( + name="SeverityInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "buckets", + django.contrib.postgres.fields.ArrayField( + base_field=models.IntegerField(), + default=list, + help_text="Scores mapped to buckets 0-10 (e.g. 0.0-0.9 -> index 0)", + size=10, + ), + ), + ( + "snapshot", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="severity_insight", + to="insights.dailysnapshot", + ), + ), + ], + ), + migrations.CreateModel( + name="ImporterInsight", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("importer", models.CharField(max_length=100)), + ("total_advisories", models.IntegerField(default=0)), + ("advisories_with_packages", models.IntegerField(default=0)), + ("advisories_with_ghost_packages", models.IntegerField(default=0)), + ("advisories_with_kev", models.IntegerField(default=0)), + ("advisories_with_metasploit", models.IntegerField(default=0)), + ("advisories_with_exploitdb", models.IntegerField(default=0)), + ( + "snapshot", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="importer_insights", + to="insights.dailysnapshot", + ), + ), + ], + options={ + "constraints": [ + models.UniqueConstraint( + fields=("snapshot", "importer"), name="unique_snapshot_importer" + ) + ], + }, + ), + migrations.AddConstraint( + model_name="packageinsight", + constraint=models.UniqueConstraint( + fields=("snapshot", "package"), name="unique_snapshot_package" + ), + ), + migrations.AddConstraint( + model_name="packagecweinsight", + constraint=models.UniqueConstraint( + fields=("package_insight", "cwe_id"), name="unique_package_cwe" + ), + ), + migrations.AddConstraint( + model_name="packagenameinsight", + constraint=models.UniqueConstraint( + fields=("package_insight", "name"), name="unique_package_name" + ), + ), + ] diff --git a/insights/migrations/__init__.py b/insights/migrations/__init__.py new file mode 100644 index 000000000..20854f2ad --- /dev/null +++ b/insights/migrations/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# diff --git a/insights/models.py b/insights/models.py new file mode 100644 index 000000000..ed01830ce --- /dev/null +++ b/insights/models.py @@ -0,0 +1,108 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from dataclasses import dataclass +from typing import Any +from typing import Callable +from typing import Dict +from typing import Optional + +from django.contrib.postgres.fields import ArrayField +from django.db import models + + +class DailySnapshot(models.Model): + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + get_latest_by = "created_at" + ordering = ["-created_at"] + + +class PackageInsight(models.Model): + snapshot = models.ForeignKey( + DailySnapshot, related_name="package_insights", on_delete=models.CASCADE + ) + package = models.CharField(max_length=50) + total_packages = models.IntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["snapshot", "package"], name="unique_snapshot_package") + ] + + +class PackageNameInsight(models.Model): + package_insight = models.ForeignKey( + PackageInsight, related_name="names", on_delete=models.CASCADE + ) + name = models.CharField(max_length=255) + count = models.IntegerField() + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["package_insight", "name"], name="unique_package_name") + ] + + +class PackageCWEInsight(models.Model): + package_insight = models.ForeignKey( + PackageInsight, related_name="cwes", on_delete=models.CASCADE + ) + cwe_id = models.CharField(max_length=50) + count = models.IntegerField() + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["package_insight", "cwe_id"], name="unique_package_cwe") + ] + + +class SeverityInsight(models.Model): + snapshot = models.OneToOneField( + DailySnapshot, related_name="severity_insight", on_delete=models.CASCADE + ) + buckets = ArrayField( + models.IntegerField(), + size=10, + default=list, + help_text="Scores mapped to buckets 0-10 (e.g. 0.0-0.9 -> index 0)", + ) + + +class ImporterInsight(models.Model): + snapshot = models.ForeignKey( + DailySnapshot, related_name="importer_insights", on_delete=models.CASCADE + ) + importer = models.CharField(max_length=100) + total_advisories = models.IntegerField(default=0) + advisories_with_packages = models.IntegerField(default=0) + advisories_with_ghost_packages = models.IntegerField(default=0) + advisories_with_kev = models.IntegerField(default=0) + advisories_with_metasploit = models.IntegerField(default=0) + advisories_with_exploitdb = models.IntegerField(default=0) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["snapshot", "importer"], name="unique_snapshot_importer" + ) + ] + + +@dataclass(frozen=True) +class ChartDefinition: + + id: str + title: str + panel: str + chart_type: str + formatter_fn: Callable[[Dict[str, Any]], Dict[str, Any]] + collect_fn: Optional[Callable] = None + is_per_package: bool = False + has_search: bool = False diff --git a/insights/static/insights/css/insights.css b/insights/static/insights/css/insights.css new file mode 100644 index 000000000..88c6f21e9 --- /dev/null +++ b/insights/static/insights/css/insights.css @@ -0,0 +1,308 @@ +/* Palette and CSS variables */ +:root { + --bulma-primary: #00D1B2; + --bulma-link: #3273DC; + --bulma-info: #209CEE; + --bulma-success: #48C774; + --bulma-warning: #FFDD57; + --bulma-danger: #FF3860; + --bulma-orange: #FF470F; + --bulma-purple: #B86BFF; + --bulma-grey: #7A7A7A; + --bulma-primary-dark: #00947e; + --bulma-black: #0A0A0A; + --bulma-white: #FFFFFF; +} + +/* Dashboard layout */ +.insights-layout { + display: flex; + align-items: flex-start; + gap: 1.5rem; + padding: 1.5rem 1rem; +} + +/* Override opacity of scatter plot bubbles */ +.bb-circles circle { + opacity: 0.7 !important; +} + +.insights-sidenav { + width: 220px; + flex-shrink: 0; + position: sticky; + top: 5rem; + height: calc(100vh - 7rem); + display: flex; + flex-direction: column; +} + +.insights-sidenav nav.menu { + flex: 1; + overflow-y: auto; + min-height: 0; + margin-bottom: 1rem; +} + +.insights-sidenav .menu-label { + color: var(--bulma-black); + font-weight: 600; +} + +.insights-sidenav .menu-list { + list-style: none !important; + padding-left: 0; + margin-left: 0; +} + +.insights-sidenav .menu-list li { + margin-bottom: 0.25rem; +} + +.insights-sidenav .menu-list a { + color: var(--bulma-black); + padding-left: 1rem; + border-radius: 0 4px 4px 0; +} + +.insights-sidenav .menu-list a:hover { + background-color: #f6f8fa; + color: var(--bulma-black); +} + +.insights-sidenav .menu-list a.is-active { + background-color: #f0f5fa; + color: var(--bulma-link); + font-weight: 600; + border-left: 3px solid var(--bulma-link); + padding-left: calc(1rem - 3px); +} + +.insights-main { + flex: 1; + min-width: 0; +} + +.chart-inner { + max-width: 860px; + margin-left: auto; + margin-right: auto; +} + +.chart-row { + margin-bottom: 1.25rem; + padding: 1.25rem 1.5rem; + background: var(--bulma-white); +} + +.chart-row h3 { + margin-bottom: 0.75rem; + font-size: 1rem; + font-weight: 600; + color: #2c3e50; +} + + + +/* Chart containers and loading state */ +.bb svg { + width: 100% !important; +} + +.chart-container { + width: 100%; + min-height: 320px; +} + +.chart-container.is-pie { + min-height: 420px; +} + +.chart-loading { + display: flex; + align-items: center; + justify-content: center; + min-height: 200px; + color: var(--bulma-grey); + font-size: 0.9rem; +} + + + +/* Billboard.js tooltip overrides */ +.bb-tooltip-container table.bb-tooltip { + min-width: 260px; + white-space: nowrap; +} + +.bb-tooltip-container table.bb-tooltip th { + padding: 0.4rem 0.8rem !important; +} + +.bb-tooltip-container table.bb-tooltip td { + padding: 0.3rem 0.8rem !important; +} + +.bb-tooltip-container table.bb-tooltip td.value, +.bb-tooltip-container table.bb-tooltip td:last-child { + text-align: right; + font-variant-numeric: tabular-nums; + font-weight: 600; + padding-right: 1.2rem !important; +} + +.sidenav-footer { + margin-top: auto; + padding: 1rem 0; + border-top: 1px solid #e8e8e8; +} + +.snapshot-text { + font-size: 0.75rem; + color: var(--bulma-black); + line-height: 1.4; +} + +.snapshot-text strong { + color: var(--bulma-black); +} + +/* Classes replacing inline styles */ +.panel-section { + background-color: var(--bulma-white); + overflow: hidden; +} + +.flat-panel { + margin-bottom: 0; + box-shadow: none; +} + +.flat-panel-heading { + border-radius: 0; + display: flex; + justify-content: space-between; + align-items: center; +} + +.w-full { + max-width: 100%; +} + +.card-chart-row { + border: 1px solid #e8e8e8; + border-radius: 6px; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.02); +} + +.card-chart-title { + border-bottom: 1px solid #e8e8e8; + padding-bottom: 0.75rem; + margin-bottom: 1.25rem; +} + +/* Package Panel Specific Styles */ +.chart-dropdown-wrapper { + display: flex; + justify-content: flex-end; + margin-bottom: 8px; +} + +.severity-chart-layout { + display: flex; + align-items: flex-start; + gap: 0; + width: 100%; + font-family: inherit; +} + +.severity-table-wrapper { + flex: 0 0 380px; + min-width: 260px; +} + +.severity-table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} + +.sev-th-left { + text-align: left; + padding: 4px 8px; + color: #555; + font-weight: 600; + border-bottom: 1px solid #ddd; +} + +.sev-th-right { + text-align: right; + padding: 4px 8px; + color: #555; + font-weight: 600; + border-bottom: 1px solid #ddd; +} + +.sev-tfoot-tr { + border-top: 1px solid #ddd; +} + +.sev-td-total-label { + padding: 5px 8px; + font-weight: 600; + color: #333; +} + +.sev-td-total-value { + padding: 5px 8px; + text-align: right; + font-weight: 600; + color: #2563eb; +} + +.severity-bb-container { + flex: 1; + min-height: 280px; + margin-top: 52px; +} + +.sev-td-bucket { + padding: 4px 8px; + vertical-align: middle; +} + +.sev-bucket-label { + display: inline-block; + width: 2.5em; + color: #444; +} + +.sev-bucket-bar { + display: inline-block; + height: 7px; + border-radius: 2px; + vertical-align: middle; + margin-left: 4px; +} + +.sev-td-count { + padding: 4px 8px; + text-align: right; + color: #2563eb; +} + +.sev-empty { + color: #aaa; +} + +.card-chart-title-with-search { + display: flex; + justify-content: space-between; + align-items: center; +} + +.severity-search-wrapper { + display: flex; + align-items: center; + margin-left: auto; +} \ No newline at end of file diff --git a/insights/static/insights/js/core.js b/insights/static/insights/js/core.js new file mode 100644 index 000000000..3b046d6e9 --- /dev/null +++ b/insights/static/insights/js/core.js @@ -0,0 +1,57 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderers } from './renderers.js'; + +export function renderChartWithData(chartId, key, chartDataMap = {}) { + // Handle dropdown charts with key + const chartData = key && chartDataMap[key] ? chartDataMap[key] : chartDataMap; + const chartContainer = document.getElementById(`chart-${chartId}`); + if (!chartContainer) return; + + if (!chartData?.columns?.length) { + chartContainer.innerHTML = "

No data available.

"; + return; + } + + const type = chartContainer.dataset.chartType; + if (renderers[type]) renderers[type](chartId, chartData); +} + +export function initDropdownChart(chartId, chartData, defaultLabel) { + const chartContainer = document.getElementById(`chart-${chartId}`); + if (!chartData || !chartContainer || chartContainer.previousElementSibling?.classList.contains("chart-dropdown-wrapper")) return; + + const options = Object.keys(chartData).filter(k => k !== "global"); + if (options.length) { + chartContainer.insertAdjacentHTML("beforebegin", ` +
+
+ +
+
+ `); + chartContainer.previousElementSibling.querySelector("select").addEventListener("change", ev => + renderChartWithData(chartId, ev.target.value, chartData) + ); + } + renderChartWithData(chartId, "global", chartData); +} + +export function initDashboard() { + const layout = document.querySelector(".insights-layout"); + if (!layout) return; + const data = JSON.parse(document.getElementById("insights-snapshot-data")?.textContent || "{}"); + document.dispatchEvent(new CustomEvent('insightsDataLoaded', { detail: { data, snapshotData: data.snapshot_data || {}, panelId: layout.dataset.panel } })); +} + +setTimeout(initDashboard, 0); diff --git a/insights/static/insights/js/importer_panel.js b/insights/static/insights/js/importer_panel.js new file mode 100644 index 000000000..4840e23a2 --- /dev/null +++ b/insights/static/insights/js/importer_panel.js @@ -0,0 +1,19 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { initDropdownChart } from './core.js'; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "importer_panel") return; + const { snapshotData } = e.detail; + + ["importer-empty-pkg-bar", "importer-exploit-bar"].forEach(id => { + initDropdownChart(id, snapshotData[id], "Top 5 Importers"); + }); +}); diff --git a/insights/static/insights/js/package_panel.js b/insights/static/insights/js/package_panel.js new file mode 100644 index 000000000..ea7fe49e7 --- /dev/null +++ b/insights/static/insights/js/package_panel.js @@ -0,0 +1,28 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderChartWithData, initDropdownChart } from './core.js'; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "package_panel") return; + + const { snapshotData } = e.detail; + const searchData = document.getElementById("chart-search-data")?.textContent; + const parsedSearch = searchData ? JSON.parse(searchData) : null; + + initDropdownChart("pkg-name-donut", snapshotData["pkg-name-donut"], "All Packages"); + + renderChartWithData( + "pkg-cwe-bar", + parsedSearch ? null : "global", + parsedSearch?.["pkg-cwe-bar"] || snapshotData["pkg-cwe-bar"] + ); + + renderChartWithData("pkg-dist-donut", "global", snapshotData["pkg-dist-donut"]); +}); diff --git a/insights/static/insights/js/renderers.js b/insights/static/insights/js/renderers.js new file mode 100644 index 000000000..7f98261e0 --- /dev/null +++ b/insights/static/insights/js/renderers.js @@ -0,0 +1,181 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +const getCssVar = (name) => getComputedStyle(document.documentElement).getPropertyValue(name).trim(); + +const paletteVars = [ + "--bulma-primary", + "--bulma-link", + "--bulma-info", + "--bulma-success", + "--bulma-warning", + "--bulma-danger", + "--bulma-orange", + "--bulma-purple", + "--bulma-grey", + "--bulma-primary-dark", +]; + +const getPalette = () => paletteVars.map(getCssVar).filter(Boolean); + +const getBucketColors = () => [ + getCssVar("--bulma-success"), getCssVar("--bulma-success"), getCssVar("--bulma-success"), getCssVar("--bulma-success"), // 0-3: Success + getCssVar("--bulma-warning"), getCssVar("--bulma-warning"), getCssVar("--bulma-warning"), // 4-6: Warning + getCssVar("--bulma-orange"), // 7: Orange + getCssVar("--bulma-danger"), getCssVar("--bulma-danger") // 8-9: Danger +]; + +const formatWholeNumbersOnly = x => Number.isInteger(x) ? x : ""; + +export const renderers = { + donut(id, config) { + const bbConfig = { + bindto: `#chart-${id}`, + data: { columns: config.columns, type: "donut", colors: { "Others": "#1b1b1b3a" } }, + color: { pattern: getPalette() }, + legend: { show: true } + }; + + if (config.others_list?.length) { + bbConfig.tooltip = { + contents(d, defaultTitle, defaultVal, color) { + if (d[0].id !== "Others") return this.internal.getTooltipContent(d, defaultTitle, defaultVal, color); + + //Customize tooltip to show table for Others + const total = config.columns.reduce((sum, col) => sum + col[1], 0); + let html = ""; + config.others_list.forEach(([name, val]) => { + html += ``; + }); + return html + "
Others
${name}${val.toLocaleString()} (${((val / total) * 100).toFixed(1)}%)
"; + } + }; + } + bb.generate(bbConfig); + }, + + colored_bar(id, config) { + const monoColor = getCssVar("--bulma-link") + bb.generate({ + bindto: `#chart-${id}`, + data: { x: "x", columns: config.columns, type: "bar", color: () => monoColor }, + axis: { + rotated: true, + x: { type: "category", label: { text: "CWE", position: "outer-middle" } }, + y: { label: { text: "Advisories", position: "outer-center" }, tick: { format: formatWholeNumbersOnly } } + }, + tooltip: { format: { title: x => config.full_labels?.[x] || x, value: val => val.toLocaleString() } }, + legend: { show: false } + }); + }, + + importer_bar(id, config) { + bb.generate({ + bindto: `#chart-${id}`, + data: { + x: "x", + columns: [["x", ...config.x_categories], ...config.columns], + type: "bar", + colors: { + total_advisories: getCssVar("--bulma-link"), + advisories_with_packages: getCssVar("--bulma-primary"), + advisories_without_packages: getCssVar("--bulma-danger"), + advisories_with_exploits: getCssVar("--bulma-success"), + advisories_without_exploits: getCssVar("--bulma-danger"), + advisories_with_kev: getCssVar("--bulma-warning"), + advisories_with_metasploit: getCssVar("--bulma-purple"), + advisories_with_exploitdb: getCssVar("--bulma-success"), + advisories_with_ghost_packages: getCssVar("--bulma-orange"), + advisories_without_ghost_packages: getCssVar("--bulma-primary-dark"), + }, + names: { + total_advisories: "All Advisories", advisories_with_packages: "Advisories with a Package", + advisories_without_packages: "Advisories without a Package", advisories_with_exploits: "Advisories with an Exploit", + advisories_without_exploits: "Advisories without an Exploit", advisories_with_kev: "Advisories with KEV", + advisories_with_metasploit: "Advisories with Metasploit", advisories_with_exploitdb: "Advisories with Exploit-DB", + advisories_with_ghost_packages: "Advisories with a Ghost Package", advisories_without_ghost_packages: "Advisories without a Ghost Package" + }, + groups: config.groups + }, + axis: { rotated: true, x: { type: "category" }, y: { tick: { format: formatWholeNumbersOnly } } }, + bar: { width: { ratio: 0.8 } }, tooltip: { grouped: true }, legend: { show: true } + }); + }, + + scatter(id, config) { + const [, ...buckets] = config.columns[0]; + const [, ...counts] = config.columns[1]; + const BUCKET_COLORS = getBucketColors(); + + const rows = document.getElementById(`chart-${id}-rows`); + const total = document.getElementById(`chart-${id}-total`); + const rowTemplate = document.getElementById(`chart-${id}-row-template`); + + if (!rows || !total || !rowTemplate) return; + + const maxCount = Math.max(1, ...counts); + const frag = document.createDocumentFragment(); + + // Build the table rows for each CVSS bucket + buckets.forEach((bucket, i) => { + const count = counts[i]; + const clone = rowTemplate.content.cloneNode(true); + + // Set bucket label (e.g. "9-10") + const labelNode = clone.querySelector('.sev-bucket-label'); + labelNode.textContent = bucket; + + // Calculate size and paint the colored bubble + const barNode = clone.querySelector('.sev-bucket-bar'); + const barWidth = Math.max(2, (count / maxCount) * 140); + barNode.style.width = `${barWidth}px`; + barNode.style.background = BUCKET_COLORS[i]; + + // Display the vulnerability count on tooltip + const countNode = clone.querySelector('.sev-td-count'); + countNode.title = `CVSS ${bucket}: ${count.toLocaleString()}`; + countNode.innerHTML = count.toLocaleString(); + + frag.appendChild(clone); + }); + + // Render the completed table to the DOM + rows.innerHTML = ""; + rows.appendChild(frag); + const sumOfCounts = counts.reduce((sum, count) => sum + count, 0); + total.textContent = sumOfCounts.toLocaleString(); + + bb.generate({ + bindto: `#chart-${id}-bb`, + data: { + x: "x", columns: config.columns, type: "bubble", + color: (defaultColor, dataPoint) => dataPoint.x !== undefined ? getBucketColors()[dataPoint.x] || defaultColor : defaultColor, + labels: false + }, + bubble: { maxR: 55 }, + axis: { + x: { + type: "category", + tick: { multiline: false }, + label: { text: "CVSS Score Range", position: "outer-center" } + }, + y: { show: false, min: 0, max: maxCount * 1.2, padding: { top: 70, bottom: 0 } } + }, + grid: { x: { show: true } }, + legend: { show: false }, + tooltip: { + format: { + title: x => `CVSS ${buckets[x]}`, + name: () => "Advisories", + value: val => val.toLocaleString() + } + } + }); + } +}; diff --git a/insights/static/insights/js/severity_panel.js b/insights/static/insights/js/severity_panel.js new file mode 100644 index 000000000..d5321a68d --- /dev/null +++ b/insights/static/insights/js/severity_panel.js @@ -0,0 +1,23 @@ +// +// Copyright (c) nexB Inc. and others. All rights reserved. +// VulnerableCode is a trademark of nexB Inc. +// SPDX-License-Identifier: Apache-2.0 +// See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +// See https://github.com/aboutcode-org/vulnerablecode for support or download. +// See https://aboutcode.org for more information about nexB OSS projects. +// + +import { renderChartWithData } from './core.js'; + +document.addEventListener('insightsDataLoaded', (e) => { + if (e.detail.panelId !== "severity_panel") return; + + const chartId = "severity-scatter-plot"; + const searchData = document.getElementById("chart-search-data")?.textContent; + + if (searchData) { + renderChartWithData(chartId, null, JSON.parse(searchData)); + } else { + renderChartWithData(chartId, "global", e.detail.snapshotData[chartId]); + } +}); diff --git a/insights/templates/insights/components/chart_card.html b/insights/templates/insights/components/chart_card.html new file mode 100644 index 000000000..a5c5aaed0 --- /dev/null +++ b/insights/templates/insights/components/chart_card.html @@ -0,0 +1,45 @@ +
+

+ {{ chart.title }} + {% if chart.has_search %} +
+ {% if not search_query %} + Showing All Packages + {% else %} + Show All Packages + {% endif %} + +
+
+
+ +
+
+ +
+
+ {% if search_error %} +

{{ search_error }}

+ {% endif %} +
+
+ {% endif %} +

+ + + {% if chart.chart_type == 'line' %} +
+
+ {% endif %} +
diff --git a/insights/templates/insights/components/package_panel_donuts.html b/insights/templates/insights/components/package_panel_donuts.html new file mode 100644 index 000000000..db589448b --- /dev/null +++ b/insights/templates/insights/components/package_panel_donuts.html @@ -0,0 +1,12 @@ +
+
+
+

{{ panel.charts.0.title }}

+
+
+
+

{{ panel.charts.1.title }}

+
+
+
+
diff --git a/insights/templates/insights/components/severity_scatter_chart.html b/insights/templates/insights/components/severity_scatter_chart.html new file mode 100644 index 000000000..a2924f96e --- /dev/null +++ b/insights/templates/insights/components/severity_scatter_chart.html @@ -0,0 +1,30 @@ +
+
+ + + + + + + + + + + + + + + +
CVSS Score RangeAdvisories
Total
+ +
+
+
diff --git a/insights/templates/insights/dashboard.html b/insights/templates/insights/dashboard.html new file mode 100644 index 000000000..bc8055219 --- /dev/null +++ b/insights/templates/insights/dashboard.html @@ -0,0 +1,74 @@ +{% extends "base.html" %} +{% load static %} + +{% block title %}Insights{% endblock %} + +{% block extrahead %} + + +{% endblock %} + +{% block content %} +
+ + + +
+ {% for panel in panels %} + {% if panel.id == current_panel_id %} +
+
+
+ {{ panel.label }} +
+
+ +
+ {% if panel.layout == 'split_top' %} + {% include 'insights/components/package_panel_donuts.html' with panel=panel %} + {% for chart in panel.charts|slice:"2:" %} + {% include 'insights/components/chart_card.html' with chart=chart current_panel_id=current_panel_id search_query=search_query search_error=search_error %} + {% endfor %} + {% else %} + {% for chart in panel.charts %} + {% include 'insights/components/chart_card.html' with chart=chart current_panel_id=current_panel_id search_query=search_query search_error=search_error %} + {% endfor %} + {% endif %} +
+
+ {% endif %} + {% endfor %} +
+
+{% endblock %} + +{% block scripts %} + + +{{ snapshot_dict|json_script:"insights-snapshot-data" }} +{% if search_results_dict %} +{{ search_results_dict|json_script:"chart-search-data" }} +{% endif %} + + + + +{% endblock %} \ No newline at end of file diff --git a/insights/tests/test_importer_panel.py b/insights/tests/test_importer_panel.py new file mode 100644 index 000000000..23a063404 --- /dev/null +++ b/insights/tests/test_importer_panel.py @@ -0,0 +1,68 @@ +from django.test import TestCase + +from insights.charts.importer_panel import exploits_queryset +from insights.charts.importer_panel import packages_queryset +from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import ImpactedPackageAffecting +from vulnerabilities.models import PackageV2 + + +def create_adv(avid, unique_id): + return AdvisoryV2.objects.create( + avid=f"github_osv/{avid}", + datasource_id="github_osv", + unique_content_id=unique_id, + is_latest=True, + pipeline_id="github_osv_pipeline", + advisory_id=avid, + url="https://example.com/" + avid, + ) + + +class TestImporterPanelQuerysets(TestCase): + def test_packages_queryset(self): + advisory_with_package = create_adv("GHSA-1", "1") + pkg1 = PackageV2.objects.create(type="npm", name="test1", version="1.0.0") + impact1 = ImpactedPackage.objects.create(advisory=advisory_with_package) + ImpactedPackageAffecting.objects.create(impacted_package=impact1, package=pkg1) + + advisory_with_ghost_package = create_adv("GHSA-2", "2") + pkg2 = PackageV2.objects.create(type="npm", name="test2", version="2.0.0", is_ghost=True) + impact2 = ImpactedPackage.objects.create(advisory=advisory_with_ghost_package) + ImpactedPackageAffecting.objects.create(impacted_package=impact2, package=pkg2) + + advisory_without_package = create_adv("GHSA-3", "3") + + qs = list(packages_queryset()) + + self.assertEqual(len(qs), 1) + stats = qs[0] + self.assertEqual(stats["datasource_id"], "github_osv") + self.assertEqual(stats["total_advisories"], 3) + self.assertEqual(stats["advisories_with_packages"], 2) # First two have packages + self.assertEqual(stats["advisories_with_ghost_packages"], 1) + + def test_exploits_queryset(self): + advisory_with_kev = create_adv("GHSA-1", "1") + AdvisoryExploit.objects.create(advisory=advisory_with_kev, data_source="KEV") + + advisory_with_metasploit = create_adv("GHSA-2", "2") + AdvisoryExploit.objects.create(advisory=advisory_with_metasploit, data_source="Metasploit") + + advisory_with_exploitdb = create_adv("GHSA-3", "3") + AdvisoryExploit.objects.create(advisory=advisory_with_exploitdb, data_source="Exploit-DB") + + advisory_without_exploits = create_adv("GHSA-4", "4") + + qs = list(exploits_queryset()) + + self.assertEqual(len(qs), 1) + stats = qs[0] + self.assertEqual(stats["datasource_id"], "github_osv") + + # GHSA-4 shouldn't count in these + self.assertEqual(stats["advisories_with_kev"], 1) + self.assertEqual(stats["advisories_with_metasploit"], 1) + self.assertEqual(stats["advisories_with_exploitdb"], 1) diff --git a/insights/tests/test_insights_snapshot_pipeline.py b/insights/tests/test_insights_snapshot_pipeline.py new file mode 100644 index 000000000..be0ed2f43 --- /dev/null +++ b/insights/tests/test_insights_snapshot_pipeline.py @@ -0,0 +1,37 @@ +from django.test import TestCase + +from insights.insights_snapshot_pipeline import InsightsSnapshotPipeline +from insights.models import DailySnapshot +from insights.tests.test_importer_panel import create_adv +from vulnerabilities.models import AdvisorySeverity +from vulnerabilities.models import PackageV2 + + +class TestInsightsSnapshotPipeline(TestCase): + def test_pipeline_execution_with_data(self): + """Test pipeline populates insights based on existing data.""" + + PackageV2.objects.create(type="pypi", name="django", version="1.0.0") + PackageV2.objects.create(type="npm", name="lodash", version="1.0.0") + + # Create Advisory for Importer and Severity charts + advisory_1 = create_adv("GHSA-1234", "1") + severity_1 = AdvisorySeverity.objects.create(scoring_system="cvssv3.1", value="9.8") + advisory_1.severities.add(severity_1) + + pipeline = InsightsSnapshotPipeline() + pipeline.execute() + + self.assertEqual(DailySnapshot.objects.count(), 1) + snapshot = DailySnapshot.objects.first() + + # Verify captured package types (pypi, npm) and 'global' + self.assertEqual(snapshot.package_insights.count(), 3) + + # Verify ImporterInsight was created for 'github_osv' + self.assertEqual(snapshot.importer_insights.count(), 1) + self.assertTrue(snapshot.importer_insights.filter(importer="github_osv").exists()) + + # Verify SeverityInsight was generated + self.assertTrue(hasattr(snapshot, "severity_insight")) + self.assertEqual(snapshot.severity_insight.buckets[9], 1) diff --git a/insights/tests/test_package_panel.py b/insights/tests/test_package_panel.py new file mode 100644 index 000000000..5bae820bb --- /dev/null +++ b/insights/tests/test_package_panel.py @@ -0,0 +1,56 @@ +from django.test import TestCase + +from insights.charts.package_panel import ecosystem_distribution_queryset +from insights.charts.package_panel import top_packages_queryset +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import ImpactedPackageAffecting +from vulnerabilities.models import PackageV2 + + +class TestPackagePanelQuerysets(TestCase): + def test_ecosystem_distribution_queryset(self): + PackageV2.objects.create(type="npm", name="test1", version="1.0.0") + PackageV2.objects.create(type="npm", name="test2", version="1.0.0") + PackageV2.objects.create(type="pypi", name="test3", version="1.0.0") + PackageV2.objects.create(type="gem", name="test4", version="1.0.0") + + tracked_types = ["npm", "pypi"] + + qs = list(ecosystem_distribution_queryset(tracked_types)) + stats = {item["type"]: item["total_package_count"] for item in qs} + + self.assertEqual(len(stats), 2) # gem should be excluded + self.assertEqual(stats["npm"], 2) + self.assertEqual(stats["pypi"], 1) + + def test_top_packages_queryset(self): + django_v1 = PackageV2.objects.create(type="pypi", name="django", version="1.0.0") + django_v2 = PackageV2.objects.create(type="pypi", name="django", version="2.0.0") + flask_v1 = PackageV2.objects.create(type="pypi", name="flask", version="1.0.0") + + def create_impact(advisory_id, package): + adv = AdvisoryV2.objects.create( + avid=f"github_osv/{advisory_id}", + datasource_id="github_osv", + unique_content_id=advisory_id, + is_latest=True, + pipeline_id="github_osv_pipeline", + advisory_id=advisory_id, + url=f"https://github.com/advisories/{advisory_id}", + ) + impact = ImpactedPackage.objects.create(advisory=adv) + ImpactedPackageAffecting.objects.create(impacted_package=impact, package=package) + + create_impact("GHSA-1111", django_v1) + create_impact("GHSA-2222", django_v2) + create_impact("GHSA-3333", django_v2) + create_impact("GHSA-4444", flask_v1) + + qs = list(top_packages_queryset("pypi")) + + self.assertEqual(len(qs), 2) + self.assertEqual(qs[0]["name"], "django") + self.assertEqual(qs[0]["count"], 3) + self.assertEqual(qs[1]["name"], "flask") + self.assertEqual(qs[1]["count"], 1) diff --git a/insights/tests/test_severity_panel.py b/insights/tests/test_severity_panel.py new file mode 100644 index 000000000..cbe8e8291 --- /dev/null +++ b/insights/tests/test_severity_panel.py @@ -0,0 +1,41 @@ +from django.test import TestCase + +from insights.charts.severity_panel import iter_severity_insights +from vulnerabilities.models import AdvisorySeverity +from vulnerabilities.models import AdvisoryV2 + + +class TestSeverityPanelQuerysets(TestCase): + def test_iter_severity_insights(self): + advisory1 = AdvisoryV2.objects.create( + avid="github_osv/GHSA-1", + datasource_id="github_osv", + unique_content_id="1", + is_latest=True, + pipeline_id="github_osv_pipeline", + advisory_id="GHSA-1", + url="https://example.com/GHSA-1", + ) + severity1 = AdvisorySeverity.objects.create(scoring_system="cvssv3.1", value="9.8") + advisory1.severities.add(severity1) + + advisory2 = AdvisoryV2.objects.create( + avid="github_osv/GHSA-2", + datasource_id="github_osv", + unique_content_id="2", + is_latest=True, + pipeline_id="github_osv_pipeline", + advisory_id="GHSA-2", + url="https://example.com/GHSA-2", + ) + severity2 = AdvisorySeverity.objects.create(scoring_system="cvssv3.1", value="4.5") + advisory2.severities.add(severity2) + + qs = list(iter_severity_insights()) + + self.assertEqual(len(qs), 1) + insight = qs[0] + + self.assertEqual(insight.buckets[9], 1) + self.assertEqual(insight.buckets[4], 1) + self.assertEqual(sum(insight.buckets), 2) diff --git a/insights/tests/test_views.py b/insights/tests/test_views.py new file mode 100644 index 000000000..85e6eedb6 --- /dev/null +++ b/insights/tests/test_views.py @@ -0,0 +1,45 @@ +from django.test import TestCase +from django.urls import reverse + +from insights.models import DailySnapshot +from vulnerabilities.models import PackageV2 + + +class TestInsightsViews(TestCase): + def test_dashboard_redirects_to_default_panel(self): + """Root insights URL should redirect to the first panel (overview_panel).""" + response = self.client.get("/insights/") + self.assertEqual(response.status_code, 302) + self.assertTrue(response.url.endswith("/insights/overview_panel/")) + + def test_dashboard_invalid_panel_redirects(self): + """Invalid panel ID should redirect to default.""" + response = self.client.get("/insights/non_existent_panel/") + self.assertEqual(response.status_code, 302) + self.assertTrue(response.url.endswith("/insights/overview_panel/")) + + def test_dashboard_valid_panel_loads(self): + """Valid panels should load with a 200 OK.""" + response = self.client.get("/insights/package_panel/") + self.assertEqual(response.status_code, 200) + self.assertContains(response, "package_panel") + + def test_dashboard_search_package_panel(self): + """Test CWE search on package panel.""" + PackageV2.objects.create(type="pypi", name="django", version="1.0.0") + response = self.client.get("/insights/package_panel/?q=pkg:pypi/django") + self.assertEqual(response.status_code, 200) + self.assertIn("search_results_dict", response.context) + + def test_dashboard_search_severity_panel(self): + """Test Severity search on severity panel.""" + PackageV2.objects.create(type="pypi", name="flask", version="1.0.0") + response = self.client.get("/insights/severity_panel/?q=pkg:pypi/flask") + self.assertEqual(response.status_code, 200) + self.assertIn("search_results_dict", response.context) + + def test_dashboard_search_no_results(self): + """Test search with no matching packages.""" + response = self.client.get("/insights/severity_panel/?q=pkg:pypi/does_not_exist") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.context["search_error"], "No data available for this query.") diff --git a/insights/urls.py b/insights/urls.py new file mode 100644 index 000000000..0b7a9c4e5 --- /dev/null +++ b/insights/urls.py @@ -0,0 +1,17 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.urls import path + +from insights import views + +urlpatterns = [ + path("", views.insights_dashboard, name="insights-dashboard"), + path("/", views.insights_dashboard, name="insights-panel"), +] diff --git a/insights/utils.py b/insights/utils.py new file mode 100644 index 000000000..e4d0e6471 --- /dev/null +++ b/insights/utils.py @@ -0,0 +1,30 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from vulnerabilities.models import Weakness + + +def get_cwe_label(cwe_id: str) -> str: + num = str(cwe_id).replace("CWE-", "") + + try: + name = Weakness().get_cwe(num).name + return f"CWE-{num} · {name}" if name else f"CWE-{num}" + except KeyError: + return f"CWE-{num}" + + +def format_importer_name(name: str) -> str: + """ + Format importer internal names into human-readable labels. + e.g., 'pysec_importer_v2' -> 'Pysec Importer' + """ + parts = name.rsplit("_v", 1) + if len(parts) == 2 and parts[1].isdigit(): + name = parts[0] + return name.replace("_", " ") diff --git a/insights/views.py b/insights/views.py new file mode 100644 index 000000000..8d01c4fca --- /dev/null +++ b/insights/views.py @@ -0,0 +1,113 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from django.db.models import Count +from django.shortcuts import redirect +from django.shortcuts import render + +from insights.charts import CHARTS +from insights.charts import PANEL_LABELS +from insights.charts import PANEL_LAYOUTS +from insights.charts import PANELS +from insights.charts.package_panel import build_cwe_chart_columns +from insights.charts.package_panel import compute_top_cwes +from insights.charts.severity_panel import aggregate_severity_buckets +from insights.charts.severity_panel import format_severity_chart_data +from insights.models import DailySnapshot +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PackageV2 + + +def _severity_search(search_query): + """Executes a severity search for a given PUrl""" + packages = PackageV2.objects.search(search_query) + error = None + + if packages.exists(): + severities = ( + AdvisoryV2.objects.filter( + impacted_packages__affecting_packages__in=packages, + severities__isnull=False, + severities__scoring_system__icontains="cvss", + ) + .values("severities__value") + .annotate(sev_count=Count("id", distinct=True)) + ) + buckets = aggregate_severity_buckets(severities, "sev_count") + else: + buckets = [0] * 10 + error = "No data available for this query." + + return format_severity_chart_data(buckets), error + + +def _cwe_search(search_query): + """Executes a CWE search for a given PUrl""" + packages = PackageV2.objects.search(search_query) + error = None + + if packages.exists(): + cwe_counts = compute_top_cwes(packages) + chart_data = build_cwe_chart_columns(cwe_counts) + else: + chart_data = {} + error = "No packages found for this query." + + return {"pkg-cwe-bar": chart_data}, error + + +def insights_dashboard(request, panel_id=None): + """Dashboard view to display insights.""" + + if not panel_id or panel_id not in PANELS: + return redirect("insights-panel", panel_id=PANELS[0]) + + panels = [] + for panel_key in PANELS: + panel_charts = [chart for chart in CHARTS if chart.panel == panel_key] + panels.append( + { + "id": panel_key, + "label": PANEL_LABELS[panel_key], + "layout": PANEL_LAYOUTS.get(panel_key, "standard"), + "charts": panel_charts, + } + ) + + try: + snapshot = DailySnapshot.objects.latest() + last_time = snapshot.created_at + + snapshot_data = {} + for chart_def in CHARTS: + if chart_def.formatter_fn: + snapshot_data[chart_def.id] = chart_def.formatter_fn(snapshot) + except DailySnapshot.DoesNotExist: + last_time = None + snapshot_data = {} + + search_query = request.GET.get("q", "").strip() + search_results_dict = None + search_error = None + + if search_query: + if panel_id == "severity_panel": + search_results_dict, search_error = _severity_search(search_query) + elif panel_id == "package_panel": + search_results_dict, search_error = _cwe_search(search_query) + + context = { + "panels": panels, + "current_panel_id": panel_id, + "last_snapshot_time": last_time, + "snapshot_dict": {"snapshot_data": snapshot_data}, + "search_query": search_query, + "search_results_dict": search_results_dict, + "search_error": search_error, + } + return render(request, "insights/dashboard.html", context) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 51694bde9..aeaa57654 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +from insights.insights_snapshot_pipeline import InsightsSnapshotPipeline from vulnerabilities.pipelines.v2_improvers import archive_urls from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 @@ -43,5 +44,6 @@ enhance_with_github_poc.GithubPocsImproverPipeline, mark_unfurl_version_range.MarkUnfurlVersionRangePipeline, group_advisories_for_packages_v2.GroupAdvisoriesForPackages, + InsightsSnapshotPipeline, ] ) diff --git a/vulnerabilities/templates/navbar.html b/vulnerabilities/templates/navbar.html index 530c2521e..b50cc3fb2 100644 --- a/vulnerabilities/templates/navbar.html +++ b/vulnerabilities/templates/navbar.html @@ -1,7 +1,7 @@ {% load utils %} -{% if STAGING %} +