From b7a3982c81a3c71c88556dc1d1c48d7e2e2f34ef Mon Sep 17 00:00:00 2001 From: mattip Date: Tue, 26 May 2026 23:17:45 +0300 Subject: [PATCH] add a dump button to admin page and manage.py import_sqlite_dump file --- codespeed/admin_views.py | 176 ++++++++++++++++++ codespeed/management/__init__.py | 0 codespeed/management/commands/__init__.py | 0 .../management/commands/import_sqlite_dump.py | 95 ++++++++++ speed_pypy/settings.py | 2 - speed_pypy/templates/admin/index.html | 73 ++++++++ speed_pypy/urls.py | 3 + 7 files changed, 347 insertions(+), 2 deletions(-) create mode 100644 codespeed/admin_views.py create mode 100644 codespeed/management/__init__.py create mode 100644 codespeed/management/commands/__init__.py create mode 100644 codespeed/management/commands/import_sqlite_dump.py create mode 100644 speed_pypy/templates/admin/index.html diff --git a/codespeed/admin_views.py b/codespeed/admin_views.py new file mode 100644 index 00000000..d0f869e3 --- /dev/null +++ b/codespeed/admin_views.py @@ -0,0 +1,176 @@ +import os +import sqlite3 +import tempfile +from datetime import datetime, date + +from django.contrib.admin.views.decorators import staff_member_required +from django.http import FileResponse + +from django.db import connection + +SIZE_LIMIT = 95 * 1024 * 1024 # 95 MB + +# Schema for the exported SQLite file, in FK-safe creation order. +# Column names must match Django's generated PostgreSQL column names exactly. +_SCHEMA = """ +PRAGMA foreign_keys = OFF; + +CREATE TABLE codespeed_project ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + repo_type TEXT NOT NULL, + repo_path TEXT NOT NULL, + repo_user TEXT NOT NULL, + repo_pass TEXT NOT NULL, + commit_browsing_url TEXT NOT NULL, + track INTEGER NOT NULL, + default_branch TEXT NOT NULL +); + +CREATE TABLE codespeed_branch ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + project_id INTEGER NOT NULL, + display_on_comparison_page INTEGER NOT NULL +); + +CREATE TABLE codespeed_executable ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL, + project_id INTEGER NOT NULL +); + +CREATE TABLE codespeed_environment ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + cpu TEXT NOT NULL, + memory TEXT NOT NULL, + os TEXT NOT NULL, + kernel TEXT NOT NULL +); + +CREATE TABLE codespeed_benchmark ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + parent_id INTEGER, + source TEXT NOT NULL, + data_type TEXT NOT NULL, + description TEXT NOT NULL, + units_title TEXT NOT NULL, + units TEXT NOT NULL, + lessisbetter INTEGER NOT NULL, + default_on_comparison INTEGER NOT NULL +); + +CREATE TABLE codespeed_revision ( + id INTEGER PRIMARY KEY, + commitid TEXT NOT NULL, + tag TEXT NOT NULL, + date TEXT, + message TEXT NOT NULL, + project_id INTEGER, + author TEXT NOT NULL, + branch_id INTEGER NOT NULL +); + +CREATE TABLE codespeed_result ( + id INTEGER PRIMARY KEY, + value REAL NOT NULL, + std_dev REAL, + val_min REAL, + val_max REAL, + q1 REAL, + q3 REAL, + suite_version TEXT NOT NULL, + date TEXT, + revision_id INTEGER NOT NULL, + executable_id INTEGER NOT NULL, + benchmark_id INTEGER NOT NULL, + environment_id INTEGER NOT NULL +); +""" + +_SMALL_TABLES = [ + 'codespeed_project', + 'codespeed_branch', + 'codespeed_executable', + 'codespeed_environment', + 'codespeed_benchmark', + 'codespeed_revision', +] + + +def _conv(v): + """Convert PostgreSQL Python types to SQLite-safe scalars.""" + if isinstance(v, (datetime, date)): + return v.isoformat() + return v + + +def _build_sqlite(path): + lite = sqlite3.connect(path) + try: + lite.executescript(_SCHEMA) + lite.commit() + + with connection.cursor() as pg: + for table in _SMALL_TABLES: + pg.execute(f'SELECT * FROM {table}') + cols = [d[0] for d in pg.description] + rows = [tuple(_conv(v) for v in row) for row in pg.fetchall()] + if rows: + ph = ', '.join(['?'] * len(cols)) + lite.executemany( + f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({ph})", + rows, + ) + lite.commit() + + # Result table: newest-first, stop at size cap + pg.execute( + 'SELECT * FROM codespeed_result ORDER BY date DESC NULLS LAST' + ) + cols = [d[0] for d in pg.description] + ph = ', '.join(['?'] * len(cols)) + insert_sql = ( + f"INSERT INTO codespeed_result ({', '.join(cols)}) VALUES ({ph})" + ) + while True: + batch = pg.fetchmany(5000) + if not batch: + break + lite.executemany( + insert_sql, + [tuple(_conv(v) for v in row) for row in batch], + ) + lite.commit() + if os.path.getsize(path) > SIZE_LIMIT: + break + finally: + lite.close() + + +@staff_member_required +def download_db(request): + fd, path = tempfile.mkstemp(suffix='.sqlite3') + os.close(fd) + try: + _build_sqlite(path) + f = open(path, 'rb') + os.unlink(path) # unlink now; data survives until f is closed + today = datetime.today().strftime('%Y-%m-%d') + response = FileResponse( + f, + as_attachment=True, + filename=f'codespeed-{today}.sqlite3', + ) + response.set_cookie( + 'codespeed_download_ready', '1', + max_age=60, path='/', samesite='Lax', + ) + return response + except Exception: + if os.path.exists(path): + os.unlink(path) + raise diff --git a/codespeed/management/__init__.py b/codespeed/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/codespeed/management/commands/__init__.py b/codespeed/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/codespeed/management/commands/import_sqlite_dump.py b/codespeed/management/commands/import_sqlite_dump.py new file mode 100644 index 00000000..623f1997 --- /dev/null +++ b/codespeed/management/commands/import_sqlite_dump.py @@ -0,0 +1,95 @@ +""" +Import a codespeed SQLite snapshot into the running database. + +Rows that already exist (by primary key or unique constraint) are silently +skipped, so the command is safe to re-run and merges cleanly into existing +data. + +Usage: + python manage.py import_sqlite_dump codespeed-YYYY-MM-DD.sqlite3 +""" +import sqlite3 +from datetime import datetime + +from django.core.management.base import BaseCommand, CommandError +from django.db import connection + +# Columns that are stored as 0/1 integers in SQLite but need Python bools +# for PostgreSQL's boolean type. +_BOOL_COLS = { + 'codespeed_project': {'track'}, + 'codespeed_branch': {'display_on_comparison_page'}, + 'codespeed_benchmark': {'lessisbetter', 'default_on_comparison'}, +} + +# Columns stored as ISO strings in SQLite that must become datetime objects +# for PostgreSQL's timestamp type. +_DT_COLS = { + 'codespeed_revision': {'date'}, + 'codespeed_result': {'date'}, +} + +# Insertion order respects FK dependencies. +_TABLES = [ + 'codespeed_project', + 'codespeed_branch', + 'codespeed_executable', + 'codespeed_environment', + 'codespeed_benchmark', + 'codespeed_revision', + 'codespeed_result', +] + + +def _adapt(value, col, bool_cols, dt_cols): + if value is None: + return None + if col in bool_cols: + return bool(value) + if col in dt_cols and isinstance(value, str): + return datetime.fromisoformat(value) + return value + + +class Command(BaseCommand): + help = 'Import a SQLite snapshot into the running PostgreSQL database' + + def add_arguments(self, parser): + parser.add_argument('sqlite_file', help='Path to the SQLite dump file') + + def handle(self, *args, **options): + sqlite_file = options['sqlite_file'] + try: + src = sqlite3.connect(sqlite_file) + except Exception as e: + raise CommandError(f'Cannot open {sqlite_file}: {e}') + + src.row_factory = sqlite3.Row + + with connection.cursor() as cur: + for table in _TABLES: + bool_cols = _BOOL_COLS.get(table, set()) + dt_cols = _DT_COLS.get(table, set()) + + rows = src.execute(f'SELECT * FROM {table}').fetchall() + if not rows: + self.stdout.write(f' {table}: 0 rows') + continue + + cols = list(rows[0].keys()) + col_list = ', '.join(cols) + placeholders = ', '.join(['%s'] * len(cols)) + sql = ( + f'INSERT INTO {table} ({col_list}) ' + f'VALUES ({placeholders}) ' + f'ON CONFLICT DO NOTHING' + ) + data = [ + tuple(_adapt(row[c], c, bool_cols, dt_cols) for c in cols) + for row in rows + ] + cur.executemany(sql, data) + self.stdout.write(f' {table}: {len(data)} rows') + + src.close() + self.stdout.write(self.style.SUCCESS('Import complete')) diff --git a/speed_pypy/settings.py b/speed_pypy/settings.py index 47efc6c5..361d4397 100644 --- a/speed_pypy/settings.py +++ b/speed_pypy/settings.py @@ -95,8 +95,6 @@ ] DEF_EXECUTABLES = [ {'name': 'pypy3.11-jit-64', 'project': 'PyPy3.11'}, - {'name': 'pypy3.10-jit-64', 'project': 'PyPy3.10'}, - {'name': 'pypy3.9-jit-64', 'project': 'PyPy3.9'}, ] DEF_ENVIRONMENT = 'benchmarker' CHART_ORIENTATION = 'horizontal' diff --git a/speed_pypy/templates/admin/index.html b/speed_pypy/templates/admin/index.html new file mode 100644 index 00000000..540c9303 --- /dev/null +++ b/speed_pypy/templates/admin/index.html @@ -0,0 +1,73 @@ +{% extends "admin/index.html" %} +{% load i18n %} + +{% block content %} +
+ {% include "admin/app_list.html" with app_list=app_list show_changelinks=True %} + +
+ + + + + + + + + + + + + + + + +
Data tools
SQLite snapshot + Download + +
+
+
+ +{% endblock %} diff --git a/speed_pypy/urls.py b/speed_pypy/urls.py index a06d619f..b149cb30 100644 --- a/speed_pypy/urls.py +++ b/speed_pypy/urls.py @@ -4,7 +4,10 @@ from django.urls import include, re_path from django.contrib import admin +from codespeed import admin_views + urlpatterns = [ + re_path(r'^admin/download-db/$', admin_views.download_db, name='admin-download-db'), re_path(r'^admin/', admin.site.urls), re_path(r'^', include('codespeed.urls')) ]