From 291c80ccaf540dd7b7940745082550f9231295f4 Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Tue, 7 Jul 2026 11:22:04 +0100 Subject: [PATCH 1/3] feat(api): add mirrors endpoints to Django Ninja API Closes part of #199 Signed-off-by: Leonidas Spyropoulos --- api/router.py | 3 +- api/routes/mirrors.py | 141 ++++++++++++++++++++++++++++++++++++++ api/schemas/mirrors.py | 71 +++++++++++++++++++ api/tests/test_mirrors.py | 96 ++++++++++++++++++++++++++ 4 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 api/routes/mirrors.py create mode 100644 api/schemas/mirrors.py create mode 100644 api/tests/test_mirrors.py diff --git a/api/router.py b/api/router.py index eaa1bc51..1622a079 100644 --- a/api/router.py +++ b/api/router.py @@ -1,6 +1,6 @@ from ninja import NinjaAPI -from api.routes import releng +from api.routes import mirrors, releng api = NinjaAPI( version="1", @@ -11,3 +11,4 @@ ) api.add_router("/v1/releng/", releng.router) +api.add_router("/v1/mirrors/", mirrors.router) diff --git a/api/routes/mirrors.py b/api/routes/mirrors.py new file mode 100644 index 00000000..d068fe3d --- /dev/null +++ b/api/routes/mirrors.py @@ -0,0 +1,141 @@ +from django.http import Http404 +from django.shortcuts import get_object_or_404 +from django.utils.timezone import now +from ninja import Router + +from api.schemas.mirrors import ( + MirrorDetailsSchema, + MirrorLocationSchema, + MirrorLocationsSchema, + MirrorLogSchema, + MirrorStatusSchema, + MirrorUrlSchema, + MirrorUrlWithLogsSchema, +) +from mirrors.models import CheckLocation, Mirror +from mirrors.utils import DEFAULT_CUTOFF, get_mirror_statuses + +router = Router(tags=["mirrors"]) + + +def _td_seconds(value) -> int: + return value.days * 24 * 3600 + value.seconds + + +def _td_seconds_or_none(value) -> int | None: + if value is None: + return None + return _td_seconds(value) + + +def _url_to_schema(url) -> MirrorUrlSchema: + return MirrorUrlSchema( + url=url.url, + protocol=str(url.protocol), + last_sync=url.last_sync, + completion_pct=url.completion_pct, + delay=_td_seconds_or_none(url.delay), + duration_avg=url.duration_avg, + duration_stddev=url.duration_stddev, + score=url.score, + active=url.active, + country=str(url.country.name), + country_code=url.country.code, + isos=url.mirror.isos, + ipv4=url.has_ipv4, + ipv6=url.has_ipv6, + details=url.get_full_url(), + ) + + +def _url_with_logs_to_schema(url, cutoff_time) -> MirrorUrlWithLogsSchema: + base = _url_to_schema(url) + logs = [ + MirrorLogSchema( + check_time=log.check_time, + last_sync=log.last_sync, + duration=log.duration, + is_success=log.is_success, + location_id=log.location_id, + error=log.error or None, + ) + for log in url.logs.filter(check_time__gte=cutoff_time).order_by('check_time') + ] + return MirrorUrlWithLogsSchema(**base.dict(), logs=logs) + + +@router.get("/status/", response=MirrorStatusSchema, url_name="mirror-status") +def status(request): + info = get_mirror_statuses() + return MirrorStatusSchema( + version=3, + urls=[_url_to_schema(u) for u in info['urls']], + cutoff=_td_seconds(info["cutoff"]), + last_check=info['last_check'], + num_checks=info['num_checks'], + check_frequency=_td_seconds_or_none(info["check_frequency"]), + ) + + +@router.get("/status/tier/{tier}/", response=MirrorStatusSchema, url_name="mirror-status-tier") +def status_tier(request, tier: int): + if tier not in [t[0] for t in Mirror.TIER_CHOICES]: + raise Http404 + info = get_mirror_statuses() + urls = [u for u in info['urls'] if u.mirror.tier == tier] + return MirrorStatusSchema( + version=3, + urls=[_url_to_schema(u) for u in urls], + cutoff=_td_seconds(info["cutoff"]), + last_check=info['last_check'], + num_checks=info['num_checks'], + check_frequency=_td_seconds_or_none(info["check_frequency"]), + ) + + +@router.get("/locations/", response=MirrorLocationsSchema, url_name="mirror-locations") +def locations(request): + return MirrorLocationsSchema( + version=1, + locations=[ + MirrorLocationSchema( + id=loc.pk, + hostname=loc.hostname, + source_ip=loc.source_ip, + country=str(loc.country.name), + country_code=loc.country.code, + ip_version=loc.ip_version, + ) + for loc in CheckLocation.objects.all().order_by('pk') + ], + ) + + +@router.get("/{name}/", response=MirrorDetailsSchema, url_name="mirror-details") +def mirror_details(request, name: str): + authorized = request.user.is_authenticated + mirror = get_object_or_404(Mirror, name=name) + if not authorized and (not mirror.public or not mirror.active): + raise Http404 + info = get_mirror_statuses(mirror_id=mirror.id, show_all=authorized) + cutoff_time = now() - DEFAULT_CUTOFF + + admin_email = None + alternate_email = None + if authorized and request.user.has_perm('mirrors.change_mirror'): + admin_email = mirror.admin_email + alternate_email = mirror.alternate_email + + return MirrorDetailsSchema( + version=5, + tier=mirror.tier, + upstream=mirror.upstream.name if mirror.upstream else None, + details=mirror.get_full_url(), + urls=[_url_with_logs_to_schema(u, cutoff_time) for u in info['urls']], + cutoff=_td_seconds(info["cutoff"]), + last_check=info['last_check'], + num_checks=info['num_checks'], + check_frequency=_td_seconds_or_none(info["check_frequency"]), + admin_email=admin_email, + alternate_email=alternate_email, + ) diff --git a/api/schemas/mirrors.py b/api/schemas/mirrors.py new file mode 100644 index 00000000..40f5fbdb --- /dev/null +++ b/api/schemas/mirrors.py @@ -0,0 +1,71 @@ +from datetime import datetime + +from ninja import Schema + + +class MirrorLogSchema(Schema): + check_time: datetime + last_sync: datetime | None = None + duration: float | None = None + is_success: bool + location_id: int | None = None + error: str | None = None + + +class MirrorUrlSchema(Schema): + url: str + protocol: str + last_sync: datetime | None = None + completion_pct: float | None = None + delay: int | None = None + duration_avg: float | None = None + duration_stddev: float | None = None + score: float | None = None + active: bool + country: str + country_code: str + isos: bool + ipv4: bool + ipv6: bool + details: str + + +class MirrorUrlWithLogsSchema(MirrorUrlSchema): + logs: list[MirrorLogSchema] = [] + + +class MirrorStatusSchema(Schema): + version: int + urls: list[MirrorUrlSchema] + cutoff: int + last_check: datetime | None = None + num_checks: int + check_frequency: int | None = None + + +class MirrorDetailsSchema(Schema): + version: int + tier: int + upstream: str | None = None + details: str + urls: list[MirrorUrlWithLogsSchema] + cutoff: int + last_check: datetime | None = None + num_checks: int + check_frequency: int | None = None + admin_email: str | None = None + alternate_email: str | None = None + + +class MirrorLocationSchema(Schema): + id: int + hostname: str + source_ip: str + country: str + country_code: str + ip_version: int + + +class MirrorLocationsSchema(Schema): + version: int + locations: list[MirrorLocationSchema] diff --git a/api/tests/test_mirrors.py b/api/tests/test_mirrors.py new file mode 100644 index 00000000..e1eb451b --- /dev/null +++ b/api/tests/test_mirrors.py @@ -0,0 +1,96 @@ +import pytest + +from mirrors.tests.conftest import ( # noqa: F401 + checklocation, + create_mirrorurl, + mirror, + mirrorprotocol, + mirrorurl, +) + + +@pytest.fixture +def public_active_mirror(mirror): # noqa: F811 + mirror.public = True + mirror.active = True + mirror.save() + return mirror + + +def test_status_empty(db, client): + response = client.get('/api/v1/mirrors/status/') + assert response.status_code == 200 + data = response.json() + assert data['version'] == 3 + assert data['urls'] == [] + assert data['num_checks'] == 0 + + +def test_status_tier_invalid(db, client): + response = client.get('/api/v1/mirrors/status/tier/999/') + assert response.status_code == 404 + + +def test_status_tier_valid_empty(db, client): + response = client.get('/api/v1/mirrors/status/tier/0/') + assert response.status_code == 200 + assert response.json()['version'] == 3 + + +def test_status_includes_active_public_mirror_url( + db, client, public_active_mirror, mirrorprotocol # noqa: F811 +): + from mirrors.models import MirrorUrl + url = MirrorUrl.objects.create( + url='https://example.org/', + protocol=mirrorprotocol, + mirror=public_active_mirror, + country='DE', + active=True, + ) + response = client.get('/api/v1/mirrors/status/') + assert response.status_code == 200 + urls = response.json()['urls'] + assert len(urls) == 1 + assert urls[0]['url'] == 'https://example.org/' + assert urls[0]['protocol'] == 'https' + assert urls[0]['country_code'] == 'DE' + url.delete() + + +def test_locations_empty(db, client): + response = client.get('/api/v1/mirrors/locations/') + assert response.status_code == 200 + data = response.json() + assert data['version'] == 1 + assert data['locations'] == [] + + +def test_locations_returns_checklocation(db, client, checklocation): # noqa: F811 + response = client.get('/api/v1/mirrors/locations/') + assert response.status_code == 200 + locs = response.json()['locations'] + assert len(locs) == 1 + assert locs[0]['hostname'] == 'archlinux.org' + assert locs[0]['country_code'] == 'DE' + + +def test_mirror_details_not_found(db, client): + response = client.get('/api/v1/mirrors/does-not-exist/') + assert response.status_code == 404 + + +def test_mirror_details_private_returns_404(db, client, mirror): # noqa: F811 + mirror.public = False + mirror.save() + response = client.get(f'/api/v1/mirrors/{mirror.name}/') + assert response.status_code == 404 + + +def test_mirror_details_public_returns_data(db, client, public_active_mirror): + response = client.get(f'/api/v1/mirrors/{public_active_mirror.name}/') + assert response.status_code == 200 + data = response.json() + assert data['version'] == 5 + assert data['tier'] == public_active_mirror.tier + assert 'admin_email' not in data or data['admin_email'] is None From 9fdccb195b3916edc798c3f7b91bf517295ffa0b Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Tue, 7 Jul 2026 11:23:38 +0100 Subject: [PATCH 2/3] feat(api): add deprecation headers to mirrors legacy JSON endpoints Signed-off-by: Leonidas Spyropoulos --- api/tests/test_deprecation.py | 34 ++++++++++++++++++++++++++++++++++ mirrors/views/api.py | 5 +++++ 2 files changed, 39 insertions(+) diff --git a/api/tests/test_deprecation.py b/api/tests/test_deprecation.py index 5e265a71..39c86238 100644 --- a/api/tests/test_deprecation.py +++ b/api/tests/test_deprecation.py @@ -1,5 +1,39 @@ +from mirrors.tests.conftest import mirror # noqa: F401 + + def test_releng_releases_json_deprecation_headers(db, client): response = client.get('/releng/releases/json/') assert response.status_code == 200 assert response['Deprecation'] == 'true' assert response['Link'] == '; rel="successor-version"' + + +def test_mirror_status_json_deprecation_headers(db, client): + response = client.get('/mirrors/status/json/') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == '; rel="successor-version"' + + +def test_mirror_status_tier_json_deprecation_headers(db, client): + response = client.get('/mirrors/status/tier/0/json/') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == '; rel="successor-version"' + + +def test_mirror_locations_json_deprecation_headers(db, client): + response = client.get('/mirrors/locations/json/') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == '; rel="successor-version"' + + +def test_mirror_details_json_deprecation_headers(db, client, mirror): # noqa: F811 + mirror.public = True + mirror.active = True + mirror.save() + response = client.get(f'/mirrors/{mirror.name}/json/') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == '; rel="successor-version"' diff --git a/mirrors/views/api.py b/mirrors/views/api.py index b0a81b71..e0de9193 100644 --- a/mirrors/views/api.py +++ b/mirrors/views/api.py @@ -7,6 +7,8 @@ from django.utils.timezone import now from django.views.decorators.cache import cache_control +from api.deprecation import deprecated_json_endpoint + from ..models import CheckLocation, Mirror, MirrorLog, MirrorProtocol, MirrorUrl from ..utils import DEFAULT_CUTOFF, get_mirror_statuses @@ -72,6 +74,7 @@ def default(self, obj): @cache_control(max_age=311) +@deprecated_json_endpoint('/api/v1/mirrors/status/') def status_json(request, tier=None): if tier is not None: tier = int(tier) @@ -87,6 +90,7 @@ def status_json(request, tier=None): return response +@deprecated_json_endpoint('/api/v1/mirrors/{name}/') def mirror_details_json(request, name): authorized = request.user.is_authenticated mirror = get_object_or_404(Mirror, name=name) @@ -107,6 +111,7 @@ def mirror_details_json(request, name): return response +@deprecated_json_endpoint('/api/v1/mirrors/locations/') def locations_json(request): data = {} data['version'] = 1 From 680d11b8f4afa31fd5dee1245d4a3b3eb8efb0cd Mon Sep 17 00:00:00 2001 From: Leonidas Spyropoulos Date: Tue, 7 Jul 2026 13:46:11 +0100 Subject: [PATCH 3/3] feat(api): add public read-only endpoints with deprecation headers Extend the Django Ninja API to cover the remaining public read-only JSON endpoints under /api/v1/ Closes part of #199. Closes #532. Signed-off-by: Leonidas Spyropoulos --- api/router.py | 5 +- api/routes/packages.py | 163 ++++++++++++++++++++++++++++++++++ api/routes/public.py | 45 ++++++++++ api/routes/todolists.py | 19 ++++ api/schemas/packages.py | 53 +++++++++++ api/schemas/public.py | 17 ++++ api/schemas/todolists.py | 19 ++++ api/tests/test_deprecation.py | 54 +++++++++++ api/tests/test_packages.py | 72 +++++++++++++++ api/tests/test_public.py | 14 +++ api/tests/test_todolists.py | 29 ++++++ packages/views/__init__.py | 34 ++----- packages/views/display.py | 10 ++- packages/views/search.py | 55 ++---------- public/views.py | 2 + todolists/views.py | 2 + 16 files changed, 513 insertions(+), 80 deletions(-) create mode 100644 api/routes/packages.py create mode 100644 api/routes/public.py create mode 100644 api/routes/todolists.py create mode 100644 api/schemas/packages.py create mode 100644 api/schemas/public.py create mode 100644 api/schemas/todolists.py create mode 100644 api/tests/test_packages.py create mode 100644 api/tests/test_public.py create mode 100644 api/tests/test_todolists.py diff --git a/api/router.py b/api/router.py index 1622a079..9d1be422 100644 --- a/api/router.py +++ b/api/router.py @@ -1,6 +1,6 @@ from ninja import NinjaAPI -from api.routes import mirrors, releng +from api.routes import mirrors, packages, public, releng, todolists api = NinjaAPI( version="1", @@ -12,3 +12,6 @@ api.add_router("/v1/releng/", releng.router) api.add_router("/v1/mirrors/", mirrors.router) +api.add_router("/v1/master-keys/", public.router) +api.add_router("/v1/packages/", packages.router) +api.add_router("/v1/todo/", todolists.router) diff --git a/api/routes/packages.py b/api/routes/packages.py new file mode 100644 index 00000000..51665c1f --- /dev/null +++ b/api/routes/packages.py @@ -0,0 +1,163 @@ +import json +from collections import defaultdict + +from django.contrib.auth.models import User +from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator +from django.http import HttpResponse, HttpResponseBadRequest +from django.shortcuts import get_object_or_404 +from ninja import Router + +from api.schemas.packages import ( + PackageFilesSchema, + PackageSchema, + PackageSearchSchema, +) +from main.models import Package, PackageFile, Soname +from packages.models import PackageRelation +from packages.utils import DEPENDENCY_TYPES, PackageJSONEncoder, attach_maintainers +from packages.views.search import PackageSearchForm, parse_form + +router = Router(tags=["packages"]) + + +def _pkg_to_schema(pkg: Package) -> PackageSchema: + all_deps = list(pkg.depends.all()) + deps_by_type = { + name: [str(d) for d in all_deps if d.deptype == deptype] + for deptype, name in DEPENDENCY_TYPES + } + return PackageSchema( + pkgname=pkg.pkgname, + pkgbase=pkg.pkgbase, + repo=pkg.repo.name.lower(), + arch=pkg.arch.name.lower(), + pkgver=pkg.pkgver, + pkgrel=pkg.pkgrel, + epoch=pkg.epoch, + pkgdesc=pkg.pkgdesc or None, + url=pkg.url or None, + filename=pkg.filename, + compressed_size=pkg.compressed_size, + installed_size=pkg.installed_size, + build_date=pkg.build_date, + last_update=pkg.last_update, + flag_date=pkg.flag_date, + maintainers=[u.username for u in pkg.maintainers], + packager=pkg.packager.username if pkg.packager else None, + groups=[g.name for g in pkg.groups.all()], + licenses=[lic.name for lic in pkg.licenses.all()], + conflicts=[str(c) for c in pkg.conflicts.all()], + provides=[str(p) for p in pkg.provides.all()], + replaces=[str(r) for r in pkg.replaces.all()], + depends=deps_by_type['depends'], + optdepends=deps_by_type['optdepends'], + makedepends=deps_by_type['makedepends'], + checkdepends=deps_by_type['checkdepends'], + ) + + +def _get_package(name: str, repo: str, arch: str) -> Package: + return get_object_or_404( + Package.objects.normal(), + pkgname=name, repo__name__iexact=repo, arch__name=arch, + ) + + +@router.get("/search/", response=PackageSearchSchema, url_name="package-search") +def search(request): + limit = 250 + container = { + 'version': 2, + 'limit': limit, + 'valid': False, + 'results': [], + } + + if request.GET: + form = PackageSearchForm(data=request.GET, + show_staging=request.user.is_authenticated) + if form.is_valid(): + form_limit = form.cleaned_data.get('limit', limit) + limit = min(limit, form_limit) if form_limit else limit + container['limit'] = limit + + packages = Package.objects.select_related('arch', 'repo', 'packager') + if not request.user.is_authenticated: + packages = packages.filter(repo__staging=False) + packages = parse_form(form, packages) + + paginator = Paginator(packages, limit) + container['num_pages'] = paginator.num_pages + container['count'] = paginator.count + + page = form.cleaned_data.get('page') + try: + page = int(page) if page else 1 + except ValueError: + return HttpResponseBadRequest('page parameter is not a number') + container['page'] = page + try: + packages = paginator.page(page) + except PageNotAnInteger: + packages = paginator.page(1) + except EmptyPage: + packages = paginator.page(paginator.num_pages) + + attach_maintainers(packages) + container['results'] = packages + container['valid'] = True + + to_json = json.dumps(container, ensure_ascii=False, cls=PackageJSONEncoder) + return HttpResponse(to_json, content_type='application/json') + + +@router.get("/{repo}/{arch}/{name}/", response=PackageSchema, url_name="package-details") +def details(request, repo: str, arch: str, name: str): + return _pkg_to_schema(_get_package(name, repo, arch)) + + +@router.get("/{repo}/{arch}/{name}/files/", response=PackageFilesSchema, url_name="package-files") +def files(request, repo: str, arch: str, name: str): + pkg = _get_package(name, repo, arch) + # files are inserted in sorted order, so preserve that + fileslist = PackageFile.objects.filter(pkg=pkg).order_by('id') + dir_count = sum(1 for f in fileslist if f.is_directory) + files_count = len(fileslist) - dir_count + return PackageFilesSchema( + pkgname=pkg.pkgname, + repo=pkg.repo.name.lower(), + arch=pkg.arch.name.lower(), + pkg_last_update=pkg.last_update, + files_last_update=pkg.files_last_update, + files_count=files_count, + dir_count=dir_count, + files=[f.directory + (f.filename or '') for f in fileslist], + ) + + +@router.get("/{repo}/{arch}/{name}/sonames/", response=list[str], url_name="package-sonames") +def sonames(request, repo: str, arch: str, name: str): + pkg = _get_package(name, repo, arch) + return list(Soname.objects.filter(pkg=pkg).values_list('name', flat=True)) + + +@router.get("/pkgbase-maintainer", response=dict[str, list[str]], url_name="pkgbase-maintainer") +def pkgbase_maintainer(request): + pkgbases = Package.objects.all().values('pkgbase') + rels = PackageRelation.objects.filter( + type=PackageRelation.MAINTAINER, pkgbase__in=pkgbases + ).values_list('pkgbase', 'user_id').order_by().distinct() + + user_ids = {rel[1] for rel in rels} + users = User.objects.in_bulk(user_ids) + + maintainers = defaultdict(list) + for pkgbase_name, user_id in rels: + maintainers[pkgbase_name].append(users[user_id].username) + + mapping = {} + for row in pkgbases: + pkgbase_name = row['pkgbase'] + if pkgbase_name not in mapping: + mapping[pkgbase_name] = maintainers[pkgbase_name] + return mapping diff --git a/api/routes/public.py b/api/routes/public.py new file mode 100644 index 00000000..9b60bb78 --- /dev/null +++ b/api/routes/public.py @@ -0,0 +1,45 @@ +from datetime import datetime, timezone + +from django.contrib.auth.models import User +from django.db.models import Q +from ninja import Router + +from api.schemas.public import PGPEdgeSchema, PGPKeysSchema, PGPNodeSchema +from devel.models import MasterKey, PGPSignature, UserProfile + +router = Router(tags=["public"]) + + +@router.get("/", response=PGPKeysSchema, url_name="pgp-keys") +def master_keys(request): + profile_ids = UserProfile.allowed_repos.through.objects.values('userprofile_id') + users = User.objects.filter( + is_active=True, userprofile__id__in=profile_ids + ).order_by('first_name', 'last_name') + nodes = [ + PGPNodeSchema( + name=user.get_full_name(), + key=user.userprofile.pgp_key or None, + group='packager', + ) + for user in users + ] + + master = MasterKey.objects.select_related('owner').filter(revoked__isnull=True) + nodes.extend( + PGPNodeSchema( + name='Master Key (%s)' % key.owner.get_full_name(), + key=key.pgp_key or None, + group='master', + ) + for key in master + ) + + not_expired = Q(expires__gt=datetime.now(timezone.utc)) | Q(expires__isnull=True) + signatures = PGPSignature.objects.filter(not_expired, revoked__isnull=True) + edges = [ + PGPEdgeSchema(signee=sig.signee, signer=sig.signer) + for sig in signatures + ] + + return PGPKeysSchema(nodes=nodes, edges=edges) diff --git a/api/routes/todolists.py b/api/routes/todolists.py new file mode 100644 index 00000000..7ce414fc --- /dev/null +++ b/api/routes/todolists.py @@ -0,0 +1,19 @@ +import json + +from django.http import HttpResponse +from django.shortcuts import get_object_or_404 +from ninja import Router + +from api.schemas.todolists import TodolistSchema +from todolists.models import Todolist +from todolists.views import TodoListJSONEncoder + +router = Router(tags=["todolists"]) + + +@router.get("/{slug}/", response=TodolistSchema, url_name="todolist-details") +def todolist_details(request, slug: str): + # Same as with /todo/{slug}/json + todolist = get_object_or_404(Todolist, slug=slug) + to_json = json.dumps(todolist, ensure_ascii=False, cls=TodoListJSONEncoder) + return HttpResponse(to_json, content_type='application/json') diff --git a/api/schemas/packages.py b/api/schemas/packages.py new file mode 100644 index 00000000..b291a142 --- /dev/null +++ b/api/schemas/packages.py @@ -0,0 +1,53 @@ +from datetime import datetime + +from ninja import Schema + + +class PackageSchema(Schema): + pkgname: str + pkgbase: str + repo: str + arch: str + pkgver: str + pkgrel: str + epoch: int + pkgdesc: str | None = None + url: str | None = None + filename: str + compressed_size: int + installed_size: int + build_date: datetime | None = None + last_update: datetime + flag_date: datetime | None = None + maintainers: list[str] + packager: str | None = None + groups: list[str] + licenses: list[str] + conflicts: list[str] + provides: list[str] + replaces: list[str] + depends: list[str] + optdepends: list[str] + makedepends: list[str] + checkdepends: list[str] + + +class PackageFilesSchema(Schema): + pkgname: str + repo: str + arch: str + pkg_last_update: datetime + files_last_update: datetime | None = None + files_count: int + dir_count: int + files: list[str] + + +class PackageSearchSchema(Schema): + version: int + limit: int + valid: bool + results: list[PackageSchema] + num_pages: int | None = None + count: int | None = None + page: int | None = None diff --git a/api/schemas/public.py b/api/schemas/public.py new file mode 100644 index 00000000..b3e28a4a --- /dev/null +++ b/api/schemas/public.py @@ -0,0 +1,17 @@ +from ninja import Schema + + +class PGPNodeSchema(Schema): + name: str + key: str | None = None + group: str + + +class PGPEdgeSchema(Schema): + signee: str + signer: str + + +class PGPKeysSchema(Schema): + nodes: list[PGPNodeSchema] + edges: list[PGPEdgeSchema] diff --git a/api/schemas/todolists.py b/api/schemas/todolists.py new file mode 100644 index 00000000..7d39ffbb --- /dev/null +++ b/api/schemas/todolists.py @@ -0,0 +1,19 @@ +from datetime import datetime + +from ninja import Schema + +from api.schemas.packages import PackageSchema + + +class TodolistPackageSchema(PackageSchema): + status_str: str + + +class TodolistSchema(Schema): + id: int + name: str + description: str + created: datetime + last_modified: datetime + packages: list[TodolistPackageSchema] + kind: str diff --git a/api/tests/test_deprecation.py b/api/tests/test_deprecation.py index 39c86238..475525e7 100644 --- a/api/tests/test_deprecation.py +++ b/api/tests/test_deprecation.py @@ -1,4 +1,5 @@ from mirrors.tests.conftest import mirror # noqa: F401 +from todolists.tests.conftest import todolist # noqa: F401 def test_releng_releases_json_deprecation_headers(db, client): @@ -37,3 +38,56 @@ def test_mirror_details_json_deprecation_headers(db, client, mirror): # noqa: F assert response.status_code == 200 assert response['Deprecation'] == 'true' assert response['Link'] == '; rel="successor-version"' + + +def test_master_keys_json_deprecation_headers(db, client): + response = client.get('/master-keys/json/') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == '; rel="successor-version"' + + +def test_package_details_json_deprecation_headers(db, client, package): + response = client.get('/packages/core/x86_64/linux/json/') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == \ + '; rel="successor-version"' + + +def test_package_files_json_deprecation_headers(db, client, package): + response = client.get('/packages/core/x86_64/linux/files/json/') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == \ + '; rel="successor-version"' + + +def test_package_sonames_json_deprecation_headers(db, client, package): + response = client.get('/packages/core/x86_64/linux/sonames/json/') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == \ + '; rel="successor-version"' + + +def test_pkgbase_maintainer_deprecation_headers(db, client): + response = client.get('/packages/pkgbase-maintainer') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == \ + '; rel="successor-version"' + + +def test_package_search_json_deprecation_headers(db, client): + response = client.get('/packages/search/json/') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == '; rel="successor-version"' + + +def test_todolist_json_deprecation_headers(db, client, todolist): # noqa: F811 + response = client.get(f'/todo/{todolist.slug}/json') + assert response.status_code == 200 + assert response['Deprecation'] == 'true' + assert response['Link'] == '; rel="successor-version"' diff --git a/api/tests/test_packages.py b/api/tests/test_packages.py new file mode 100644 index 00000000..503392d9 --- /dev/null +++ b/api/tests/test_packages.py @@ -0,0 +1,72 @@ +import pytest + + +@pytest.fixture +def linux_pkg(package): + # linux, repo=Core, arch=x86_64 + return ('core', 'x86_64', 'linux') + + +def test_details_parity(db, client, linux_pkg): + repo, arch, name = linux_pkg + v1 = client.get(f'/api/v1/packages/{repo}/{arch}/{name}/') + legacy = client.get(f'/packages/{repo}/{arch}/{name}/json/') + assert v1.status_code == 200 + assert legacy.status_code == 200 + assert v1.json() == legacy.json() + + +def test_details_not_found(db, client, repos): + response = client.get('/api/v1/packages/core/x86_64/does-not-exist/') + assert response.status_code == 404 + + +def test_files_parity(db, client, linux_pkg): + repo, arch, name = linux_pkg + v1 = client.get(f'/api/v1/packages/{repo}/{arch}/{name}/files/') + legacy = client.get(f'/packages/{repo}/{arch}/{name}/files/json/') + assert v1.status_code == 200 + assert legacy.status_code == 200 + assert v1.json() == legacy.json() + + +def test_sonames_parity(db, client, linux_pkg): + repo, arch, name = linux_pkg + v1 = client.get(f'/api/v1/packages/{repo}/{arch}/{name}/sonames/') + legacy = client.get(f'/packages/{repo}/{arch}/{name}/sonames/json/') + assert v1.status_code == 200 + assert legacy.status_code == 200 + assert v1.json() == legacy.json() + + +def test_pkgbase_maintainer_parity(db, client, linux_pkg): + v1 = client.get('/api/v1/packages/pkgbase-maintainer') + legacy = client.get('/packages/pkgbase-maintainer') + assert v1.status_code == 200 + assert legacy.status_code == 200 + assert v1.json() == legacy.json() + + +def test_search_no_params_parity(db, client): + v1 = client.get('/api/v1/packages/search/') + legacy = client.get('/packages/search/json/') + assert v1.status_code == 200 + assert legacy.status_code == 200 + body = v1.json() + assert body['valid'] is False + assert body['results'] == [] + assert 'num_pages' not in body # omitted when query is invalid, like legacy + assert body == legacy.json() + + +def test_search_with_query_parity(db, client, linux_pkg): + v1 = client.get('/api/v1/packages/search/?name=linux') + legacy = client.get('/packages/search/json/?name=linux') + assert v1.status_code == 200 + assert legacy.status_code == 200 + assert v1.json() == legacy.json() + + +def test_search_bad_page_returns_400(db, client): + response = client.get('/api/v1/packages/search/?page=notanumber') + assert response.status_code == 400 diff --git a/api/tests/test_public.py b/api/tests/test_public.py new file mode 100644 index 00000000..66b63f63 --- /dev/null +++ b/api/tests/test_public.py @@ -0,0 +1,14 @@ +def test_master_keys_empty(db, client): + response = client.get('/api/v1/master-keys/') + assert response.status_code == 200 + data = response.json() + assert data['nodes'] == [] + assert data['edges'] == [] + + +def test_master_keys_parity_with_legacy(db, client): + v1 = client.get('/api/v1/master-keys/') + legacy = client.get('/master-keys/json/') + assert v1.status_code == 200 + assert legacy.status_code == 200 + assert v1.json() == legacy.json() diff --git a/api/tests/test_todolists.py b/api/tests/test_todolists.py new file mode 100644 index 00000000..e56b55f7 --- /dev/null +++ b/api/tests/test_todolists.py @@ -0,0 +1,29 @@ +from todolists.tests.conftest import ( # noqa: F401 + todolist, + todolistpackage, +) + + +def test_todolist_not_found(db, client): + response = client.get('/api/v1/todo/does-not-exist/') + assert response.status_code == 404 + + +def test_todolist_parity(db, client, todolist): # noqa: F811 + v1 = client.get(f'/api/v1/todo/{todolist.slug}/') + legacy = client.get(f'/todo/{todolist.slug}/json') + assert v1.status_code == 200 + assert legacy.status_code == 200 + assert v1.json() == legacy.json() + + +def test_todolist_with_package_parity(db, client, todolistpackage): # noqa: F811 + slug = todolistpackage.todolist.slug + v1 = client.get(f'/api/v1/todo/{slug}/') + legacy = client.get(f'/todo/{slug}/json') + assert v1.status_code == 200 + assert legacy.status_code == 200 + body = v1.json() + assert len(body['packages']) == 1 + assert 'status_str' in body['packages'][0] + assert body == legacy.json() diff --git a/packages/views/__init__.py b/packages/views/__init__.py index 361af96a..cdae41ae 100644 --- a/packages/views/__init__.py +++ b/packages/views/__init__.py @@ -1,10 +1,8 @@ import hashlib import json -from collections import defaultdict from django.contrib import messages from django.contrib.auth.decorators import permission_required -from django.contrib.auth.models import User from django.contrib.sites.models import Site from django.core.cache import cache from django.db.models import Q @@ -13,6 +11,7 @@ from django.views.decorators.cache import cache_control from django.views.decorators.http import require_POST, require_safe +from api.deprecation import deprecated_json_endpoint from main.models import Package, Soname from packages.models import PackageRelation @@ -160,33 +159,12 @@ def sonames(request): @cache_control(public=True, max_age=300) +@deprecated_json_endpoint('/api/v1/packages/pkgbase-maintainer') def pkgbase_mapping(request): - if request.method == 'GET': - pkgbases = Package.objects.all().values('pkgbase') - rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER, - pkgbase__in=pkgbases).values_list( - 'pkgbase', 'user_id').order_by().distinct() - - # get all the user objects we will need - user_ids = {rel[1] for rel in rels} - users = User.objects.in_bulk(user_ids) - - # now build a pkgbase -> [maintainers...] map - maintainers = defaultdict(list) - for rel in rels: - user = users[rel[1]] - maintainers[rel[0]].append(user.username) - - pkgbase_maintainer_mapping = {} - for pkgbase in pkgbases: - pkgbase = pkgbase['pkgbase'] - if pkgbase in pkgbase_maintainer_mapping: - continue - pkgbase_maintainer_mapping[pkgbase] = maintainers[pkgbase] - - to_json = json.dumps(pkgbase_maintainer_mapping, ensure_ascii=False) - return HttpResponse(to_json, content_type='application/json') - else: + if request.method != 'GET': return HttpResponseBadRequest('only GET is allowed') + from api.routes.packages import pkgbase_maintainer + to_json = json.dumps(pkgbase_maintainer(request), ensure_ascii=False) + return HttpResponse(to_json, content_type='application/json') # vim: set ts=4 sw=4 et: diff --git a/packages/views/display.py b/packages/views/display.py index 358eb893..cbebf5e4 100644 --- a/packages/views/display.py +++ b/packages/views/display.py @@ -6,6 +6,7 @@ from django.shortcuts import get_object_or_404, redirect, render from django.utils.timezone import now +from api.deprecation import deprecated_json_endpoint from main.models import Arch, Package, PackageFile, RebuilderdStatus, Repo, Soname from main.utils import empty_response from mirrors.utils import get_mirror_url_for_download @@ -211,13 +212,15 @@ def sonames(request, name, repo, arch): return render(request, template, context) +@deprecated_json_endpoint('/api/v1/packages/{repo}/{arch}/{name}/sonames/') def sonames_json(request, name, repo, arch): - pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) - sonames = list(Soname.objects.filter(pkg=pkg).values_list('name', flat=True)) - to_json = json.dumps(sonames, ensure_ascii=False) + from api.routes.packages import sonames as sonames_api + data = sonames_api(request, repo=repo, arch=arch, name=name) + to_json = json.dumps(data, ensure_ascii=False) return HttpResponse(to_json, content_type='application/json') +@deprecated_json_endpoint('/api/v1/packages/{repo}/{arch}/{name}/') def details_json(request, name, repo, arch): pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) @@ -225,6 +228,7 @@ def details_json(request, name, repo, arch): return HttpResponse(to_json, content_type='application/json') +@deprecated_json_endpoint('/api/v1/packages/{repo}/{arch}/{name}/files/') def files_json(request, name, repo, arch): pkg = get_object_or_404(Package.objects.normal(), pkgname=name, repo__name__iexact=repo, arch__name=arch) diff --git a/packages/views/search.py b/packages/views/search.py index b9d383d2..0752b7e3 100644 --- a/packages/views/search.py +++ b/packages/views/search.py @@ -4,17 +4,17 @@ from django import forms from django.contrib.auth.models import User -from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator from django.db.models import Q -from django.http import HttpResponse, HttpResponseBadRequest +from django.http import HttpResponse from django.views.generic import ListView +from api.deprecation import deprecated_json_endpoint from devel.models import UserProfile from main.models import Arch, Package, Repo from main.utils import empty_response, make_choice from ..models import PackageRelation -from ..utils import PackageJSONEncoder, attach_maintainers +from ..utils import PackageJSONEncoder class GroupSearchForm(forms.Form): @@ -208,51 +208,10 @@ def group_search_json(request) -> HttpResponse: return HttpResponse(to_json, content_type='application/json') +@deprecated_json_endpoint('/api/v1/packages/search/') def search_json(request): - limit = 250 - - container = { - 'version': 2, - 'limit': limit, - 'valid': False, - 'results': [], - } - - if request.GET: - form = PackageSearchForm(data=request.GET, - show_staging=request.user.is_authenticated) - if form.is_valid(): - form_limit = form.cleaned_data.get('limit', limit) - limit = min(limit, form_limit) if form_limit else limit - container['limit'] = limit - - packages = Package.objects.select_related('arch', 'repo', 'packager') - if not request.user.is_authenticated: - packages = packages.filter(repo__staging=False) - packages = parse_form(form, packages) - - paginator = Paginator(packages, limit) - container['num_pages'] = paginator.num_pages - container['count'] = paginator.count - - page = form.cleaned_data.get('page') - try: - page = int(page) if page else 1 - except ValueError: - return HttpResponseBadRequest('page parameter is not a number') - container['page'] = page - try: - packages = paginator.page(page) - except PageNotAnInteger: - packages = paginator.page(1) - except EmptyPage: - packages = paginator.page(paginator.num_pages) - - attach_maintainers(packages) - container['results'] = packages - container['valid'] = True - - to_json = json.dumps(container, ensure_ascii=False, cls=PackageJSONEncoder) - return HttpResponse(to_json, content_type='application/json') + # Delegate to the API v1 implementation. + from api.routes.packages import search + return search(request) # vim: set ts=4 sw=4 et: diff --git a/public/views.py b/public/views.py index 7c5fcc41..dba4b287 100644 --- a/public/views.py +++ b/public/views.py @@ -9,6 +9,7 @@ from django.shortcuts import get_object_or_404, render from django.views.decorators.cache import cache_control, cache_page +from api.deprecation import deprecated_json_endpoint from devel.models import DeveloperKey, MasterKey, PGPSignature, StaffGroup, UserProfile from main.models import Arch, Donor, Repo from mirrors.models import MirrorUrl @@ -137,6 +138,7 @@ def keys(request): @cache_page(1789) +@deprecated_json_endpoint('/api/v1/master-keys/') def keys_json(request): profile_ids = UserProfile.allowed_repos.through.objects.values('userprofile_id') users = User.objects.filter( diff --git a/todolists/views.py b/todolists/views.py index f0401354..137c5024 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -11,6 +11,7 @@ from django.views.decorators.cache import never_cache from django.views.generic import DeleteView, ListView +from api.deprecation import deprecated_json_endpoint from main.models import Package from main.utils import find_unique_slug from packages.utils import PackageJSONEncoder, attach_maintainers @@ -244,6 +245,7 @@ def default(self, obj): return super().default(obj) +@deprecated_json_endpoint('/api/v1/todo/{slug}/') def view_json(request, slug): todolist = get_object_or_404(Todolist, slug=slug) to_json = json.dumps(todolist, ensure_ascii=False, cls=TodoListJSONEncoder)