Add project management and GitHub source support

scripts/project_sources.py

@@ -6,7 +6,6 @@ from __future__ import annotations
 import argparse
 import configparser
 import hashlib
-import ipaddress
 import io
 import json
 import os
@@ -19,11 +18,18 @@ import tempfile
 from datetime import date, datetime, timezone
 from typing import Any
 from urllib.error import HTTPError, URLError
-from urllib.parse import quote, urlparse
+from urllib.parse import quote
 from urllib.request import Request, urlopen
 
 from lektor.metaformat import tokenize
 
+from project_providers import (
+    ProjectProviderError,
+    repository_provider,
+    repository_readme_url,
+    validate_public_https_url,
+)
+
 
 SCHEMA_VERSION = "1"
 MAX_FILES = 100
@@ -108,19 +114,10 @@ def run_git(git_dir: Path, *args: str, text: bool = True) -> str | bytes:
 
 
 def validate_url(value: str, field: str) -> str:
-    parsed = urlparse(value)
-    if parsed.scheme != "https" or not parsed.hostname or parsed.username or parsed.password:
-        raise ProjectSourceError(f"{field} must be a credential-free HTTPS URL")
-    if parsed.hostname.lower() == "localhost":
-        raise ProjectSourceError(f"{field} must not target localhost")
     try:
-        address = ipaddress.ip_address(parsed.hostname)
-    except ValueError:
-        pass
-    else:
-        if not address.is_global:
-            raise ProjectSourceError(f"{field} must not target a private or local address")
-    return value.rstrip("/")
+        return validate_public_https_url(value, field)
+    except ProjectProviderError as exc:
+        raise ProjectSourceError(str(exc)) from exc
 
 
 def load_registry(site_root: Path) -> list[dict[str, str | int]]:
@@ -454,6 +451,14 @@ def fetch_json(
     except HTTPError as exc:
         if allow_not_found and exc.code == 404:
             return None
+        if (
+            require_public
+            and exc.code == 403
+            and exc.headers.get("X-RateLimit-Remaining") == "0"
+        ):
+            raise ProjectSourceError(
+                "anonymous provider API rate limit was reached"
+            ) from exc
         if require_public and exc.code in {401, 403, 404}:
             raise RepositoryNotPublicError(
                 "repository API is not anonymously accessible"
@@ -490,11 +495,14 @@ def detect_license(mirror: Path, commit: str) -> str:
 def collect_metadata(
     source: dict[str, str], mirror: Path, commit: str, previous: dict[str, Any] | None
 ) -> tuple[dict[str, Any], bool]:
+    provider = repository_provider(str(source["web_url"]))
     metadata: dict[str, Any] = commit_metadata(mirror, commit)
     metadata.update(
         {
             "repository_url": source["web_url"],
-            "readme_url": f"{source['web_url']}/src/branch/{quote(source['branch'], safe='')}/README.md",
+            "readme_url": repository_readme_url(
+                str(source["web_url"]), str(source["branch"])
+            ),
             "commit_url": f"{source['web_url']}/commit/{commit}",
             "default_branch": source["branch"],
             "license": detect_license(mirror, commit),
@@ -516,7 +524,8 @@ def collect_metadata(
         if repository.get("default_branch") != source["branch"]:
             raise ProjectSourceError(f"{source['project_id']}: API default branch differs from registry")
         metadata["open_issues"] = int(repository.get("open_issues_count") or 0)
-        metadata["stars"] = int(repository.get("stars_count") or 0)
+        stars_field = "stargazers_count" if provider == "github" else "stars_count"
+        metadata["stars"] = int(repository.get(stars_field) or 0)
         metadata["forks"] = int(repository.get("forks_count") or 0)
         languages = fetch_json(f"{source['api_url']}/languages")
         if isinstance(languages, dict):