Support fetchFromGitLab with forceFetchGit

[pbsds]

I've been trying to debug why nix-update does not work for spade in nixpkgs. The issue is that its use of fetchFromGitLab relies on fetchSubmodules, which in turn makes fetchFromGitLab use fetchgit with a .git url.

I've been playing around trying to solve this problem, and I now see three different solutions:

(1) implement a gitlab version fetcher for .git urls
(2) passthru the gitlab api url in fetchFromGitLab which is used instead of .url
(3) implement a generic version fetcher for .git urls (git ls-remote --tags --sort)

Possible nix-update patch for (1)

diff --git a/nix_update/version/__init__.py b/nix_update/version/__init__.py
index 11dcb71..d567b12 100644
--- a/nix_update/version/__init__.py
+++ b/nix_update/version/__init__.py
@@ -9,7 +9,7 @@ from .bitbucket import fetch_bitbucket_snapshots, fetch_bitbucket_versions
 from .crate import fetch_crate_versions
 from .gitea import fetch_gitea_snapshots, fetch_gitea_versions
 from .github import fetch_github_snapshots, fetch_github_versions
-from .gitlab import fetch_gitlab_snapshots, fetch_gitlab_versions
+from .gitlab import fetch_gitlab_snapshots, fetch_gitlab_versions, fetch_gitlab_dotgit_versions
 from .npm import fetch_npm_versions
 from .pypi import fetch_pypi_versions
 from .rubygems import fetch_rubygem_versions
@@ -42,6 +42,7 @@ fetchers: list[Callable[[ParseResult], list[Version]]] = [
     fetch_sourcehut_versions,
     fetch_bitbucket_versions,
     # all entries below perform requests to check if the target url is of that type
+    fetch_gitlab_dotgit_versions,
     fetch_gitea_versions,
 ]
 
diff --git a/nix_update/version/gitlab.py b/nix_update/version/gitlab.py
index 2bc0703..b97d4b3 100644
--- a/nix_update/version/gitlab.py
+++ b/nix_update/version/gitlab.py
@@ -8,9 +8,27 @@ from ..errors import VersionError
 from ..utils import info
 from .version import Version
 
+# found when fetchFromGitLab internally uses fetchzip
 GITLAB_API = re.compile(
     r"http(s)?://(?P<domain>[^/]+)/api/v4/projects/(?P<project_id>[^/]*)/repository/archive.tar.gz\?sha=(?P<version>.+)"
 )
+# found when fetchFromGitLab internally uses fetchgit
+GITLAB_GIT = re.compile(
+    r"http(s)?://(?P<domain>[^/]+)/((?P<group>[^/]*)/)?(?P<owner>[^/]*)/(?P<repo>[^/]*).git"
+)
+
+KNOWN_GITLAB_HOSTS = [
+    "code.videolan.org"
+    "framagit.org"
+    "gitlab.com"
+    "gitlab.freedesktop.org"
+    "gitlab.gnome.org"
+    "gitlab.inria.fr"
+    "gitlab.linphone.org"
+    "gitlab.torproject.org"
+    "invent.kde.org"
+    "salsa.debian.org"
+]
 
 
 def fetch_gitlab_versions(url: ParseResult) -> list[Version]:
@@ -19,6 +37,34 @@ def fetch_gitlab_versions(url: ParseResult) -> list[Version]:
         return []
     domain = match.group("domain")
     project_id = match.group("project_id")
+    return _fetch_gitlab_versions(domain, project_id)
+
+def fetch_gitlab_dotgit_versions(url: ParseResult) -> list[Version]:
+    match = GITLAB_GIT.match(url.geturl())
+    if not match:
+        return []
+
+    domain = match.group("domain")
+    group = match.group("group")
+    owner = match.group("owner")
+    repo = match.group("repo")
+    if group is None:
+        project_id = quote_plus(f"{owner}/{repo}")
+    else:
+        project_id = quote_plus(f"{group}/{owner}/{repo}")
+
+    if not domain in KNOWN_GITLAB_HOSTS:
+        endpoint = f"https://{domain}/api/v4/projects/{project_id}"
+        try:
+            resp = urllib.request.urlopen(endpoint)
+        except URLError:
+            return []
+        if resp.status != 200:
+            return []
+
+    return _fetch_gitlab_versions(domain, project_id)
+
+def _fetch_gitlab_versions(domain: str, project_id: str) -> list[Version]:
     gitlab_url = f"https://{domain}/api/v4/projects/{project_id}/repository/tags"
     info(f"fetch {gitlab_url}")
     resp = urllib.request.urlopen(gitlab_url)
diff --git a/tests/test_gitlab.py b/tests/test_gitlab.py
index 6c5733c..fa1a2d2 100644
--- a/tests/test_gitlab.py
+++ b/tests/test_gitlab.py
@@ -1,13 +1,15 @@
 import subprocess
 
 import conftest
+import pytest
 
 from nix_update import main
 
 
-def test_main(helpers: conftest.Helpers) -> None:
+@pytest.mark.parametrize("attrpath", ["gitlab", "gitlab-git"])
+def test_main(helpers: conftest.Helpers, attrpath: str) -> None:
     with helpers.testpkgs(init_git=True) as path:
-        main(["--file", str(path), "--commit", "gitlab"])
+        main(["--file", str(path), "--commit", attrpath])
         version = subprocess.run(
             [
                 "nix",
@@ -17,7 +19,7 @@ def test_main(helpers: conftest.Helpers) -> None:
                 "nix-command",
                 "-f",
                 path,
-                "gitlab.version",
+                f"{attrpath}.version",
             ],
             check=True,
             text=True,
@@ -32,7 +34,8 @@ def test_main(helpers: conftest.Helpers) -> None:
         ).stdout.strip()
         print(commit)
         assert version in commit
-        assert "gitlab" in commit
-        assert (
-            "https://gitlab.gnome.org/world/phosh/phosh/-/compare/v0.20.0...v" in commit
-        )
+        assert attrpath in commit
+        if attrpath == "gitlab":
+            assert (
+                "https://gitlab.gnome.org/world/phosh/phosh/-/compare/v0.20.0...v" in commit
+            )
diff --git a/tests/testpkgs/default.nix b/tests/testpkgs/default.nix
index 006bfc0..8d47553 100644
--- a/tests/testpkgs/default.nix
+++ b/tests/testpkgs/default.nix
@@ -13,6 +13,7 @@
   github = pkgs.callPackage ./github.nix { };
   github-no-release = pkgs.callPackage ./github-no-release.nix { };
   gitlab = pkgs.callPackage ./gitlab.nix { };
+  gitlab-git = pkgs.callPackage ./gitlab-git.nix { };
   pypi = pkgs.python3.pkgs.callPackage ./pypi.nix { };
   sourcehut = pkgs.python3.pkgs.callPackage ./sourcehut.nix { };
   savanna = pkgs.python3.pkgs.callPackage ./savanna.nix { };
diff --git a/tests/testpkgs/gitlab-git.nix b/tests/testpkgs/gitlab-git.nix
new file mode 100644
index 0000000..9df9e98
--- /dev/null
+++ b/tests/testpkgs/gitlab-git.nix
@@ -0,0 +1,16 @@
+{ stdenv, fetchFromGitLab }:
+
+stdenv.mkDerivation rec {
+  pname = "phosh";
+  version = "0.20.0";
+
+  src = fetchFromGitLab {
+    domain = "gitlab.gnome.org";
+    group = "world";
+    owner = "phosh";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
+    forceFetchGit = true;
+  };
+}

Possible nixpkgs patch for (2)

diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix
index 1b000fb49a99..00d59b8a61bd 100644
--- a/pkgs/build-support/fetchgit/default.nix
+++ b/pkgs/build-support/fetchgit/default.nix
@@ -28,6 +28,7 @@ lib.makeOverridable (lib.fetchers.withNormalizedHash { } (
 , # Impure env vars (https://nixos.org/nix/manual/#sec-advanced-attributes)
   # needed for netrcPhase
   netrcImpureEnvVars ? []
+, passthru ? {}
 , meta ? {}
 , allowedRequisites ? null
 }:
@@ -96,6 +97,6 @@ stdenvNoCC.mkDerivation {
 
   passthru = {
     gitRepoUrl = url;
-  };
+  } // passthru;
 }
 ))
diff --git a/pkgs/build-support/fetchgitlab/default.nix b/pkgs/build-support/fetchgitlab/default.nix
index 749883f2365e..bad0fabc7e69 100644
--- a/pkgs/build-support/fetchgitlab/default.nix
+++ b/pkgs/build-support/fetchgitlab/default.nix
@@ -6,6 +6,7 @@ lib.makeOverridable (
 , fetchSubmodules ? false, leaveDotGit ? false
 , deepClone ? false, forceFetchGit ? false
 , sparseCheckout ? []
+, passthru ? {}
 , ... # For hash agility
 } @ args:
 
@@ -13,7 +14,9 @@ let
   slug = lib.concatStringsSep "/" ((lib.optional (group != null) group) ++ [ owner repo ]);
   escapedSlug = lib.replaceStrings [ "." "/" ] [ "%2E" "%2F" ] slug;
   escapedRev = lib.replaceStrings [ "+" "%" "/" ] [ "%2B" "%25" "%2F" ] rev;
-  passthruAttrs = removeAttrs args [ "protocol" "domain" "owner" "group" "repo" "rev" "fetchSubmodules" "forceFetchGit" "leaveDotGit" "deepClone" ];
+  passthruAttrs = removeAttrs args [ "protocol" "domain" "owner" "group" "repo" "rev" "fetchSubmodules" "forceFetchGit" "leaveDotGit" "deepClone" "passthru" ];
+
+  gitlabApiUrl = "${protocol}://${domain}/api/v4/projects/${escapedSlug}";
 
   useFetchGit = fetchSubmodules || leaveDotGit || deepClone || forceFetchGit || (sparseCheckout != []);
   fetcher = if useFetchGit then fetchgit else fetchzip;
@@ -23,12 +26,16 @@ let
   fetcherArgs = (if useFetchGit then {
     inherit rev deepClone fetchSubmodules sparseCheckout leaveDotGit;
     url = gitRepoUrl;
+
+    passthru = {
+      inherit gitlabApiUrl;
+    } // passthru;
   } else {
-    url = "${protocol}://${domain}/api/v4/projects/${escapedSlug}/repository/archive.tar.gz?sha=${escapedRev}";
+    url = "${gitlabApiUrl}/repository/archive.tar.gz?sha=${escapedRev}";
 
     passthru = {
-      inherit gitRepoUrl;
-    };
+      inherit gitRepoUrl gitlabApiUrl;
+    } // passthru;
   }) // passthruAttrs // { inherit name; };
 in

Any preferences?

[Mic92]

I think the nixpkgs patch looks more elegant and also helps other nix tooling beyond nix-update.