sapling: simplify dependency generation script

This commit is contained in:
Daniel Nagy 2022-11-24 13:00:00 +01:00 committed by Austin Seipp
parent 1338b8e4d5
commit aab14dc769
6 changed files with 123 additions and 197 deletions

1
.gitattributes vendored

@ -1,4 +1,5 @@
**/deps.nix linguist-generated
**/deps.json linguist-generated
**/node-packages.nix linguist-generated
pkgs/applications/editors/emacs-modes/*-generated.nix linguist-generated

@ -1,9 +1,7 @@
{ lib, stdenv, python3Packages, fetchFromGitHub, fetchurl, sd, curl, pkg-config, openssl, rustPlatform, fetchYarnDeps, yarn, nodejs, fixup_yarn_lock }:
let
deps = import ./deps.nix { inherit lib fetchurl; };
version = deps.version;
versionHash = deps.versionHash;
inherit (lib.importJSON ./deps.json) links version versionHash;
src = fetchFromGitHub {
owner = "facebook";
@ -83,9 +81,7 @@ let
# with filesystem paths for the curl calls.
postUnpack = ''
mkdir $sourceRoot/hack_pydeps
( cd $sourceRoot/hack_pydeps
${deps.links}
)
${lib.concatStrings (map (li: "ln -s ${fetchurl li} $sourceRoot/hack_pydeps/${baseNameOf li.url}\n") links)}
sed -i "s|https://files.pythonhosted.org/packages/[[:alnum:]]*/[[:alnum:]]*/[[:alnum:]]*/|file://$NIX_BUILD_TOP/$sourceRoot/hack_pydeps/|g" $sourceRoot/setup.py
'';

@ -0,0 +1,78 @@
{
"links": [
{
"sha256": "0dgg5x4nvdpfiz552diy11xg72y14s38hjz9qxygafnfgybg6hab",
"url": "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl"
},
{
"sha256": "0dq9f7irmml1nm9a2rx8dd6m2dqmzjj1x40mk0rg619wrdfsaj0b",
"url": "https://files.pythonhosted.org/packages/39/7b/88dbb785881c28a102619d46423cb853b46dbccc70d3ac362d99773a78ce/pexpect-4.8.0-py2.py3-none-any.whl"
},
{
"sha256": "0r4xy2sqwyhwlwj81zvhzbnx7a7r4xdz9xyzzkjczlx7gk1cig1d",
"url": "https://files.pythonhosted.org/packages/23/6a/210816c943c9aeeb29e4e18a298f14bf0e118fe222a23e13bfcc2d41b0a4/ipython-7.16.1-py3-none-any.whl"
},
{
"sha256": "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw",
"url": "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip"
},
{
"sha256": "16sgpg57kxx5jh467d9qwc2hwshfvdbl0xkafdp3qspvbfp46qc0",
"url": "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl"
},
{
"sha256": "1qn3bnyd7gdwkyk8nvlhiy3c6zbwjd49fjxj0gp8xxi9faiysiwz",
"url": "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl"
},
{
"sha256": "0mfj5d1bgpy1clfgwrkmjqm2pv70pm01jp4iyyhphc96kyifdg7v",
"url": "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl"
},
{
"sha256": "0yxz45fzjsq6zh5f9cjl0gf4vfg1l7rd79zyb3ih544layjg3ff8",
"url": "https://files.pythonhosted.org/packages/4e/78/56aa1b5f4d8ac548755ae767d84f0be54fdd9d404197a3d9e4659d272348/setuptools-57.0.0-py3-none-any.whl"
},
{
"sha256": "1177pfa343r378020a85l3b16ak479qgyvh8k5719fgbkhm81d5y",
"url": "https://files.pythonhosted.org/packages/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl"
},
{
"sha256": "1r55ffffaq4q3dpvha7iipgxlqwvjg5cklf9izr42xj5rr226r26",
"url": "https://files.pythonhosted.org/packages/87/61/2dfea88583d5454e3a64f9308a686071d58d59a55db638268a6413e1eb6d/prompt_toolkit-2.0.10-py3-none-any.whl"
},
{
"sha256": "08v36wa0kckc892bk4nypl6sszbysarm8jhslviz1agp2sf1jp3f",
"url": "https://files.pythonhosted.org/packages/6a/36/b1b9bfdf28690ae01d9ca0aa5b0d07cb4448ac65fb91dc7e2d094e3d992f/decorator-5.0.9-py3-none-any.whl"
},
{
"sha256": "0mnzcb714ynl1qlv9dwnh50rv75mmj18ywaxbl8xzm3l9m0syjcn",
"url": "https://files.pythonhosted.org/packages/9a/41/220f49aaea88bc6fa6cba8d05ecf24676326156c23b991e80b3f2fc24c77/pickleshare-0.7.5-py2.py3-none-any.whl"
},
{
"sha256": "13iv73575lilrm01ffhr8y8sxn8kxhvfqsgwckbr919725280vnn",
"url": "https://files.pythonhosted.org/packages/a6/c9/be11fce9810793676017f79ffab3c6cb18575844a6c7b8d4ed92f95de604/Pygments-2.9.0-py3-none-any.whl"
},
{
"sha256": "0i7ycyjad9kq6lgq5ih7j8xsm639z24250s6d17pp781v6hwdd3h",
"url": "https://files.pythonhosted.org/packages/ca/ab/872a23e29cec3cf2594af7e857f18b687ad21039c1f9b922fac5b9b142d5/traitlets-4.3.3-py2.py3-none-any.whl"
},
{
"sha256": "0m02dsi8lvrjf4bi20ab6lm7rr6krz7pg6lzk3xjs2l9hqfjzfwa",
"url": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl"
},
{
"sha256": "1f7sc4ydjj33gadcgfz8fcx02d1wm2frlqwzdik1krlr6wikgpbj",
"url": "https://files.pythonhosted.org/packages/fa/bc/9bd3b5c2b4774d5f33b2d544f1460be9df7df2fe42f352135381c347c69a/ipython_genutils-0.2.0-py2.py3-none-any.whl"
},
{
"sha256": "1ibql99agjf2gj7y0svzd5m0h81hailf4p3sj3yl9i1i8ykdj6wm",
"url": "https://files.pythonhosted.org/packages/fc/56/9f67dcd4a4b9960373173a31be1b8c47fe351a1c9385677a7bdd82810e57/ipdb-0.13.9.tar.gz"
},
{
"sha256": "04m31z011arz2b70rwwkhvzkb9d4yxcfbxpw27d6fa3n79a7sdxg",
"url": "https://files.pythonhosted.org/packages/bc/fa/8604d92ef753e0036d807f1b3179813ab2fa283e3b19c926e11673c8205b/Cython-0.29.26.tar.gz"
}
],
"version": "0.1.20221118-210929-cfbb68aa",
"versionHash": "5535144625961033752"
}

@ -1,104 +0,0 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# Generated by `gen-deps`
{ lib, fetchurl }:
let
ptyprocess = fetchurl {
url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl";
sha256 = "0dgg5x4nvdpfiz552diy11xg72y14s38hjz9qxygafnfgybg6hab";
};
pexpect = fetchurl {
url = "https://files.pythonhosted.org/packages/39/7b/88dbb785881c28a102619d46423cb853b46dbccc70d3ac362d99773a78ce/pexpect-4.8.0-py2.py3-none-any.whl";
sha256 = "0dq9f7irmml1nm9a2rx8dd6m2dqmzjj1x40mk0rg619wrdfsaj0b";
};
ipython = fetchurl {
url = "https://files.pythonhosted.org/packages/23/6a/210816c943c9aeeb29e4e18a298f14bf0e118fe222a23e13bfcc2d41b0a4/ipython-7.16.1-py3-none-any.whl";
sha256 = "0r4xy2sqwyhwlwj81zvhzbnx7a7r4xdz9xyzzkjczlx7gk1cig1d";
};
simplegeneric = fetchurl {
url = "https://files.pythonhosted.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
sha256 = "0wwi1c6md4vkbcsfsf8dklf3vr4mcdj4mpxkanwgb6jb1432x5yw";
};
toml = fetchurl {
url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl";
sha256 = "16sgpg57kxx5jh467d9qwc2hwshfvdbl0xkafdp3qspvbfp46qc0";
};
colorama = fetchurl {
url = "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl";
sha256 = "1qn3bnyd7gdwkyk8nvlhiy3c6zbwjd49fjxj0gp8xxi9faiysiwz";
};
backcall = fetchurl {
url = "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl";
sha256 = "0mfj5d1bgpy1clfgwrkmjqm2pv70pm01jp4iyyhphc96kyifdg7v";
};
setuptools = fetchurl {
url = "https://files.pythonhosted.org/packages/4e/78/56aa1b5f4d8ac548755ae767d84f0be54fdd9d404197a3d9e4659d272348/setuptools-57.0.0-py3-none-any.whl";
sha256 = "0yxz45fzjsq6zh5f9cjl0gf4vfg1l7rd79zyb3ih544layjg3ff8";
};
wcwidth = fetchurl {
url = "https://files.pythonhosted.org/packages/59/7c/e39aca596badaf1b78e8f547c807b04dae603a433d3e7a7e04d67f2ef3e5/wcwidth-0.2.5-py2.py3-none-any.whl";
sha256 = "1177pfa343r378020a85l3b16ak479qgyvh8k5719fgbkhm81d5y";
};
prompt_toolkit = fetchurl {
url = "https://files.pythonhosted.org/packages/87/61/2dfea88583d5454e3a64f9308a686071d58d59a55db638268a6413e1eb6d/prompt_toolkit-2.0.10-py3-none-any.whl";
sha256 = "1r55ffffaq4q3dpvha7iipgxlqwvjg5cklf9izr42xj5rr226r26";
};
decorator = fetchurl {
url = "https://files.pythonhosted.org/packages/6a/36/b1b9bfdf28690ae01d9ca0aa5b0d07cb4448ac65fb91dc7e2d094e3d992f/decorator-5.0.9-py3-none-any.whl";
sha256 = "08v36wa0kckc892bk4nypl6sszbysarm8jhslviz1agp2sf1jp3f";
};
pickleshare = fetchurl {
url = "https://files.pythonhosted.org/packages/9a/41/220f49aaea88bc6fa6cba8d05ecf24676326156c23b991e80b3f2fc24c77/pickleshare-0.7.5-py2.py3-none-any.whl";
sha256 = "0mnzcb714ynl1qlv9dwnh50rv75mmj18ywaxbl8xzm3l9m0syjcn";
};
Pygments = fetchurl {
url = "https://files.pythonhosted.org/packages/a6/c9/be11fce9810793676017f79ffab3c6cb18575844a6c7b8d4ed92f95de604/Pygments-2.9.0-py3-none-any.whl";
sha256 = "13iv73575lilrm01ffhr8y8sxn8kxhvfqsgwckbr919725280vnn";
};
traitlets = fetchurl {
url = "https://files.pythonhosted.org/packages/ca/ab/872a23e29cec3cf2594af7e857f18b687ad21039c1f9b922fac5b9b142d5/traitlets-4.3.3-py2.py3-none-any.whl";
sha256 = "0i7ycyjad9kq6lgq5ih7j8xsm639z24250s6d17pp781v6hwdd3h";
};
six = fetchurl {
url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl";
sha256 = "0m02dsi8lvrjf4bi20ab6lm7rr6krz7pg6lzk3xjs2l9hqfjzfwa";
};
ipython_genutils = fetchurl {
url = "https://files.pythonhosted.org/packages/fa/bc/9bd3b5c2b4774d5f33b2d544f1460be9df7df2fe42f352135381c347c69a/ipython_genutils-0.2.0-py2.py3-none-any.whl";
sha256 = "1f7sc4ydjj33gadcgfz8fcx02d1wm2frlqwzdik1krlr6wikgpbj";
};
ipdb = fetchurl {
url = "https://files.pythonhosted.org/packages/fc/56/9f67dcd4a4b9960373173a31be1b8c47fe351a1c9385677a7bdd82810e57/ipdb-0.13.9.tar.gz";
sha256 = "1ibql99agjf2gj7y0svzd5m0h81hailf4p3sj3yl9i1i8ykdj6wm";
};
Cython = fetchurl {
url = "https://files.pythonhosted.org/packages/bc/fa/8604d92ef753e0036d807f1b3179813ab2fa283e3b19c926e11673c8205b/Cython-0.29.26.tar.gz";
sha256 = "04m31z011arz2b70rwwkhvzkb9d4yxcfbxpw27d6fa3n79a7sdxg";
};
in
{
links = ''
ln -s ${ptyprocess} ptyprocess-0.7.0-py2.py3-none-any.whl
ln -s ${pexpect} pexpect-4.8.0-py2.py3-none-any.whl
ln -s ${ipython} ipython-7.16.1-py3-none-any.whl
ln -s ${simplegeneric} simplegeneric-0.8.1.zip
ln -s ${toml} toml-0.10.2-py2.py3-none-any.whl
ln -s ${colorama} colorama-0.4.4-py2.py3-none-any.whl
ln -s ${backcall} backcall-0.2.0-py2.py3-none-any.whl
ln -s ${setuptools} setuptools-57.0.0-py3-none-any.whl
ln -s ${wcwidth} wcwidth-0.2.5-py2.py3-none-any.whl
ln -s ${prompt_toolkit} prompt_toolkit-2.0.10-py3-none-any.whl
ln -s ${decorator} decorator-5.0.9-py3-none-any.whl
ln -s ${pickleshare} pickleshare-0.7.5-py2.py3-none-any.whl
ln -s ${Pygments} Pygments-2.9.0-py3-none-any.whl
ln -s ${traitlets} traitlets-4.3.3-py2.py3-none-any.whl
ln -s ${six} six-1.16.0-py2.py3-none-any.whl
ln -s ${ipython_genutils} ipython_genutils-0.2.0-py2.py3-none-any.whl
ln -s ${ipdb} ipdb-0.13.9.tar.gz
ln -s ${Cython} Cython-0.29.26.tar.gz
'';
version = "0.1.20221118-210929-cfbb68aa";
versionHash = "5535144625961033752";
}

@ -1,87 +0,0 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests ])" nixpkgs-fmt
# vim: set syntax=python:
import hashlib
import json
import re
import struct
import subprocess
import requests
gen_file = "deps.nix"
repo = "facebook/sapling"
source_file = "eden/scm/setup.py"
# Fetch the latest stable release metadata from GitHub
latest = requests.get(f"https://api.github.com/repos/{repo}/releases/latest").text
latest = json.loads(latest)
# Find latest's git tag which corresponds to the Sapling version. Also needed
# is a hash of the version, so calculate that here. Taken from Sapling source
# `$root/eden/scm/setup_with_version.py`.
version = str(latest["tag_name"])
version_hash = str(
struct.unpack(">Q", hashlib.sha1(version.encode("ascii")).digest()[:8])[0]
)
# Fetch the `setup.py` source and look for instances of assets being downloaded
# from files.pythonhosted.org.
source = requests.get(
f"https://raw.githubusercontent.com/{repo}/{version}/{source_file}"
).text
matches = re.findall(
r'(https://files\.pythonhosted\.org/packages/.*?/.*?/.*?/)(.*?)(-.*?)"', source
)
# For each matched asset URL, prefetch said asset and build the corresponding
# Nix code needed to call `fetchurl` and perform symlinks in the actual build.
assets = ""
links = ""
for m in matches:
sha256 = (
subprocess.Popen(
["nix-prefetch-url", "--type", "sha256", f"{m[0]}{m[1]}{m[2]}"],
text=True,
stdout=subprocess.PIPE,
)
.stdout.read()
.strip()
)
assets += """ {} = fetchurl {{
url = "{}{}{}";
sha256 = "{}";
}};\n""".format(
m[1], m[0], m[1], m[2], sha256
)
links += " ln -s ${{{}}} {}{}\n".format(m[1], m[1], m[2])
assets = assets.strip()
links = links.strip()
# Render an autogenerated Nix file with the fetched dependency and version
# information for import in the main derivation.
out = f"""
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# Generated by `gen-deps`
{{ lib, fetchurl }}:
let
{assets}
in
{{
links = ''
{links}
'';
version = "{version}";
versionHash = "{version_hash}";
}}
"""
with open(gen_file, "w+") as f:
f.write(out.strip())
# Formamt the generated Nix file, which also serves as a syntax sanity check.
subprocess.run(["nixpkgs-fmt", gen_file], check=True)

@ -0,0 +1,42 @@
#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests ])"
import json
import re
from hashlib import sha1
from struct import unpack
from subprocess import run
from requests import get
# Fetch the latest stable release metadata from GitHub
latestTag = get("https://api.github.com/repos/facebook/sapling/releases/latest").json()[
"tag_name"
]
def nixPrefetchUrl(url):
return run(
["nix-prefetch-url", "--type", "sha256", url],
check=True,
text=True,
capture_output=True,
).stdout.rstrip()
# Fetch the `setup.py` source and look for instances of assets being downloaded
# from files.pythonhosted.org.
setupPy = get(
f"https://github.com/facebook/sapling/raw/{latestTag}/eden/scm/setup.py"
).text
foundUrls = re.findall(r'(https://files\.pythonhosted\.org/packages/[^\s]+)"', setupPy)
dataDeps = {
"links": [{"url": url, "sha256": nixPrefetchUrl(url)} for url in foundUrls],
"version": latestTag,
# Find latest's git tag which corresponds to the Sapling version. Also
# needed is a hash of the version, so calculate that here. Taken from
# Sapling source `$root/eden/scm/setup_with_version.py`.
"versionHash": str(unpack(">Q", sha1(latestTag.encode("ascii")).digest()[:8])[0]),
}
open("deps.json", "w").write(json.dumps(dataDeps, indent=2, sort_keys=True) + "\n")