Merge pull request #194310 from lilyinstarlight/pkg/curl-impersonate

curl-impersonate: init at 0.5.4 and replace curl-impersonate-bin
This commit is contained in:
7c6f434c 2023-07-23 17:00:17 +00:00 committed by GitHub
commit b02fd49f16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 473 additions and 21 deletions

@ -203,6 +203,7 @@ in {
couchdb = handleTest ./couchdb.nix {};
cri-o = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cri-o.nix {};
cups-pdf = handleTest ./cups-pdf.nix {};
curl-impersonate = handleTest ./curl-impersonate.nix {};
custom-ca = handleTest ./custom-ca.nix {};
croc = handleTest ./croc.nix {};
darling = handleTest ./darling.nix {};

@ -0,0 +1,157 @@
/*
Test suite for curl-impersonate
Abstract:
Uses the test suite from the curl-impersonate source repo which:
1. Performs requests with libcurl and captures the TLS client-hello
packets with tcpdump to compare against known-good signatures
2. Spins up an nghttpd2 server to test client HTTP/2 headers against
known-good headers
See https://github.com/lwthiker/curl-impersonate/tree/main/tests/signatures
for details.
Notes:
- We need to have our own web server running because the tests expect to be able
to hit domains like wikipedia.org and the sandbox has no internet
- We need to be able to do (verifying) TLS handshakes without internet access.
We do that by creating a trusted CA and issuing a cert that includes
all of the test domains as subject-alternative names and then spoofs the
hostnames in /etc/hosts.
*/
import ./make-test-python.nix ({ pkgs, lib, ... }: let
# Update with domains in TestImpersonate.TEST_URLS if needed from:
# https://github.com/lwthiker/curl-impersonate/blob/main/tests/test_impersonate.py
domains = [
"www.wikimedia.org"
"www.wikipedia.org"
"www.mozilla.org"
"www.apache.org"
"www.kernel.org"
"git-scm.com"
];
tls-certs = let
# Configure CA with X.509 v3 extensions that would be trusted by curl
ca-cert-conf = pkgs.writeText "curl-impersonate-ca.cnf" ''
basicConstraints = critical, CA:TRUE
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer:always
keyUsage = critical, cRLSign, digitalSignature, keyCertSign
'';
# Configure leaf certificate with X.509 v3 extensions that would be trusted
# by curl and set subject-alternative names for test domains
tls-cert-conf = pkgs.writeText "curl-impersonate-tls.cnf" ''
basicConstraints = critical, CA:FALSE
subjectKeyIdentifier = hash
authorityKeyIdentifier = keyid:always, issuer:always
keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment, keyAgreement
extendedKeyUsage = critical, serverAuth
subjectAltName = @alt_names
[alt_names]
${lib.concatStringsSep "\n" (lib.imap0 (idx: domain: "DNS.${toString idx} = ${domain}") domains)}
'';
in pkgs.runCommand "curl-impersonate-test-certs" {
nativeBuildInputs = [ pkgs.openssl ];
} ''
# create CA certificate and key
openssl req -newkey rsa:4096 -keyout ca-key.pem -out ca-csr.pem -nodes -subj '/CN=curl-impersonate-ca.nixos.test'
openssl x509 -req -sha512 -in ca-csr.pem -key ca-key.pem -out ca.pem -extfile ${ca-cert-conf} -days 36500
openssl x509 -in ca.pem -text
# create server certificate and key
openssl req -newkey rsa:4096 -keyout key.pem -out csr.pem -nodes -subj '/CN=curl-impersonate.nixos.test'
openssl x509 -req -sha512 -in csr.pem -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -extfile ${tls-cert-conf} -days 36500
openssl x509 -in cert.pem -text
# output CA cert and server cert and key
mkdir -p $out
cp key.pem cert.pem ca.pem $out
'';
# Test script
curl-impersonate-test = let
# Build miniature libcurl client used by test driver
minicurl = pkgs.runCommandCC "minicurl" {
buildInputs = [ pkgs.curl ];
} ''
mkdir -p $out/bin
$CC -Wall -Werror -o $out/bin/minicurl ${pkgs.curl-impersonate.src}/tests/minicurl.c `curl-config --libs`
'';
in pkgs.writeShellScript "curl-impersonate-test" ''
set -euxo pipefail
# Test driver requirements
export PATH="${with pkgs; lib.makeBinPath [
bash
coreutils
python3Packages.pytest
nghttp2
tcpdump
]}"
export PYTHONPATH="${with pkgs.python3Packages; makePythonPath [
pyyaml
pytest-asyncio
dpkt
]}"
# Prepare test root prefix
mkdir -p usr/{bin,lib}
cp -rs ${pkgs.curl-impersonate}/* ${minicurl}/* usr/
cp -r ${pkgs.curl-impersonate.src}/tests ./
# Run tests
cd tests
pytest . --install-dir ../usr --capture-interface eth1
'';
in {
name = "curl-impersonate";
meta = with lib.maintainers; {
maintainers = [ lilyinstarlight ];
};
nodes = {
web = { nodes, pkgs, lib, config, ... }: {
networking.firewall.allowedTCPPorts = [ 80 443 ];
services = {
nginx = {
enable = true;
virtualHosts."curl-impersonate.nixos.test" = {
default = true;
addSSL = true;
sslCertificate = "${tls-certs}/cert.pem";
sslCertificateKey = "${tls-certs}/key.pem";
};
};
};
};
curl = { nodes, pkgs, lib, config, ... }: {
networking.extraHosts = lib.concatStringsSep "\n" (map (domain: "${nodes.web.networking.primaryIPAddress} ${domain}") domains);
security.pki.certificateFiles = [ "${tls-certs}/ca.pem" ];
};
};
testScript = { nodes, ... }: ''
start_all()
with subtest("Wait for network"):
web.wait_for_unit("network-online.target")
curl.wait_for_unit("network-online.target")
with subtest("Wait for web server"):
web.wait_for_unit("nginx.service")
web.wait_for_open_port(443)
with subtest("Run curl-impersonate tests"):
curl.succeed("${curl-impersonate-test}")
'';
})

@ -0,0 +1,13 @@
diff --git a/Makefile.in b/Makefile.in
index 877c54f..3e39ed1 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -209,6 +209,8 @@ $(NSS_VERSION).tar.gz:
$(nss_static_libs): $(NSS_VERSION).tar.gz
tar xf $(NSS_VERSION).tar.gz
+ sed -i -e "1s@#!/usr/bin/env bash@#!$$(type -p bash)@" $(NSS_VERSION)/nss/build.sh
+ sed -i -e "s@/usr/bin/env grep@$$(type -p grep)@" $(NSS_VERSION)/nss/coreconf/config.gypi
ifeq ($(host),$(build))
# Native build, use NSS' build script.

@ -1,27 +1,186 @@
#TODO: It should be possible to build this from source, but it's currently a lot faster to just package the binaries.
{ lib, stdenv, fetchzip, zlib, autoPatchelfHook }:
stdenv.mkDerivation rec {
pname = "curl-impersonate-bin";
version = "v0.5.3";
{ lib
, stdenv
, fetchFromGitHub
, fetchpatch
, callPackage
, buildGoModule
, installShellFiles
, symlinkJoin
, zlib
, sqlite
, cmake
, python3
, ninja
, perl
, autoconf
, automake
, libtool
, darwin
, cacert
, unzip
, go
, p11-kit
, nixosTests
}:
src = fetchzip {
url = "https://github.com/lwthiker/curl-impersonate/releases/download/${version}/curl-impersonate-${version}.x86_64-linux-gnu.tar.gz";
sha256 = "sha256-+cH1swAIadIrWG9anzf0dcW6qyBjcKsUHFWdv75F49g=";
stripRoot = false;
let
makeCurlImpersonate = { name, target }: stdenv.mkDerivation rec {
pname = "curl-impersonate-${name}";
version = "0.5.4";
src = fetchFromGitHub {
owner = "lwthiker";
repo = "curl-impersonate";
rev = "v${version}";
hash = "sha256-LBGWFal2szqgURIBCLB84kHWpdpt5quvBBZu6buGj2A=";
};
patches = [
# Fix shebangs in the NSS build script
# (can't just patchShebangs since makefile unpacks it)
./curl-impersonate-0.5.2-fix-shebangs.patch
];
strictDeps = true;
nativeBuildInputs = lib.optionals stdenv.isDarwin [
# Must come first so that it shadows the 'libtool' command but leaves 'libtoolize'
darwin.cctools
] ++ [
installShellFiles
cmake
python3
python3.pkgs.gyp
ninja
perl
autoconf
automake
libtool
unzip
go
];
buildInputs = [
zlib
sqlite
];
configureFlags = [
"--with-ca-bundle=${if stdenv.isDarwin then "/etc/ssl/cert.pem" else "/etc/ssl/certs/ca-certificates.crt"}"
"--with-ca-path=${cacert}/etc/ssl/certs"
];
buildFlags = [ "${target}-build" ];
checkTarget = "${target}-checkbuild";
installTargets = [ "${target}-install" ];
doCheck = true;
dontUseCmakeConfigure = true;
dontUseNinjaBuild = true;
dontUseNinjaInstall = true;
dontUseNinjaCheck = true;
postUnpack = lib.concatStringsSep "\n" (lib.mapAttrsToList (name: dep: "ln -sT ${dep.outPath} source/${name}") (lib.filterAttrs (n: v: v ? outPath) passthru.deps));
preConfigure = ''
export GOCACHE=$TMPDIR/go-cache
export GOPATH=$TMPDIR/go
export GOPROXY=file://${passthru.boringssl-go-modules}
export GOSUMDB=off
# Need to get value of $out for this flag
configureFlagsArray+=("--with-libnssckbi=$out/lib")
'';
postInstall = ''
# Remove vestigial *-config script
rm $out/bin/curl-impersonate-${name}-config
# Patch all shebangs of installed scripts
patchShebangs $out/bin
# Build and install completions for each curl binary
# Patch in correct binary name and alias it to all scripts
perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell zsh >$TMPDIR/curl-impersonate-${name}.zsh
substituteInPlace $TMPDIR/curl-impersonate-${name}.zsh \
--replace \
'#compdef curl' \
"#compdef curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' %f=curl-impersonate-${name}')"
perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell fish >$TMPDIR/curl-impersonate-${name}.fish
substituteInPlace $TMPDIR/curl-impersonate-${name}.fish \
--replace \
'--command curl' \
"--command curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' --command %f')"
# Install zsh and fish completions
installShellCompletion $TMPDIR/curl-impersonate-${name}.{zsh,fish}
'';
preFixup = let
libext = stdenv.hostPlatform.extensions.sharedLibrary;
in ''
# If libnssckbi.so is needed, link libnssckbi.so without needing nss in closure
if grep -F nssckbi $out/lib/libcurl-impersonate-*${libext} &>/dev/null; then
# NOTE: "p11-kit-trust" always ends in ".so" even when on darwin
ln -s ${p11-kit}/lib/pkcs11/p11-kit-trust.so $out/lib/libnssckbi${libext}
${lib.optionalString stdenv.isLinux "patchelf --add-needed libnssckbi${libext} $out/lib/libcurl-impersonate-*${libext}"}
fi
'';
disallowedReferences = [ go ];
passthru = {
deps = callPackage ./deps.nix {};
boringssl-go-modules = (buildGoModule {
inherit (passthru.deps."boringssl.zip") name;
src = passthru.deps."boringssl.zip";
vendorHash = "sha256-ISmRdumckvSu7hBXrjvs5ZApShDiGLdD3T5B0fJ1x2Q=";
nativeBuildInputs = [ unzip ];
proxyVendor = true;
}).go-modules;
};
meta = with lib; {
description = "A special build of curl that can impersonate Chrome & Firefox";
homepage = "https://github.com/lwthiker/curl-impersonate";
license = with licenses; [ curl mit ];
maintainers = with maintainers; [ deliciouslytyped lilyinstarlight ];
platforms = platforms.unix;
knownVulnerabilities = [
"CVE-2023-32001" # fopen TOCTOU race condition - https://curl.se/docs/CVE-2023-32001.html
"CVE-2022-43551" # HSTS bypass - https://curl.se/docs/CVE-2022-43551.html
"CVE-2022-42916" # HSTS bypass - https://curl.se/docs/CVE-2022-42916.html
];
};
};
in
nativeBuildInputs = [ autoPatchelfHook zlib ];
symlinkJoin rec {
pname = "curl-impersonate";
inherit (passthru.curl-impersonate-ff) version meta;
installPhase = ''
mkdir -p $out/bin
cp * $out/bin
'';
name = "${pname}-${version}";
meta = with lib; {
description = "curl-impersonate: A special build of curl that can impersonate Chrome & Firefox ";
homepage = "https://github.com/lwthiker/curl-impersonate";
license = with licenses; [ curl mit ];
maintainers = with maintainers; [ deliciouslytyped ];
platforms = platforms.linux; #TODO I'm unsure about the restrictions here, feel free to expand the platforms it if it works elsewhere.
paths = [
passthru.curl-impersonate-ff
passthru.curl-impersonate-chrome
];
passthru = {
curl-impersonate-ff = makeCurlImpersonate { name = "ff"; target = "firefox"; };
curl-impersonate-chrome = makeCurlImpersonate { name = "chrome"; target = "chrome"; };
updateScript = ./update.sh;
inherit (passthru.curl-impersonate-ff) src;
tests = { inherit (nixosTests) curl-impersonate; };
};
}

@ -0,0 +1,29 @@
# Generated by update.sh
{ fetchurl }:
{
"curl-7.84.0.tar.xz" = fetchurl {
url = "https://curl.se/download/curl-7.84.0.tar.xz";
hash = "sha256-LRGLQ/VHv+W66AbY1HtOWW6lslpsHwgK70n7zYF8Xbg=";
};
"brotli-1.0.9.tar.gz" = fetchurl {
url = "https://github.com/google/brotli/archive/refs/tags/v1.0.9.tar.gz";
hash = "sha256-+ejYHQQFumbRgVKa9CozVPg4yTkJX/mZMNpqqc32/kY=";
};
"nss-3.87.tar.gz" = fetchurl {
url = "https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_87_RTM/src/nss-3.87-with-nspr-4.35.tar.gz";
hash = "sha256-63DqC1jc5pqkkOnp/s0TKn1kTh2j1jHhYzdqDcwRoCI=";
};
"boringssl.zip" = fetchurl {
url = "https://github.com/google/boringssl/archive/3a667d10e94186fd503966f5638e134fe9fb4080.zip";
hash = "sha256-HsDIkd1x5IH49fUF07dJaabMIMsQygW+NI7GneULpA8=";
};
"nghttp2-1.46.0.tar.bz2" = fetchurl {
url = "https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2";
hash = "sha256-moKXjIcAcbdp8n0riBkct3/clFpRwdaFx/YafhP8Ryk=";
};
}

@ -0,0 +1,91 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p git nix jq coreutils gnugrep gnused curl common-updater-scripts
set -euo pipefail
nixpkgs="$(git rev-parse --show-toplevel || (printf 'Could not find root of nixpkgs repo\nAre we running from within the nixpkgs git repo?\n' >&2; exit 1))"
stripwhitespace() {
sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'
}
narhash() {
nix --extra-experimental-features nix-command store prefetch-file --json "$1" | jq -r .hash
}
nixeval() {
nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1" | jq -r .
}
vendorhash() {
(nix --extra-experimental-features nix-command build --no-link -f "$nixpkgs" --no-link "$1" 2>&1 >/dev/null | tail -n3 | grep -F got: | cut -d: -f2- | stripwhitespace) 2>/dev/null || true
}
findpath() {
path="$(nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1.meta.position" | jq -r . | cut -d: -f1)"
outpath="$(nix --extra-experimental-features nix-command eval --json --impure --expr "builtins.fetchGit \"$nixpkgs\"")"
if [ -n "$outpath" ]; then
path="${path/$(echo "$outpath" | jq -r .)/$nixpkgs}"
fi
echo "$path"
}
getvar() {
echo "$2" | grep -F "$1" | sed -e 's/:=/:/g' | cut -d: -f2- | stripwhitespace
}
attr="${UPDATE_NIX_ATTR_PATH:-curl-impersonate}"
version="$(curl -sSL "https://api.github.com/repos/lwthiker/curl-impersonate/releases/latest" | jq -r .tag_name | sed -e 's/^v//')"
pkgpath="$(findpath "$attr")"
updated="$(cd "$nixpkgs" && update-source-version "$attr" "$version" --file="$pkgpath" --print-changes | jq -r length)"
if [ "$updated" -eq 0 ]; then
echo 'update.sh: Package version not updated, nothing to do.'
exit 0
fi
vars="$(curl -sSL "https://github.com/lwthiker/curl-impersonate/raw/v$version/Makefile.in" | grep '^ *[^ ]*_\(VERSION\|URL\|COMMIT\) *:=')"
cat >"$(dirname "$pkgpath")"/deps.nix <<EOF
# Generated by update.sh
{ fetchurl }:
{
"$(getvar CURL_VERSION "$vars").tar.xz" = fetchurl {
url = "https://curl.se/download/$(getvar CURL_VERSION "$vars").tar.xz";
hash = "$(narhash "https://curl.se/download/$(getvar CURL_VERSION "$vars").tar.xz")";
};
"brotli-$(getvar BROTLI_VERSION "$vars").tar.gz" = fetchurl {
url = "https://github.com/google/brotli/archive/refs/tags/v$(getvar BROTLI_VERSION "$vars").tar.gz";
hash = "$(narhash "https://github.com/google/brotli/archive/refs/tags/v$(getvar BROTLI_VERSION "$vars").tar.gz")";
};
"$(getvar NSS_VERSION "$vars").tar.gz" = fetchurl {
url = "$(getvar NSS_URL "$vars")";
hash = "$(narhash "$(getvar NSS_URL "$vars")")";
};
"boringssl.zip" = fetchurl {
url = "https://github.com/google/boringssl/archive/$(getvar BORING_SSL_COMMIT "$vars").zip";
hash = "$(narhash "https://github.com/google/boringssl/archive/$(getvar BORING_SSL_COMMIT "$vars").zip")";
};
"$(getvar NGHTTP2_VERSION "$vars").tar.bz2" = fetchurl {
url = "$(getvar NGHTTP2_URL "$vars")";
hash = "$(narhash "$(getvar NGHTTP2_URL "$vars")")";
};
}
EOF
curhash="$(nixeval "$attr.curl-impersonate-chrome.boringssl-go-modules.outputHash")"
newhash="$(vendorhash "$attr.curl-impersonate-chrome.boringssl-go-modules")"
if [ -n "$newhash" ] && [ "$curhash" != "$newhash" ]; then
sed -i -e "s|\"$curhash\"|\"$newhash\"|" "$pkgpath"
else
echo 'update.sh: New vendorHash same as old vendorHash, nothing to do.'
fi

@ -351,6 +351,7 @@ mapAliases ({
cups-kyodialog3 = cups-kyodialog; # Added 2022-11-12
cupsBjnp = throw "'cupsBjnp' has been renamed to/replaced by 'cups-bjnp'"; # Converted to throw 2022-02-22
cups_filters = throw "'cups_filters' has been renamed to/replaced by 'cups-filters'"; # Converted to throw 2022-02-22
curl-impersonate-bin = throw "'curl-impersonate-bin' has been replaced by 'curl-impersonate'"; # Added 2022-10-08
curlcpp = throw "curlcpp has been removed, no active maintainers and no usage within nixpkgs"; # Added 2022-05-10
curaByDagoma = throw "curaByDagoma has been removed from nixpkgs, because it was unmaintained and dependent on python2 packages"; # Added 2022-01-12
curaLulzbot = throw "curaLulzbot has been removed due to insufficient upstream support for a modern dependency chain"; # Added 2021-10-23

@ -7017,7 +7017,8 @@ with pkgs;
curlWithGnuTls = curl.override { gnutlsSupport = true; opensslSupport = false; };
curl-impersonate-bin = callPackage ../tools/networking/curl-impersonate { };
curl-impersonate = darwin.apple_sdk_11_0.callPackage ../tools/networking/curl-impersonate { };
inherit (curl-impersonate) curl-impersonate-ff curl-impersonate-chrome;
curlie = callPackage ../tools/networking/curlie { };