mixRelease: improve the implementation (#266397)
* mixRelease: format code by nixpkgs-format * mixRelease: investigate why erlang is referenced in resulting derivation * mixRelease: organize nativeBuildInputs and buildInputs It: + organizes `nativeBuildInputs` in a structured way. + moves `builtins.attrValues mixNixDeps` to `nativeBuildInputs`, because it's only used in compile-time * mixRelease: remove current attempt for removing erlang references in resulting derivation As said in the comment about "remove erlang references in resulting derivation", for now, we don't have a robust method to do that. Although these removed code did some work, they did not achieve the final goal - remove erlang references in resulting derivation. Therefore, it is better to remove them and provide future implementation with a clean foundation. > If you want to find these old codes, you can also retrieve them from > the git history. * mixRelease: remove all files for Microsoft Windows * mixRelease: add new option - `removeCookie` * mixRelease: polish comments + Capitalize the sentences. + Add punctuation marks. + Format a little code. * mixRelease: wrap programs in $out/bin with their runtime deps * mixRelease: commit what happysalada suggests --------- Co-authored-by: c4710n <c4710n@users.noreply.github.com>
This commit is contained in:
Zeke Dou
GitHub
parent
201f93fb19
commit
993c8f162d
@ -1,4 +1,19 @@
|
||||
{ stdenv, lib, elixir, erlang, findutils, hex, rebar, rebar3, fetchMixDeps, makeWrapper, git, ripgrep }@inputs:
|
||||
{ stdenv
|
||||
, lib
|
||||
, elixir
|
||||
, erlang
|
||||
, hex
|
||||
, git
|
||||
, rebar
|
||||
, rebar3
|
||||
, fetchMixDeps
|
||||
, findutils
|
||||
, makeWrapper
|
||||
, coreutils
|
||||
, gnused
|
||||
, gnugrep
|
||||
, gawk
|
||||
}@inputs:
|
||||
|
||||
{ pname
|
||||
, version
|
||||
@ -10,80 +25,104 @@
|
||||
, mixEnv ? "prod"
|
||||
, compileFlags ? [ ]
|
||||
|
||||
# mix fixed output derivation dependencies
|
||||
# Mix dependencies provided as a fixed output derivation
|
||||
, mixFodDeps ? null
|
||||
|
||||
# mix dependencies generated by mix2nix
|
||||
# this assumes each dependency is built by buildMix or buildRebar3
|
||||
# each dependency needs to have a setup hook to add the lib path to $ERL_LIBS
|
||||
# this is how mix will find dependencies
|
||||
# Mix dependencies generated by mix2nix
|
||||
#
|
||||
# This assumes each dependency is built by buildMix or buildRebar3. Each
|
||||
# dependency needs to have a setup hook to add the lib path to $ERL_LIBS.
|
||||
# This is how Mix finds dependencies.
|
||||
, mixNixDeps ? { }
|
||||
|
||||
, elixir ? inputs.elixir
|
||||
, hex ? inputs.hex.override { inherit elixir; }
|
||||
|
||||
# This reduces closure size, but can lead to some hard to understand runtime
|
||||
# errors, so use with caution. See e.g.
|
||||
# https://github.com/whitfin/cachex/issues/205
|
||||
# https://framagit.org/framasoft/mobilizon/-/issues/1169
|
||||
# Remove releases/COOKIE
|
||||
#
|
||||
# People have different views on the nature of cookies. Some believe that they are
|
||||
# secrets, while others believe they are just ids for clustering nodes instead of
|
||||
# secrets.
|
||||
#
|
||||
# If you think cookie is secret, you can set this attr to true, then it will be
|
||||
# removed from nix store. If not, you can set it to false.
|
||||
#
|
||||
# For backward compatibility, it is set to true by default.
|
||||
#
|
||||
# You can always specify a custom cookie by using RELEASE_COOKIE environment
|
||||
# variable, regardless of the value of this attr.
|
||||
, removeCookie ? true
|
||||
|
||||
# This reduces closure size, but can lead to some hard to understand runtime
|
||||
# errors, so use with caution. See e.g.
|
||||
# https://github.com/whitfin/cachex/issues/205
|
||||
# https://framagit.org/framasoft/mobilizon/-/issues/1169
|
||||
, stripDebug ? false
|
||||
|
||||
, ...
|
||||
}@attrs:
|
||||
let
|
||||
# remove non standard attributes that cannot be coerced to strings
|
||||
# Remove non standard attributes that cannot be coerced to strings
|
||||
overridable = builtins.removeAttrs attrs [ "compileFlags" "mixNixDeps" ];
|
||||
in
|
||||
assert mixNixDeps != { } -> mixFodDeps == null;
|
||||
assert stripDebug -> !enableDebugInfo;
|
||||
|
||||
stdenv.mkDerivation (overridable // {
|
||||
# rg is used as a better grep to search for erlang references in the final release
|
||||
nativeBuildInputs = nativeBuildInputs ++ [ erlang hex elixir makeWrapper git ripgrep ];
|
||||
buildInputs = buildInputs ++ builtins.attrValues mixNixDeps;
|
||||
nativeBuildInputs = nativeBuildInputs ++
|
||||
# Erlang/Elixir deps
|
||||
[ erlang elixir hex git ] ++
|
||||
# Mix deps
|
||||
(builtins.attrValues mixNixDeps) ++
|
||||
# other compile-time deps
|
||||
[ findutils makeWrapper ];
|
||||
|
||||
buildInputs = buildInputs;
|
||||
|
||||
MIX_ENV = mixEnv;
|
||||
MIX_DEBUG = if enableDebugInfo then 1 else 0;
|
||||
HEX_OFFLINE = 1;
|
||||
|
||||
DEBUG = if enableDebugInfo then 1 else 0; # for Rebar3 compilation
|
||||
# the api with `mix local.rebar rebar path` makes a copy of the binary
|
||||
# some older dependencies still use rebar
|
||||
# The API with `mix local.rebar rebar path` makes a copy of the binary
|
||||
# some older dependencies still use rebar.
|
||||
MIX_REBAR = "${rebar}/bin/rebar";
|
||||
MIX_REBAR3 = "${rebar3}/bin/rebar3";
|
||||
|
||||
LC_ALL = "C.UTF-8";
|
||||
|
||||
postUnpack = ''
|
||||
export HEX_HOME="$TEMPDIR/hex"
|
||||
# Mix and Hex
|
||||
export MIX_HOME="$TEMPDIR/mix"
|
||||
export HEX_HOME="$TEMPDIR/hex"
|
||||
|
||||
# Rebar
|
||||
export REBAR_GLOBAL_CONFIG_DIR="$TEMPDIR/rebar3"
|
||||
export REBAR_CACHE_DIR="$TEMPDIR/rebar3.cache"
|
||||
|
||||
${lib.optionalString (mixFodDeps != null) ''
|
||||
# compilation of the dependencies will require
|
||||
# that the dependency path is writable
|
||||
# thus a copy to the TEMPDIR is inevitable here
|
||||
# Compilation of the dependencies will require that the dependency path is
|
||||
# writable, thus a copy to the $TEMPDIR is inevitable here.
|
||||
export MIX_DEPS_PATH="$TEMPDIR/deps"
|
||||
cp --no-preserve=mode -R "${mixFodDeps}" "$MIX_DEPS_PATH"
|
||||
''
|
||||
}
|
||||
|
||||
''}
|
||||
'' + (attrs.postUnpack or "");
|
||||
|
||||
configurePhase = attrs.configurePhase or ''
|
||||
runHook preConfigure
|
||||
|
||||
${./mix-configure-hook.sh}
|
||||
# this is needed for projects that have a specific compile step
|
||||
|
||||
# This is needed for projects that have a specific compile step
|
||||
# the dependency needs to be compiled in order for the task
|
||||
# to be available
|
||||
# Phoenix projects for example will need compile.phoenix
|
||||
# to be available.
|
||||
#
|
||||
# Phoenix projects for example will need compile.phoenix.
|
||||
mix deps.compile --no-deps-check --skip-umbrella-children
|
||||
|
||||
# Symlink dependency sources. This is needed for projects that require
|
||||
# access to the source of their dependencies. For example, Phoenix
|
||||
# applications need javascript assets to build asset bundles.
|
||||
# projects need javascript assets to build asset bundles.
|
||||
${lib.optionalString (mixNixDeps != { }) ''
|
||||
mkdir -p deps
|
||||
|
||||
@ -113,7 +152,6 @@ stdenv.mkDerivation (overridable // {
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
|
||||
installPhase = attrs.installPhase or ''
|
||||
runHook preInstall
|
||||
|
||||
@ -122,42 +160,50 @@ stdenv.mkDerivation (overridable // {
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
# Stripping of the binary is intentional
|
||||
# even though it does not affect beam files
|
||||
# it is necessary for NIFs binaries
|
||||
postFixup = ''
|
||||
if [ -e "$out/bin/${pname}.bat" ]; then # absent in special cases, i.e. elixir-ls
|
||||
rm "$out/bin/${pname}.bat" # windows file
|
||||
fi
|
||||
# contains secrets and should not be in the nix store
|
||||
# TODO document how to handle RELEASE_COOKIE
|
||||
# secrets should not be in the nix store.
|
||||
# This is only used for connecting multiple nodes
|
||||
if [ -e $out/releases/COOKIE ]; then # absent in special cases, i.e. elixir-ls
|
||||
# Remove files for Microsoft Windows
|
||||
rm -f "$out"/bin/*.bat
|
||||
|
||||
# Wrap programs in $out/bin with their runtime deps
|
||||
for f in $(find $out/bin/ -type f -executable); do
|
||||
wrapProgram "$f" \
|
||||
--prefix PATH : ${lib.makeBinPath [
|
||||
coreutils
|
||||
gnused
|
||||
gnugrep
|
||||
gawk
|
||||
]}
|
||||
done
|
||||
'' + lib.optionalString removeCookie ''
|
||||
if [ -e $out/releases/COOKIE ]; then
|
||||
rm $out/releases/COOKIE
|
||||
fi
|
||||
# removing unused erlang reference from resulting derivation to reduce
|
||||
# closure size
|
||||
if [ -e $out/erts-* ]; then
|
||||
echo "ERTS found in $out - removing references to erlang to reduce closure size"
|
||||
# there is a link in $out/erts-*/bin/start always
|
||||
# TODO:
|
||||
# sometimes there are links in dependencies like bcrypt compiled binaries
|
||||
# at the moment those are not removed since substituteInPlace will
|
||||
# error on binaries
|
||||
for file in $(rg "${erlang}/lib/erlang" "$out" --files-with-matches); do
|
||||
echo "removing reference to erlang in $file"
|
||||
substituteInPlace "$file" --replace "${erlang}/lib/erlang" "$out"
|
||||
done
|
||||
fi
|
||||
'' + lib.optionalString stripDebug ''
|
||||
# strip debug symbols to avoid hardreferences to "foreign" closures actually
|
||||
# Strip debug symbols to avoid hardreferences to "foreign" closures actually
|
||||
# not needed at runtime, while at the same time reduce size of BEAM files.
|
||||
erl -noinput -eval 'lists:foreach(fun(F) -> io:format("Stripping ~p.~n", [F]), beam_lib:strip(F) end, filelib:wildcard("'"$out"'/**/*.beam"))' -s init stop
|
||||
'';
|
||||
|
||||
# TODO investigate why the resulting closure still has
|
||||
# a reference to erlang.
|
||||
# uncommenting the following will fail the build
|
||||
# disallowedReferences = [ erlang ];
|
||||
# TODO: remove erlang references in resulting derivation
|
||||
#
|
||||
# # Step 1 - investigate why the resulting derivation still has references to erlang.
|
||||
#
|
||||
# The reason is that the generated binaries contains erlang reference. Here's a repo to
|
||||
# demonstrate the problem - <https://github.com/plastic-gun/nix-mix-release-unwanted-references>.
|
||||
#
|
||||
#
|
||||
# # Step 2 - remove erlang references from the binaries
|
||||
#
|
||||
# As said in above repo, it's hard to remove erlang references from `.beam` binaries.
|
||||
#
|
||||
# We need more experienced developers to resolve this issue.
|
||||
#
|
||||
#
|
||||
# # Tips
|
||||
#
|
||||
# When resolving this issue, it is convenient to fail the build when erlang is referenced,
|
||||
# which can be achieved by using:
|
||||
#
|
||||
# disallowedReferences = [ erlang ];
|
||||
#
|
||||
})
|
||||
|
||||
Loading…
Reference in New Issue
Block a user