From 351cec5db3b7ca839779316a8b7c524ea8941f0f Mon Sep 17 00:00:00 2001 From: Matthieu Coudron Date: Fri, 14 Apr 2023 22:02:17 +0200 Subject: [PATCH] use subcommands in plugin updaters (#223164) * update.py: introduce subparsers for plugin updaters This is preliminary work to help create more powerful plugin updaters. Namely I would like to be able to "just add" plugins without refreshing the older ones (helpful when github temporarily removes a user from github due to automated bot detection). Also concerning the lua updater, we pin some of the dependencies, and I would like to be able to unpin the package without editing the csv (coming in later PRs). * doc/updaters: update command to update editor plugins including vim, kakoune and lua packages Co-authored-by: figsoda --- doc/languages-frameworks/lua.section.md | 15 ++- doc/languages-frameworks/vim.section.md | 4 +- maintainers/scripts/pluginupdate.py | 127 +++++++++++++----- maintainers/scripts/update-luarocks-packages | 6 +- .../editors/kakoune/plugins/update.py | 5 +- .../editors/vim/plugins/get-plugins.nix | 19 +++ .../editors/vim/plugins/update.py | 63 ++++----- 7 files changed, 150 insertions(+), 89 deletions(-) create mode 100644 pkgs/applications/editors/vim/plugins/get-plugins.nix diff --git a/doc/languages-frameworks/lua.section.md b/doc/languages-frameworks/lua.section.md index 92ce8af1331b..2ed02ab9d6c7 100644 --- a/doc/languages-frameworks/lua.section.md +++ b/doc/languages-frameworks/lua.section.md @@ -129,16 +129,21 @@ Let's present the luarocks way first and the manual one in a second time. ### Packaging a library on luarocks {#packaging-a-library-on-luarocks} [Luarocks.org](https://luarocks.org/) is the main repository of lua packages. -The site proposes two types of packages, the rockspec and the src.rock +The site proposes two types of packages, the `rockspec` and the `src.rock` (equivalent of a [rockspec](https://github.com/luarocks/luarocks/wiki/Rockspec-format) but with the source). -These packages can have different build types such as `cmake`, `builtin` etc . -Luarocks-based packages are generated in pkgs/development/lua-modules/generated-packages.nix from -the whitelist maintainers/scripts/luarocks-packages.csv and updated by running maintainers/scripts/update-luarocks-packages. +Luarocks-based packages are generated in [pkgs/development/lua-modules/generated-packages.nix](https://github.com/NixOS/nixpkgs/tree/master/pkgs/development/lua-modules/generated-packages.nix) from +the whitelist maintainers/scripts/luarocks-packages.csv and updated by running +the script +[maintainers/scripts/update-luarocks-packages](https://github.com/NixOS/nixpkgs/tree/master/maintainers/scripts/update-luarocks-packages): + +```sh +./maintainers/scripts/update-luarocks-packages update +``` [luarocks2nix](https://github.com/nix-community/luarocks) is a tool capable of generating nix derivations from both rockspec and src.rock (and favors the src.rock). The automation only goes so far though and some packages need to be customized. -These customizations go in `pkgs/development/lua-modules/overrides.nix`. +These customizations go in [pkgs/development/lua-modules/overrides.nix](https://github.com/NixOS/nixpkgs/tree/master/pkgs/development/lua-modules/overrides.nix). For instance if the rockspec defines `external_dependencies`, these need to be manually added to the overrides.nix. You can try converting luarocks packages to nix packages with the command `nix-shell -p luarocks-nix` and then `luarocks nix PKG_NAME`. diff --git a/doc/languages-frameworks/vim.section.md b/doc/languages-frameworks/vim.section.md index 39e38c6a123d..bf0d663179b9 100644 --- a/doc/languages-frameworks/vim.section.md +++ b/doc/languages-frameworks/vim.section.md @@ -212,7 +212,7 @@ Note: this is not possible anymore for Neovim. ## Adding new plugins to nixpkgs {#adding-new-plugins-to-nixpkgs} -Nix expressions for Vim plugins are stored in [pkgs/applications/editors/vim/plugins](https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/editors/vim/plugins). For the vast majority of plugins, Nix expressions are automatically generated by running [`./update.py`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/update.py). This creates a [generated.nix](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/generated.nix) file based on the plugins listed in [vim-plugin-names](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/vim-plugin-names). Plugins are listed in alphabetical order in `vim-plugin-names` using the format `[github username]/[repository]@[gitref]`. For example https://github.com/scrooloose/nerdtree becomes `scrooloose/nerdtree`. +Nix expressions for Vim plugins are stored in [pkgs/applications/editors/vim/plugins](https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/editors/vim/plugins). For the vast majority of plugins, Nix expressions are automatically generated by running [`./update.py`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/update.py). This creates a [generated.nix](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/generated.nix) file based on the plugins listed in [vim-plugin-names](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/vim-plugin-names). After running `./update.py`, if nvim-treesitter received an update, also run [`nvim-treesitter/update.py`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/update.py) to update the tree sitter grammars for `nvim-treesitter`. @@ -226,7 +226,7 @@ deoplete-fish = super.deoplete-fish.overrideAttrs(old: { Sometimes plugins require an override that must be changed when the plugin is updated. This can cause issues when Vim plugins are auto-updated but the associated override isn't updated. For these plugins, the override should be written so that it specifies all information required to install the plugin, and running `./update.py` doesn't change the derivation for the plugin. Manually updating the override is required to update these types of plugins. An example of such a plugin is `LanguageClient-neovim`. -To add a new plugin, run `./update.py --add "[owner]/[name]"`. **NOTE**: This script automatically commits to your git repository. Be sure to check out a fresh branch before running. +To add a new plugin, run `./update.py add "[owner]/[name]"`. **NOTE**: This script automatically commits to your git repository. Be sure to check out a fresh branch before running. Finally, there are some plugins that are also packaged in nodePackages because they have Javascript-related build steps, such as running webpack. Those plugins are not listed in `vim-plugin-names` or managed by `update.py` at all, and are included separately in `overrides.nix`. Currently, all these plugins are related to the `coc.nvim` ecosystem of the Language Server Protocol integration with Vim/Neovim. diff --git a/maintainers/scripts/pluginupdate.py b/maintainers/scripts/pluginupdate.py index 3341fea29106..7c6cfd4fed7f 100644 --- a/maintainers/scripts/pluginupdate.py +++ b/maintainers/scripts/pluginupdate.py @@ -1,4 +1,7 @@ -# Used by pkgs/applications/editors/vim/plugins/update.py and pkgs/applications/editors/kakoune/plugins/update.py +# python library used to update plugins: +# - pkgs/applications/editors/vim/plugins/update.py +# - pkgs/applications/editors/kakoune/plugins/update.py +# - maintainers/scripts/update-luarocks-packages # format: # $ nix run nixpkgs.python3Packages.black -c black update.py @@ -315,10 +318,10 @@ def run_nix_expr(expr): with CleanEnvironment(): cmd = ["nix", "eval", "--extra-experimental-features", "nix-command", "--impure", "--json", "--expr", expr] - log.debug("Running command %s", cmd) + log.debug("Running command %s", " ".join(cmd)) out = subprocess.check_output(cmd) - data = json.loads(out) - return data + data = json.loads(out) + return data class Editor: @@ -344,12 +347,39 @@ class Editor: self.cache_file = cache_file or f"{name}-plugin-cache.json" self.nixpkgs_repo = None + def add(self, args): + '''CSV spec''' + log.debug("called the 'add' command") + fetch_config = FetchConfig(args.proc, args.github_token) + editor = self + for plugin_line in args.add_plugins: + log.debug("using plugin_line", plugin_line) + pdesc = PluginDesc.load_from_string(fetch_config, plugin_line) + log.debug("loaded as pdesc", pdesc) + append = [ pdesc ] + editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=append) + plugin, _ = prefetch_plugin(pdesc, ) + autocommit = not args.no_commit + if autocommit: + commit( + editor.nixpkgs_repo, + "{drv_name}: init at {version}".format( + drv_name=editor.get_drv_name(plugin.normalized_name), + version=plugin.version + ), + [args.outfile, args.input_file], + ) + + # Expects arguments generated by 'update' subparser + def update(self, args ): + '''CSV spec''' + print("the update member function should be overriden in subclasses") + def get_current_plugins(self) -> List[Plugin]: """To fill the cache""" data = run_nix_expr(self.get_plugins) plugins = [] for name, attr in data.items(): - print("get_current_plugins: name %s" % name) p = Plugin(name, attr["rev"], attr["submodules"], attr["sha256"]) plugins.append(p) return plugins @@ -358,7 +388,7 @@ class Editor: '''CSV spec''' return load_plugins_from_csv(config, plugin_file) - def generate_nix(self, plugins, outfile: str): + def generate_nix(self, _plugins, _outfile: str): '''Returns nothing for now, writes directly to outfile''' raise NotImplementedError() @@ -395,34 +425,28 @@ class Editor: return rewrite_input(*args, **kwargs) def create_parser(self): - parser = argparse.ArgumentParser( + common = argparse.ArgumentParser( + add_help=False, description=(f""" Updates nix derivations for {self.name} plugins.\n By default from {self.default_in} to {self.default_out}""" ) ) - parser.add_argument( - "--add", - dest="add_plugins", - default=[], - action="append", - help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo", - ) - parser.add_argument( + common.add_argument( "--input-names", "-i", dest="input_file", default=self.default_in, help="A list of plugins in the form owner/repo", ) - parser.add_argument( + common.add_argument( "--out", "-o", dest="outfile", default=self.default_out, help="Filename to save generated nix code", ) - parser.add_argument( + common.add_argument( "--proc", "-p", dest="proc", @@ -430,7 +454,7 @@ class Editor: default=30, help="Number of concurrent processes to spawn. Setting --github-token allows higher values.", ) - parser.add_argument( + common.add_argument( "--github-token", "-t", type=str, @@ -438,16 +462,61 @@ class Editor: help="""Allows to set --proc to higher values. Uses GITHUB_API_TOKEN environment variables as the default value.""", ) - parser.add_argument( + common.add_argument( "--no-commit", "-n", action="store_true", default=False, help="Whether to autocommit changes" ) - parser.add_argument( + common.add_argument( "--debug", "-d", choices=LOG_LEVELS.keys(), default=logging.getLevelName(logging.WARN), help="Adjust log level" ) - return parser + + main = argparse.ArgumentParser( + parents=[common], + description=(f""" + Updates nix derivations for {self.name} plugins.\n + By default from {self.default_in} to {self.default_out}""" + ) + ) + + subparsers = main.add_subparsers(dest="command", required=False) + padd = subparsers.add_parser( + "add", parents=[], + description="Add new plugin", + add_help=False, + ) + padd.set_defaults(func=self.add) + padd.add_argument( + "add_plugins", + default=None, + nargs="+", + help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo", + ) + + pupdate = subparsers.add_parser( + "update", + description="Update all or a subset of existing plugins", + add_help=False, + ) + pupdate.set_defaults(func=self.update) + return main + + def run(self,): + ''' + Convenience function + ''' + parser = self.create_parser() + args = parser.parse_args() + command = args.command or "update" + log.setLevel(LOG_LEVELS[args.debug]) + log.info("Chose to run command: %s", command) + + if not args.no_commit: + self.nixpkgs_repo = git.Repo(self.root, search_parent_directories=True) + + getattr(self, command)(args) + @@ -661,7 +730,6 @@ def commit(repo: git.Repo, message: str, files: List[Path]) -> None: def update_plugins(editor: Editor, args): """The main entry function of this module. All input arguments are grouped in the `Editor`.""" - log.setLevel(LOG_LEVELS[args.debug]) log.info("Start updating plugins") fetch_config = FetchConfig(args.proc, args.github_token) update = editor.get_update(args.input_file, args.outfile, fetch_config) @@ -684,18 +752,3 @@ def update_plugins(editor: Editor, args): [args.outfile, args.input_file, editor.deprecated], ) - for plugin_line in args.add_plugins: - pdesc = PluginDesc.load_from_string(fetch_config, plugin_line) - append = [ pdesc ] - editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, append=append) - update() - plugin, _ = prefetch_plugin(pdesc, ) - if autocommit: - commit( - editor.nixpkgs_repo, - "{drv_name}: init at {version}".format( - drv_name=editor.get_drv_name(plugin.normalized_name), - version=plugin.version - ), - [args.outfile, args.input_file], - ) diff --git a/maintainers/scripts/update-luarocks-packages b/maintainers/scripts/update-luarocks-packages index f34aa53626da..0e1e188e0b33 100755 --- a/maintainers/scripts/update-luarocks-packages +++ b/maintainers/scripts/update-luarocks-packages @@ -203,11 +203,7 @@ def main(): default_out = ROOT.joinpath(GENERATED_NIXFILE) ) - parser = editor.create_parser() - args = parser.parse_args() - - update_plugins(editor, args) - + editor.run() if __name__ == "__main__": diff --git a/pkgs/applications/editors/kakoune/plugins/update.py b/pkgs/applications/editors/kakoune/plugins/update.py index 49662a0e8e2e..ab5aa9242839 100755 --- a/pkgs/applications/editors/kakoune/plugins/update.py +++ b/pkgs/applications/editors/kakoune/plugins/update.py @@ -81,10 +81,7 @@ in lib.fix' (lib.extends overrides packages) def main(): editor = KakouneEditor("kakoune", ROOT, GET_PLUGINS) - parser = editor.create_parser() - args = parser.parse_args() - - pluginupdate.update_plugins(editor, args) + editor.run() if __name__ == "__main__": diff --git a/pkgs/applications/editors/vim/plugins/get-plugins.nix b/pkgs/applications/editors/vim/plugins/get-plugins.nix new file mode 100644 index 000000000000..0aaec333c2d7 --- /dev/null +++ b/pkgs/applications/editors/vim/plugins/get-plugins.nix @@ -0,0 +1,19 @@ +with import {}; +let + inherit (vimUtils.override {inherit vim;}) buildVimPluginFrom2Nix; + inherit (neovimUtils) buildNeovimPluginFrom2Nix; + + generated = callPackage { + inherit buildNeovimPluginFrom2Nix buildVimPluginFrom2Nix; + } {} {}; + hasChecksum = value: + lib.isAttrs value && lib.hasAttrByPath ["src" "outputHash"] value; + getChecksum = name: value: + if hasChecksum value then { + submodules = value.src.fetchSubmodules or false; + sha256 = value.src.outputHash; + rev = value.src.rev; + } else null; + checksums = lib.mapAttrs getChecksum generated; +in + lib.filterAttrs (n: v: v != null) checksums diff --git a/pkgs/applications/editors/vim/plugins/update.py b/pkgs/applications/editors/vim/plugins/update.py index 20336c66ed6e..09606634f9e5 100755 --- a/pkgs/applications/editors/vim/plugins/update.py +++ b/pkgs/applications/editors/vim/plugins/update.py @@ -42,28 +42,14 @@ sys.path.insert(0, os.path.join(ROOT.parent.parent.parent.parent.parent, "mainta import pluginupdate from pluginupdate import run_nix_expr, PluginDesc -GET_PLUGINS = f"""(with import {{}}; -let - inherit (vimUtils.override {{inherit vim;}}) buildNeovimPluginFrom2Nix buildVimPluginFrom2Nix; - generated = callPackage {ROOT}/generated.nix {{ - inherit buildNeovimPluginFrom2Nix buildVimPluginFrom2Nix; - }}; - hasChecksum = value: lib.isAttrs value && lib.hasAttrByPath ["src" "outputHash"] value; - getChecksum = name: value: - if hasChecksum value then {{ - submodules = value.src.fetchSubmodules or false; - sha256 = value.src.outputHash; - rev = value.src.rev; - }} else null; - checksums = lib.mapAttrs getChecksum generated; -in lib.filterAttrs (n: v: v != null) checksums)""" + GET_PLUGINS_LUA = """ with import {}; lib.attrNames lua51Packages""" HEADER = ( - "# This file has been generated by ./pkgs/applications/editors/vim/plugins/update.py. Do not edit!" + "# GENERATED by ./pkgs/applications/editors/vim/plugins/update.py. Do not edit!" ) def isNeovimPlugin(plug: pluginupdate.Plugin) -> bool: @@ -118,34 +104,39 @@ class VimEditor(pluginupdate.Editor): """.format( buildFn="buildNeovimPluginFrom2Nix" if isNeovim else "buildVimPluginFrom2Nix", plugin=plugin, src_nix=src_nix, repo=repo) - print(content) + log.debug(content) return content + + def update(self, args): + pluginupdate.update_plugins(self, args) + + if self.nvim_treesitter_updated: + print("updating nvim-treesitter grammars") + nvim_treesitter_dir = ROOT.joinpath("nvim-treesitter") + subprocess.check_call([nvim_treesitter_dir.joinpath("update.py")]) + + if self.nixpkgs_repo: + index = self.nixpkgs_repo.index + for diff in index.diff(None): + if diff.a_path == "pkgs/applications/editors/vim/plugins/nvim-treesitter/generated.nix": + msg = "vimPlugins.nvim-treesitter: update grammars" + print(f"committing to nixpkgs: {msg}") + index.add([str(nvim_treesitter_dir.joinpath("generated.nix"))]) + index.commit(msg) + return + print("no updates to nvim-treesitter grammars") + + def main(): global luaPlugins luaPlugins = run_nix_expr(GET_PLUGINS_LUA) + with open(f"{ROOT}/get-plugins.nix") as f: + GET_PLUGINS = f.read() editor = VimEditor("vim", ROOT, GET_PLUGINS) - parser = editor.create_parser() - args = parser.parse_args() - pluginupdate.update_plugins(editor, args) - - if editor.nvim_treesitter_updated: - print("updating nvim-treesitter grammars") - nvim_treesitter_dir = ROOT.joinpath("nvim-treesitter") - subprocess.check_call([nvim_treesitter_dir.joinpath("update.py")]) - - if editor.nixpkgs_repo: - index = editor.nixpkgs_repo.index - for diff in index.diff(None): - if diff.a_path == "pkgs/applications/editors/vim/plugins/nvim-treesitter/generated.nix": - msg = "vimPlugins.nvim-treesitter: update grammars" - print(f"committing to nixpkgs: {msg}") - index.add([str(nvim_treesitter_dir.joinpath("generated.nix"))]) - index.commit(msg) - return - print("no updates to nvim-treesitter grammars") + editor.run() if __name__ == "__main__":