Nix shell updates for develop (#18131)

* shell.nix: Update `tomlkit` to 0.11.4 using a Nixpkgs overlay

The used Nixpkgs snapshot contains `tomlkit` version 0.7.0, which is
affected by https://www.github.com/sdispater/tomlkit/issues/148; that
bug is triggered by `pyproject.toml` from `jsonschema` >= 4.11.0,
preventing the build of that module.

Just adding `tomlkit = "*"` to the `[tool.poetry.dev-dependencies]`
section of `nix/pyproject.toml` does not fix the `jsonschema` build,
because `makeRemoveSpecialDependenciesHook` inside `poetry2nix` is not
affected by `nix/pyproject.toml`.  Add a Nixpkgs overlay which updates
the `tomlkit` Python module globally, so that `poetry2nix` would also
use the updated version internally.

* shell.nix: Bump `poetry2nix` to the most recent version

The new `poetry2nix` version includes overrides which are required for
recent versions of some Python packages (in particular, `jsonschema` and
`dotty-dict`).

* shell.nix: Bump QMK CLI to 1.1.1; update other Python deps

Update `pyproject.toml` to match `requirements*.txt`:

- add `pyserial = "*"`
- replace `qmk-dotty-dict = "*"` with `dotty-dict = "*"` (#18117, also
  required for compatibility with `qmk` 1.1.1, where this replacement
  had already been performed)

Add build dependencies of various Python modules to `pyproject.toml`:

- `hatchling`, `hatch-vcs`, `hatch-fancy-pypi-readme` (required by
  `jsonschema` >= 4.11.0)
- `pytest` (a newer version is required to solve the dependency conflict
  with the `hatchling` module due to the upper bound on `pluggy`)
- `flit-core` (a more recent version is required to build `tomli`)
- `poetry-core` (required for `dotty-dict` >= 1.3.1, and the version
  from Nixpkgs does not build on Darwin due to NixOS/nix#4758)

Update `poetry.lock` to use the most recent versions of Python modules.

The complete list of Python module updates as listed in `poetry.lock`
(note that other modules might be present in the Python environment,
e.g., if they come from Nixpkgs):

- `atomicwrites`: none -> 1.4.1 (but this module is not actually used,
  because the corresponding dependency of `pytest` is win32-only)
- `attrs`: 21.4.0 -> 22.1.0
- `colorama`: 0.4.4 -> 0.4.5
- `coverage`: 6.4 -> none
- `dotty-dict`: none -> 1.3.1 (used instead of `qmk-dotty-dict`)
- `editables`: none -> 0.3
- `flake8`: 4.0.1 -> 5.0.4
- `flake8-polyfill`: 1.0.2 -> none
- `flit-core`: none -> 3.7.1
- `hatch-fancy-pypi-readme`: none -> 22.3.0
- `hatch-vcs`: none -> 0.2.0
- `hatchling`: none -> 1.8.0
- `hjson`: 3.0.2 -> 3.1.0
- `importlib-resources`: 5.7.1 -> 5.9.0
- `iniconfig`: none -> 1.1.1
- `jsonschema`: 4.5.1 -> 4.14.0
- `mccabe`: 0.6.1 -> 0.7.0
- `nose2`: 0.11.0 -> 0.12.0
- `packaging`: none -> 21.3
- `pathspec`: none -> 0.9.0
- `pep8-naming`: 0.12.1 -> 0.13.2
- `pillow`: 9.1.1 -> 9.2.0
- `pkgutil-resolve-name`: none -> 1.3.10
- `pluggy`: none -> 1.0.0
- `poetry-core`: none -> 1.0.8
- `py`: none -> 1.11.0
- `pycodestyle`: 2.8.0 -> 2.9.1
- `pyflakes`: 2.4.0 -> 2.5.0
- `pygments`: 2.12.0 -> 2.13.0
- `pyparsing`: none -> 3.0.9
- `pyserial`: none -> 3.5
- `pytest`: none -> 7.1.2
- `qmk`: 1.1.0 -> 1.1.1
- `qmk-dotty-dict`: 1.3.0.post1 -> none (replaced by `dotty-dict`)
- `setuptools-scm`: none -> 7.0.5
- `tomli`: none -> 2.0.1
- `typing-extensions`: none -> 4.3.0
- `zipp`: 3.8.0 -> 3.8.1

shell.nix

@@ -1,9 +1,29 @@
 let
   # We specify sources via Niv: use "niv update nixpkgs" to update nixpkgs, for example.
   sources = import ./util/nix/sources.nix { };
+
+  # `tomlkit` >= 0.8.0 is required to build `jsonschema` >= 4.11.0 (older
+  # version do not support some valid TOML syntax: sdispater/tomlkit#148).  The
+  # updated `tomlkit` must be used by `makeRemoveSpecialDependenciesHook`
+  # inside `poetry2nix`, therefore just providing the updated version through
+  # our `nix/pyproject.toml` does not work, and using an overlay is required.
+  pythonOverlay = final: prev: {
+    python3 = prev.python3.override {
+      packageOverrides = self: super: {
+        tomlkit = super.tomlkit.overridePythonAttrs(old: rec {
+          version = "0.11.4";
+          src = super.fetchPypi {
+            inherit (old) pname;
+            inherit version;
+            sha256 = "sha256-MjWpAQ+uVDI+cnw6wG+3IHUv5mNbNCbjedrsYPvUSoM=";
+          };
+        });
+      };
+    };
+  };
 in
 # However, if you want to override Niv's inputs, this will let you do that.
-{ pkgs ? import sources.nixpkgs { }
+{ pkgs ? import sources.nixpkgs { overlays = [ pythonOverlay ]; }
 , poetry2nix ? pkgs.callPackage (import sources.poetry2nix) { }
 , avr ? true
 , arm ? true

util/nix/pyproject.toml

@@ -12,13 +12,14 @@ python = "^3.8"
 appdirs = "*"
 argcomplete = "*"
 colorama = "*"
+dotty-dict = "*"
 hid = "*"
 hjson = "*"
 jsonschema = ">=4"
 milc = ">=1.4.2"
 Pygments = "*"
+pyserial = "*"
 pyusb = "*"
-qmk-dotty-dict = "*"
 pillow = "*"
 
 # This dependency is not mentioned in requirements.txt (QMK CLI is not a
@@ -33,6 +34,28 @@ pep8-naming = "*"
 pyflakes = "*"
 yapf = "*"
 
+# These dependencies are required by the jsonschema >= 4.11.0 build system, but
+# are not detected automatically; they are also not present in the used Nixpkgs
+# snapshot, so need to be obtained through Poetry.
+hatchling = "*"
+hatch-vcs = "*"
+hatch-fancy-pypi-readme = "*"
+
+# The `pytest` module in the used Nixpkgs snapshot has an upper bound on the
+# `pluggy` dependency, which conflicts with the dependency of the `hatchling`
+# module; upgrading the `pytest` module fixes the conflict.
+pytest = "*"
+
+# Building the `tomli` module, which is in the dependency tree of `hatchling`,
+# requires a newer `flit-core` module than found in the used Nixpkgs snapshot.
+flit-core = "*"
+
+# Building `dotty-dict` >= 1.3.1 requires the `poetry-core` module, and the
+# version of that module provided by the used Nixpkgs snapshot cannot be built
+# on Darwin due to the regex compatibility issue in the old Nixpkgs code
+# (https://github.com/NixOS/nix/issues/4758).
+poetry-core = "*"
+
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"

util/nix/sources.json

@@ -29,10 +29,10 @@
         "homepage": "",
         "owner": "nix-community",
         "repo": "poetry2nix",
-        "rev": "88ffae91c605aaafc2797f4096ca9f065152796a",
-        "sha256": "0iq9jlzz92r3ax1ymg00cn4s8c1wi3jgh1693abyyn0baq7gixrb",
+        "rev": "11c0df8e348c0f169cd73a2e3d63f65c92baf666",
+        "sha256": "0i3wbp2p0x6bpj07sqpvkbx4lvjm0irvpmv2bjqx8k02mpjm7dg2",
         "type": "tarball",
-        "url": "https://github.com/nix-community/poetry2nix/archive/88ffae91c605aaafc2797f4096ca9f065152796a.tar.gz",
+        "url": "https://github.com/nix-community/poetry2nix/archive/11c0df8e348c0f169cd73a2e3d63f65c92baf666.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }