gnome: Package all the GNOME extensions

An automatic way to do this that scales up and requires little manual intervention is really needed. It works by scraping extensions.gnome.org with a python script, that writes all relevant information into the `extensions.json`. Every attribute of besaid file can be built into a package using `buildShellExtension`.

Extensions are grouped by GNOME Shell version for practical reasons. Only extensions for GNOME 40 and 3.38 were added, as we don't support legacy GNOME versions.
The extensions are exposed as an attrset, `pkgs.gnome40Extensions` and `pkgs.gnome38Extensions` respectively. The package name of each extensions is generated automatically from its UUID.

The attribute `pkgs.gnomeExtensions` contains the officially packaged and supported extensions set. It contains all the automatically packaged extensions for the current GNOME Shell version, which are overwritten by manually packaged ones where needed. Unlike gnomeXYExtensions, the names are not UUIDs, but automatically generated human-friendly names. Naming collisions – which are tracked in collisions.json – need to be manually resolved in the `extensionRenames` attrset.
This commit is contained in:
piegames 2021-05-10 23:01:02 +02:00 committed by Jan Tojnar
parent 1085727d81
commit b642ac7b70
No known key found for this signature in database
GPG Key ID: 7FAB2A15F7A607A4
9 changed files with 1002 additions and 45 deletions

@ -0,0 +1,26 @@
# GNOME Shell extensions
All extensions are packaged automatically. They can be found in the `pkgs.gnomeXYExtensions` for XY being a GNOME version. The package names are the extensions UUID, which can be a bit unwieldy to use. `pkgs.gnomeExtensions` is a set of manually curated extensions that match the current `gnome.gnome-shell` versions. Their name is human-friendly, compared to the other extensions sets. Some of its extensions are manually packaged.
## Automatically packaged extensions
The actual packages are created by `buildGnomeExtensions.nix`, provided the correct arguments are fed into it. The important extension data is stored in `extensions.json`, one line/item per extension. That file is generated by running `update-extensions.py`. Furthermore, the automatic generated names are dumped in `collisions.json` for manual inspection. `extensionRenames.nix` contains provides new names for all extensions that collide.
### Extensions updates
For everyday updates,
1. Run `update-extensions.py`.
2. Update `extensionRenames.nix` according to the comment at the top.
For GNOME updates,
1. Add a new `gnomeXYExtensions` set
2. Remove old ones for GNOME versions we dont want to support any more
3. Update `supported_versions` in `./update-extensions.py` and re-run it
4. Change `gnomeExtensions` to the new version
5. Update `./extensionsRenames.nix` accordingly
## Manually packaged extensions
Manually packaged extensions overwrite some of the automatically packaged ones in `pkgs.gnomeExtensions`. They are listed in `manuallyPackaged.nix`, every extension has its own sub-folder.

@ -0,0 +1,54 @@
{ pkgs, lib, stdenv, fetchzip }:
let
buildGnomeExtension = {
# Every gnome extension has a UUID. It's the name of the extension folder once unpacked
# and can always be found in the metadata.json of every extension.
uuid,
name,
pname,
description,
# extensions.gnome.org extension URL
link,
# Extension version numbers are integers
version,
sha256,
# Hex-encoded string of JSON bytes
metadata,
}:
stdenv.mkDerivation {
inherit pname;
version = builtins.toString version;
src = fetchzip {
url = "https://extensions.gnome.org/extension-data/${
builtins.replaceStrings [ "@" ] [ "" ] uuid
}.v${builtins.toString version}.shell-extension.zip";
inherit sha256;
stripRoot = false;
# The download URL may change content over time. This is because the
# metadata.json is automatically generated, and parts of it can be changed
# without making a new release. We simply substitute the possibly changed fields
# with their content from when we last updated, and thus get a deterministic output
# hash.
extraPostFetch = ''
echo "${metadata}" | base64 --decode > $out/metadata.json
'';
};
buildCommand = ''
mkdir -p $out/share/gnome-shell/extensions/
cp -r -T $src $out/share/gnome-shell/extensions/${uuid}
'';
meta = {
description = builtins.head (lib.splitString "\n" description);
longDescription = description;
homepage = link;
license = lib.licenses.gpl2Plus; # https://wiki.gnome.org/Projects/GnomeShell/Extensions/Review#Licensing
maintainers = with lib.maintainers; [ piegames ];
};
# Store the extension's UUID, because we might need it at some places
passthru.extensionUuid = uuid;
};
in
lib.makeOverridable buildGnomeExtension

@ -0,0 +1,42 @@
{
"38": {
"applications-menu": [
"apps-menu@gnome-shell-extensions.gcampax.github.com",
"Applications_Menu@rmy.pobox.com"
],
"workspace-indicator": [
"workspace-indicator@gnome-shell-extensions.gcampax.github.com",
"horizontal-workspace-indicator@tty2.io"
],
"lock-keys": [
"lockkeys@vaina.lt",
"lockkeys@fawtytoo"
],
"fuzzy-clock": [
"Fuzzy_Clock@dallagi",
"fuzzy-clock@keepawayfromfire.co.uk"
],
"transparent-window": [
"transparent-window@pbxqdown.github.com",
"transparentwindows.mdirshad07"
],
"floating-dock": [
"floatingDock@sun.wxg@gmail.com",
"floating-dock@nandoferreira_prof@hotmail.com"
]
},
"40": {
"applications-menu": [
"apps-menu@gnome-shell-extensions.gcampax.github.com",
"Applications_Menu@rmy.pobox.com"
],
"workspace-indicator": [
"workspace-indicator@gnome-shell-extensions.gcampax.github.com",
"horizontal-workspace-indicator@tty2.io"
],
"lock-keys": [
"lockkeys@vaina.lt",
"lockkeys@fawtytoo"
]
}
}

@ -0,0 +1,73 @@
{ lib
, callPackage
, config
}:
let
buildShellExtension = callPackage ./buildGnomeExtension.nix { };
# Index of all scraped extensions (with supported versions)
extensionsIndex = lib.importJSON ./extensions.json;
# A list of UUIDs that have the same pname and we need to rename them
extensionRenames = import ./extensionRenames.nix;
# Take all extensions from the index that match the gnome version, build them and put them into a list of derivations
produceExtensionsList = shell-version:
lib.trivial.pipe extensionsIndex [
# Does a given extension match our current shell version?
(builtins.filter
(extension: (builtins.hasAttr shell-version extension."shell_version_map"))
)
# Take in an `extension` object from the JSON and transform it into the correct args to call `buildShellExtension`
(map
(extension: {
inherit (extension) uuid name description link pname;
inherit (extension.shell_version_map.${shell-version}) version sha256 metadata;
})
)
# Build them
(map buildShellExtension)
];
# Map the list of extensions to an attrset based on the UUID as key
mapUuidNames = extensions:
lib.trivial.pipe extensions [
(map (extension: lib.nameValuePair extension.extensionUuid extension))
builtins.listToAttrs
];
# Map the list of extensions to an attrset based on the pname as key, which is more human readable than the UUID
# We also take care of conflict renaming in here
mapReadableNames = extensionsList: lib.trivial.pipe extensionsList [
# Filter out all extensions that map to null
(lib.filter (extension:
!(
(builtins.hasAttr extension.extensionUuid extensionRenames)
&& ((builtins.getAttr extension.extensionUuid extensionRenames) == null)
)
))
# Map all extensions to their pname, with potential overwrites
(map (extension:
lib.nameValuePair (extensionRenames.${extension.extensionUuid} or extension.pname) extension
))
builtins.listToAttrs
];
in rec {
inherit buildShellExtension;
gnome38Extensions = mapUuidNames (produceExtensionsList "38");
gnome40Extensions = mapUuidNames (produceExtensionsList "40");
gnomeExtensions = lib.recurseIntoAttrs (
(mapReadableNames (produceExtensionsList "40"))
// (callPackage ./manuallyPackaged.nix {})
// lib.optionalAttrs (config.allowAliases or false) {
unite-shell = gnomeExtensions.unite; # added 2021-01-19
arc-menu = gnomeExtensions.arcmenu; # added 2021-02-14
nohotcorner = throw "gnomeExtensions.nohotcorner removed since 2019-10-09: Since 3.34, it is a part of GNOME Shell configurable through GNOME Tweaks.";
mediaplayer = throw "gnomeExtensions.mediaplayer deprecated since 2019-09-23: retired upstream https://github.com/JasonLG1979/gnome-shell-extensions-mediaplayer/blob/master/README.md";
}
);
}

@ -0,0 +1,29 @@
# A list of UUIDs that have the same pname and we need to rename them
# MAINTENANCE:
# - Every item from ./collisions.json (for the respective Shell version) should have an entry in here
# - Set the value to `null` for filtering (duplicate or unmaintained extensions)
# - Sort the entries in order of appearance in the collisions.json
{
"apps-menu@gnome-shell-extensions.gcampax.github.com" = "applications-menu";
"Applications_Menu@rmy.pobox.com" = "frippery-applications-menu";
"workspace-indicator@gnome-shell-extensions.gcampax.github.com" = "workspace-indicator";
"horizontal-workspace-indicator@tty2.io" = "workspace-indicator-2";
"lockkeys@vaina.lt" = "lock-keys";
"lockkeys@fawtytoo" = "lock-keys-2";
# These are conflicts for 3.38 extensions. They will very probably come back
# once more of them support 40.
# See https://github.com/pbxqdown/gnome-shell-extension-transparent-window/issues/12#issuecomment-800765381
#"transparent-window@pbxqdown.github.com" = "transparent-window";
#"transparentwindows.mdirshad07" = null;
#"floatingDock@sun.wxg@gmail.com" = "floating-dock";
#"floating-dock@nandoferreira_prof@hotmail.com" = "floating-dock-2";
# That extension is broken because of https://github.com/NixOS/nixpkgs/issues/118612
#"flypie@schneegans.github.com" = null;
}

File diff suppressed because one or more lines are too long

@ -0,0 +1,40 @@
{ callPackage }:
{
appindicator = callPackage ./appindicator { };
arcmenu = callPackage ./arcmenu { };
caffeine = callPackage ./caffeine { };
clipboard-indicator = callPackage ./clipboard-indicator { };
clock-override = callPackage ./clock-override { };
dash-to-dock = callPackage ./dash-to-dock { };
dash-to-panel = callPackage ./dash-to-panel { };
disable-unredirect = callPackage ./disable-unredirect { };
draw-on-your-screen = callPackage ./draw-on-your-screen { };
drop-down-terminal = callPackage ./drop-down-terminal { };
dynamic-panel-transparency = callPackage ./dynamic-panel-transparency { };
easyScreenCast = callPackage ./EasyScreenCast { };
emoji-selector = callPackage ./emoji-selector { };
freon = callPackage ./freon { };
fuzzy-app-search = callPackage ./fuzzy-app-search { };
gsconnect = callPackage ./gsconnect { };
hot-edge = callPackage ./hot-edge { };
icon-hider = callPackage ./icon-hider { };
impatience = callPackage ./impatience { };
material-shell = callPackage ./material-shell { };
mpris-indicator-button = callPackage ./mpris-indicator-button { };
night-theme-switcher = callPackage ./night-theme-switcher { };
no-title-bar = callPackage ./no-title-bar { };
noannoyance = callPackage ./noannoyance { };
paperwm = callPackage ./paperwm { };
pidgin-im-integration = callPackage ./pidgin-im-integration { };
remove-dropdown-arrows = callPackage ./remove-dropdown-arrows { };
sound-output-device-chooser = callPackage ./sound-output-device-chooser { };
system-monitor = callPackage ./system-monitor { };
taskwhisperer = callPackage ./taskwhisperer { };
tilingnome = callPackage ./tilingnome { };
timepp = callPackage ./timepp { };
topicons-plus = callPackage ./topicons-plus { };
unite = callPackage ./unite { };
window-corner-preview = callPackage ./window-corner-preview { };
window-is-ready-remover = callPackage ./window-is-ready-remover { };
workspace-matrix = callPackage ./workspace-matrix { };
}

@ -0,0 +1,284 @@
#!/usr/bin/env nix-shell
#!nix-shell -I nixpkgs=../../../.. -i python3 -p python3
import json
import urllib.request
import urllib.error
from typing import List, Dict, Optional, Any, Tuple
import logging
from operator import itemgetter
import subprocess
import zipfile
import io
import base64
# We don't want all those deprecated legacy extensions
# Group extensions by GNOME "major" version for compatibility reasons
supported_versions = {
"38": "3.38",
"40": "40",
}
# Some type alias to increase readility of complex compound types
PackageName = str
ShellVersion = str
Uuid = str
ExtensionVersion = int
# Keep track of all names that have been used till now to detect collisions.
# This works because we deterministically process all extensions in historical order
# The outer dict level is the shell version, as we are tracking duplicates only per same Shell version.
# key: shell version, value: Dict with key: pname, value: list of UUIDs with that pname
package_name_registry: Dict[ShellVersion, Dict[PackageName, List[Uuid]]] = {}
for shell_version in supported_versions.keys():
package_name_registry[shell_version] = {}
def fetch_extension_data(uuid: str, version: str) -> Tuple[str, str]:
"""
Download the extension and hash it. We use `nix-prefetch-url` for this for efficiency reasons.
Returns a tuple with the hash (Nix-compatible) of the zip file's content and the base64-encoded content of its metadata.json.
"""
# The download URLs follow this schema
uuid = uuid.replace("@", "")
url: str = f"https://extensions.gnome.org/extension-data/{uuid}.v{version}.shell-extension.zip"
# Yes, we download that file three times:
# The first time is for the maintainter, so they may have a personal backup to fix potential issues
# subprocess.run(
# ["wget", url], capture_output=True, text=True
# )
# The second time, we extract the metadata.json because we need it too
with urllib.request.urlopen(url) as response:
data = zipfile.ZipFile(io.BytesIO(response.read()), 'r')
metadata = base64.b64encode(data.read('metadata.json')).decode()
# The third time is to get the file into the store and to get its hash
hash = subprocess.run(
["nix-prefetch-url", "--unpack", url], capture_output=True, text=True
).stdout.strip()
return hash, metadata
def generate_extension_versions(
extension_version_map: Dict[ShellVersion, ExtensionVersion], uuid: str
) -> Dict[ShellVersion, Dict[str, str]]:
"""
Takes in a mapping from shell versions to extension versions and transforms it the way we need it:
- Only take one extension version per GNOME Shell major version (as per `supported_versions`)
- Filter out versions that only support old GNOME versions
- Download the extension and hash it
"""
extension_versions: Dict[ShellVersion, Dict[str, str]] = {}
for shell_version, version_prefix in supported_versions.items():
# Newest compatible extension version
extension_version: Optional[int] = max(
(
int(ext_ver)
for shell_ver, ext_ver in extension_version_map.items()
if (shell_ver.startswith(version_prefix))
),
default=None,
)
# Extension is not compatible with this GNOME version
if not extension_version:
continue
logging.debug(
f"[{shell_version}] Downloading '{uuid}' v{extension_version}"
)
sha256, metadata = fetch_extension_data(uuid, str(extension_version))
extension_versions[shell_version] = {
"version": str(extension_version),
"sha256": sha256,
# The downloads are impure, their metadata.json may change at any time.
# Thus, be back it up / pin it to remain deterministic
# Upstream issue: https://gitlab.gnome.org/Infrastructure/extensions-web/-/issues/137
"metadata": metadata,
}
return extension_versions
def pname_from_url(url: str) -> Tuple[str, str]:
"""
Parse something like "/extension/1475/battery-time/" and output ("battery-time", "1475")
"""
url = url.split("/") # type: ignore
return (url[3], url[2])
def process_extension(extension: Dict[str, Any]) -> Optional[Dict[str, Any]]:
"""
Process an extension. It takes in raw scraped data and downloads all the necessary information that buildGnomeExtension.nix requires
Input: a json object of one extension queried from the site. It has the following schema (only important key listed):
{
"uuid": str,
"name": str,
"description": str,
"link": str,
"shell_version_map": {
str: { "version": int, },
},
}
"uuid" is an extension UUID that looks like this (most of the time): "extension-name@username.domain.tld".
Don't make any assumptions on it, and treat it like an opaque string!
"link" follows the following schema: "/extension/$number/$string/"
The number is monotonically increasing and unique to every extension.
The string is usually derived from the extensions's name (but shortened, kebab-cased and URL friendly).
It may diverge from the actual name.
The keys of "shell_version_map" are GNOME Shell version numbers.
Output: a json object to be stored, or None if the extension should be skipped. Schema:
{
"uuid": str,
"name": str,
"pname": str,
"description": str,
"link": str,
"shell_version_map": {
str: { "version": int, "sha256": str, "metadata": <hex> },
}
}
Only "uuid" gets passed along unmodified. "name", "description" and "link" are taken from the input, but sanitized.
"pname" gets generated from other fields and "shell_version_map" has a completely different structure than the input
field with the same name.
"""
uuid = extension["uuid"]
# Yeah, there are some extensions without any releases
if not extension["shell_version_map"]:
return None
logging.info(f"Processing '{uuid}'")
# Input is a mapping str -> { version: int, … }
# We want to map shell versions to extension versions
shell_version_map: Dict[ShellVersion, int] = {
k: v["version"] for k, v in extension["shell_version_map"].items()
}
# Transform shell_version_map to be more useful for us. Also throw away unwanted versions
shell_version_map: Dict[ShellVersion, Dict[str, str]] = generate_extension_versions(shell_version_map, uuid) # type: ignore
# No compatible versions found
if not shell_version_map:
return None
# Fetch a human-readable name for the package.
(pname, _pname_id) = pname_from_url(extension["link"])
for shell_version in shell_version_map.keys():
if pname in package_name_registry[shell_version]:
logging.warning(f"Package name '{pname}' is colliding.")
package_name_registry[shell_version][pname].append(uuid)
else:
package_name_registry[shell_version][pname] = [uuid]
return {
"uuid": uuid,
"name": extension["name"],
"pname": pname,
"description": extension["description"],
"link": "https://extensions.gnome.org" + extension["link"],
"shell_version_map": shell_version_map,
}
def scrape_extensions_index() -> List[Dict[str, Any]]:
"""
Scrape the list of extensions by sending search queries to the API. We simply go over it
page by page until we hit a non-full page or a 404 error.
The returned list is sorted by the age of the extension, in order to be deterministic.
"""
page = 0
extensions = []
while True:
page += 1
logging.info("Scraping page " + str(page))
try:
with urllib.request.urlopen(
f"https://extensions.gnome.org/extension-query/?n_per_page=25&page={page}"
) as response:
data = json.loads(response.read().decode())["extensions"]
responseLength = len(data)
for extension in data:
extensions.append(extension)
# If our page isn't "full", it must have been the last one
if responseLength < 25:
logging.debug(
f"\tThis page only has {responseLength} entries, so it must be the last one."
)
break
except urllib.error.HTTPError as e:
if e.code == 404:
# We reached past the last page and are done now
break
else:
raise
# `pk` is the primary key in the extensions.gnome.org database. Sorting on it will give us a stable,
# deterministic ordering.
extensions.sort(key=itemgetter("pk"))
return extensions
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
raw_extensions = scrape_extensions_index()
logging.info(f"Downloaded {len(raw_extensions)} extensions. Processing …")
processed_extensions: List[Dict[str, Any]] = []
for num, raw_extension in enumerate(raw_extensions):
processed_extension = process_extension(raw_extension)
if processed_extension:
processed_extensions.append(processed_extension)
logging.debug(f"Processed {num + 1} / {len(raw_extensions)}")
logging.info(
f"Done. Writing results to extensions.json ({len(processed_extensions)} extensions in total)"
)
with open("extensions.json", "w") as out:
# Manually pretty-print the outer level, but then do one compact line per extension
# This allows for the diffs to be manageable (one line of change per extension) despite their quantity
for index, extension in enumerate(processed_extensions):
if index == 0:
out.write("[ ")
else:
out.write(", ")
json.dump(extension, out, ensure_ascii=False)
out.write("\n")
out.write("]\n")
with open("extensions.json", "r") as out:
# Check that the generated file actually is valid JSON, just to be sure
json.load(out)
logging.info(
"Done. Writing name collisions to collisions.json (please check manually)"
)
with open("collisions.json", "w") as out:
# Filter out those that are not duplicates
package_name_registry_filtered: Dict[ShellVersion, Dict[PackageName, List[Uuid]]] = {
# The outer level keys are shell versions
shell_version: {
# The inner keys are extension names, with a list of all extensions with that name as value.
pname: extensions for pname, extensions in collisions.items() if len(extensions) > 1
} for shell_version, collisions in package_name_registry.items()
}
json.dump(package_name_registry_filtered, out, indent=2, ensure_ascii=False)
out.write("\n")

@ -28844,51 +28844,10 @@ in
gnome = recurseIntoAttrs (callPackage ../desktops/gnome { });
gnomeExtensions = recurseIntoAttrs {
appindicator = callPackage ../desktops/gnome/extensions/appindicator { };
arcmenu = callPackage ../desktops/gnome/extensions/arcmenu { };
caffeine = callPackage ../desktops/gnome/extensions/caffeine { };
clipboard-indicator = callPackage ../desktops/gnome/extensions/clipboard-indicator { };
clock-override = callPackage ../desktops/gnome/extensions/clock-override { };
dash-to-dock = callPackage ../desktops/gnome/extensions/dash-to-dock { };
dash-to-panel = callPackage ../desktops/gnome/extensions/dash-to-panel { };
disable-unredirect = callPackage ../desktops/gnome/extensions/disable-unredirect { };
draw-on-your-screen = callPackage ../desktops/gnome/extensions/draw-on-your-screen { };
drop-down-terminal = callPackage ../desktops/gnome/extensions/drop-down-terminal { };
dynamic-panel-transparency = callPackage ../desktops/gnome/extensions/dynamic-panel-transparency { };
easyScreenCast = callPackage ../desktops/gnome/extensions/EasyScreenCast { };
emoji-selector = callPackage ../desktops/gnome/extensions/emoji-selector { };
freon = callPackage ../desktops/gnome/extensions/freon { };
fuzzy-app-search = callPackage ../desktops/gnome/extensions/fuzzy-app-search { };
gsconnect = callPackage ../desktops/gnome/extensions/gsconnect { };
hot-edge = callPackage ../desktops/gnome/extensions/hot-edge { };
icon-hider = callPackage ../desktops/gnome/extensions/icon-hider { };
impatience = callPackage ../desktops/gnome/extensions/impatience { };
material-shell = callPackage ../desktops/gnome/extensions/material-shell { };
mpris-indicator-button = callPackage ../desktops/gnome/extensions/mpris-indicator-button { };
night-theme-switcher = callPackage ../desktops/gnome/extensions/night-theme-switcher { };
no-title-bar = callPackage ../desktops/gnome/extensions/no-title-bar { };
noannoyance = callPackage ../desktops/gnome/extensions/noannoyance { };
paperwm = callPackage ../desktops/gnome/extensions/paperwm { };
pidgin-im-integration = callPackage ../desktops/gnome/extensions/pidgin-im-integration { };
remove-dropdown-arrows = callPackage ../desktops/gnome/extensions/remove-dropdown-arrows { };
sound-output-device-chooser = callPackage ../desktops/gnome/extensions/sound-output-device-chooser { };
system-monitor = callPackage ../desktops/gnome/extensions/system-monitor { };
taskwhisperer = callPackage ../desktops/gnome/extensions/taskwhisperer { };
tilingnome = callPackage ../desktops/gnome/extensions/tilingnome { };
timepp = callPackage ../desktops/gnome/extensions/timepp { };
topicons-plus = callPackage ../desktops/gnome/extensions/topicons-plus { };
unite = callPackage ../desktops/gnome/extensions/unite { };
window-corner-preview = callPackage ../desktops/gnome/extensions/window-corner-preview { };
window-is-ready-remover = callPackage ../desktops/gnome/extensions/window-is-ready-remover { };
workspace-matrix = callPackage ../desktops/gnome/extensions/workspace-matrix { };
} // lib.optionalAttrs (config.allowAliases or false) {
nohotcorner = throw "gnomeExtensions.nohotcorner removed since 2019-10-09: Since 3.34, it is a part of GNOME Shell configurable through GNOME Tweaks.";
mediaplayer = throw "gnomeExtensions.mediaplayer deprecated since 2019-09-23: retired upstream https://github.com/JasonLG1979/gnome-shell-extensions-mediaplayer/blob/master/README.md";
unite-shell = gnomeExtensions.unite; # added 2021-01-19
arc-menu = gnomeExtensions.arcmenu; # added 2021-02-14
};
inherit (callPackage ../desktops/gnome/extensions { })
gnomeExtensions
gnome38Extensions
gnome40Extensions;
gnome-connections = callPackage ../desktops/gnome/apps/gnome-connections { };