Merge pull request #41080 from dotlambda/home-assistant-recursive-reqs
home-assistant: include requirements of dependencies
This commit is contained in:
Peter Hoeg
GitHub
commit
b94e1f1fbf
File diff suppressed because it is too large
Load Diff
@ -1,28 +1,30 @@
|
||||
#! /usr/bin/env nix-shell
|
||||
#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ ])"
|
||||
#! nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ requests pyyaml pytz pip jinja2 voluptuous typing aiohttp async-timeout astral certifi attrs ])"
|
||||
#
|
||||
# This script downloads https://github.com/home-assistant/home-assistant/blob/master/requirements_all.txt.
|
||||
# This file contains lines of the form
|
||||
# This script downloads Home Assistant's source tarball.
|
||||
# Inside the homeassistant/components directory, each component has an associated .py file,
|
||||
# specifying required packages and other components it depends on:
|
||||
#
|
||||
# # homeassistant.components.foo
|
||||
# # homeassistant.components.bar
|
||||
# foobar==1.2.3
|
||||
# REQUIREMENTS = [ 'package==1.2.3' ]
|
||||
# DEPENDENCIES = [ 'component' ]
|
||||
#
|
||||
# i.e. it lists dependencies and the components that require them.
|
||||
# By parsing the file, a dictionary mapping component to dependencies is created.
|
||||
# For all of these dependencies, Nixpkgs' python3Packages are searched for appropriate names.
|
||||
# By parsing the files, a dictionary mapping component to requirements and dependencies is created.
|
||||
# For all of these requirements and the dependencies' requirements,
|
||||
# Nixpkgs' python3Packages are searched for appropriate names.
|
||||
# Then, a Nix attribute set mapping component name to dependencies is created.
|
||||
|
||||
from urllib.request import urlopen
|
||||
from collections import OrderedDict
|
||||
import tempfile
|
||||
from io import BytesIO
|
||||
import tarfile
|
||||
import importlib
|
||||
import subprocess
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
|
||||
GENERAL_PREFIX = '# homeassistant.'
|
||||
COMPONENT_PREFIX = GENERAL_PREFIX + 'components.'
|
||||
COMPONENT_PREFIX = 'homeassistant.components'
|
||||
PKG_SET = 'python3Packages'
|
||||
|
||||
# If some requirements are matched by multiple python packages,
|
||||
@ -37,28 +39,32 @@ def get_version():
|
||||
m = re.search('hassVersion = "([\\d\\.]+)";', f.read())
|
||||
return m.group(1)
|
||||
|
||||
def fetch_reqs(version='master'):
|
||||
requirements = {}
|
||||
with urlopen('https://github.com/home-assistant/home-assistant/raw/{}/requirements_all.txt'.format(version)) as response:
|
||||
components = []
|
||||
for line in response.read().decode().splitlines():
|
||||
if line == '':
|
||||
components = []
|
||||
elif line[:len(COMPONENT_PREFIX)] == COMPONENT_PREFIX:
|
||||
component = line[len(COMPONENT_PREFIX):]
|
||||
components.append(component)
|
||||
if component not in requirements:
|
||||
requirements[component] = []
|
||||
elif line[:len(GENERAL_PREFIX)] != GENERAL_PREFIX: # skip lines like "# homeassistant.scripts.xyz"
|
||||
# Some dependencies are commented out because they don't build on all platforms
|
||||
# Since they are still required for running the component, don't skip them
|
||||
if line[:2] == '# ':
|
||||
line = line[2:]
|
||||
# Some requirements are specified by url, e.g. https://example.org/foobar#xyz==1.0.0
|
||||
# Therefore, if there's a "#" in the line, only take the part after it
|
||||
line = line[line.find('#') + 1:]
|
||||
for component in components:
|
||||
requirements[component].append(line)
|
||||
def parse_components(version='master'):
|
||||
components = {}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with urlopen('https://github.com/home-assistant/home-assistant/archive/{}.tar.gz'.format(version)) as response:
|
||||
tarfile.open(fileobj=BytesIO(response.read())).extractall(tmp)
|
||||
# Use part of a script from the Home Assistant codebase
|
||||
sys.path.append(tmp + '/home-assistant-{}'.format(version))
|
||||
from script.gen_requirements_all import explore_module
|
||||
for package in explore_module(COMPONENT_PREFIX, True):
|
||||
# Remove 'homeassistant.components.' prefix
|
||||
component = package[len(COMPONENT_PREFIX + '.'):]
|
||||
try:
|
||||
module = importlib.import_module(package)
|
||||
components[component] = {}
|
||||
components[component]['requirements'] = getattr(module, 'REQUIREMENTS', [])
|
||||
components[component]['dependencies'] = getattr(module, 'DEPENDENCIES', [])
|
||||
# If there is an ImportError, the imported file is not the main file of the component
|
||||
except ImportError:
|
||||
continue
|
||||
return components
|
||||
|
||||
# Recursively get the requirements of a component and its dependencies
|
||||
def get_reqs(components, component):
|
||||
requirements = set(components[component]['requirements'])
|
||||
for dependency in components[component]['dependencies']:
|
||||
requirements.update(get_reqs(components, dependency))
|
||||
return requirements
|
||||
|
||||
# Store a JSON dump of Nixpkgs' python3Packages
|
||||
@ -95,11 +101,14 @@ def name_to_attr_path(req):
|
||||
|
||||
version = get_version()
|
||||
print('Generating component-packages.nix for version {}'.format(version))
|
||||
requirements = fetch_reqs(version=version)
|
||||
components = parse_components(version=version)
|
||||
build_inputs = {}
|
||||
for component, reqs in OrderedDict(sorted(requirements.items())).items():
|
||||
for component in sorted(components.keys()):
|
||||
attr_paths = []
|
||||
for req in reqs:
|
||||
for req in sorted(get_reqs(components, component)):
|
||||
# Some requirements are specified by url, e.g. https://example.org/foobar#xyz==1.0.0
|
||||
# Therefore, if there's a "#" in the line, only take the part after it
|
||||
req = req[req.find('#') + 1:]
|
||||
name = req.split('==')[0]
|
||||
attr_path = name_to_attr_path(name)
|
||||
if attr_path is not None:
|
||||
@ -108,11 +117,8 @@ for component, reqs in OrderedDict(sorted(requirements.items())).items():
|
||||
else:
|
||||
build_inputs[component] = attr_paths
|
||||
|
||||
# Only select components which have any dependency
|
||||
#build_inputs = {k: v for k, v in build_inputs.items() if len(v) > 0}
|
||||
|
||||
with open(os.path.dirname(sys.argv[0]) + '/component-packages.nix', 'w') as f:
|
||||
f.write('# Generated from parse-requirements.py\n')
|
||||
f.write('# Generated by parse-requirements.py\n')
|
||||
f.write('# Do not edit!\n\n')
|
||||
f.write('{\n')
|
||||
f.write(' version = "{}";\n'.format(version))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user