vpp_config: Rework for Python2/3 compatibility.

On ubuntu:
   $cd <basedir>/extras/vpp_config
   $./scripts/clean.sh
   $./scripts/cp-data.sh
   $sudo apt-get install python3-pip python3-setuptools
   $python3 -m pip install .
   $vpp-config

Changes:
   * Convert to print() function.
   * raw_input changes.
   * floor division changes.
   * replace vpp-config.py with a setuptools 'vpp-config' entry_point.
   * replace netaddr with ipaddress from the standard library and backport.
   * .decode() subprocess.Popen's stdout because in python3 they are bytes.

Change-Id: Id98894ee54e0c31a0ba0304134b159caef415705
Signed-off-by: Paul Vinciguerra <pvinci@vinciconsulting.com>
This commit is contained in:
Paul Vinciguerra
2018-12-19 02:05:25 -08:00
committed by Dave Barach
parent 6ad7231c00
commit 339bc6b51d
12 changed files with 548 additions and 400 deletions

View File

@ -11,7 +11,7 @@ Use:
The installation and executing of the VPP configuration utility is simple. First
install the python pip module. Using pip install, then pip install vpp-config.
Then simply type Òvpp-configÓ and answer the questions. If you are not sure what
Then simply type <EFBFBD>vpp-config<69> and answer the questions. If you are not sure what
to answer choose the default. For yes or no questions the capital letter
designates the default. For example, for a question that shows [Y/n] Y is the
default. For numbers the default is within the brackets for example for a
@ -36,14 +36,15 @@ environment so you can start from scratch. These are the steps to run the utilit
in this environment. The scripts are meant to be run from the root directory.
./scripts/clean.sh
./scripts/cp-data.sh
./vpp_config.py
./scripts/cp-data.sh
./vpp-config
When the utility is installed with pip the wrapper scripts/vpp-config is written to
/usr/local/bin. However, the starting point when debugging this script locally is
./vpp_config.py. Run the utility by executing ./vpp_config.py from the root directory.
vpp-config. Run the utility by executing vpp-config.
The start point in the code is in vpp_config.py. However, most of the work is done in
The start point in the code is in vpp_config.py. However, most of the work is
done in
the files in ./vpplib
Uploading to PyPi:
@ -51,7 +52,7 @@ Uploading to PyPi:
To upload this utility to PpPi simple do the following. Currently, I have my own account
when we want everyone to contribute we will need to change that.
sudo ÐH bash
sudo <EFBFBD>H bash
cd vpp_config
python setup.py sdist bdist_wheel
twine upload dist/*
@ -137,7 +138,7 @@ There are no VPP packages on node localhost.
Do you want to install VPP [Y/n]?
INFO:root: Local Command: ls /etc/apt/sources.list.d/99fd.io.list.orig
INFO:root: /etc/apt/sources.list.d/99fd.io.list.orig
ÉÉ..
<EFBFBD><EFBFBD>..
What would you like to do?

View File

@ -1,4 +1,4 @@
#! /usr/bin/python
#! /usr/bin/env python
#
# BSD LICENSE
#
@ -647,5 +647,6 @@ def main():
get_crypto_details()
do_arg_actions()
if __name__ == "__main__":
main()

View File

@ -1,22 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2016 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""VPP Configuration Utility Wrapper"""
import os
import sys
import vpp_config as vpp
vpp.config_main()

View File

@ -1,25 +1,33 @@
from setuptools import setup
setup(name="vpp_config",
version="18.10.4",
version="19.01.1",
author="John DeNisco",
author_email="jdenisco@cisco.com",
description="VPP Configuration Utility",
license = 'Apache-2.0',
license='Apache-2.0',
keywords="vppconfig",
url = 'https://wiki.fd.io/view/VPP',
url='https://wiki.fd.io/view/VPP',
py_modules=['vpp_config'],
install_requires=['pyyaml','netaddr', 'requests'],
install_requires=['pyyaml', 'requests'],
extra_requires=["ipaddress; python_version < '3.3'"],
packages=['vpplib'],
scripts=['scripts/vpp-config'],
entry_points={
'console_scripts': ['vpp-config=vpp_config:config_main'],
},
data_files=[('vpp/vpp-config/scripts', ['scripts/dpdk-devbind.py']),
('vpp/vpp-config/configs', ['data/auto-config.yaml']),
('vpp/vpp-config/configs', ['data/cloud-config.iso']),
('vpp/vpp-config/configs', ['data/iperf-centos.xml.template']),
('vpp/vpp-config/configs', ['data/iperf-ubuntu.xml.template']),
('vpp/vpp-config/dryrun/sysctl.d', ['data/80-vpp.conf.template']),
('vpp/vpp-config/configs',
['data/iperf-centos.xml.template']),
('vpp/vpp-config/configs',
['data/iperf-ubuntu.xml.template']),
('vpp/vpp-config/dryrun/sysctl.d',
['data/80-vpp.conf.template']),
('vpp/vpp-config/dryrun/default', ['data/grub.template']),
('vpp/vpp-config/dryrun/vpp', ['data/startup.conf.template']),
],
long_description="The VPP configuration utility can be used to easily configure VPP.",
('vpp/vpp-config/dryrun/vpp',
['data/startup.conf.template']),
],
long_description="The VPP configuration utility can be used to "
"easily configure VPP.",
)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -12,7 +12,7 @@
# limitations under the License.
"""CPU utilities library."""
from __future__ import absolute_import, division
import re
from vpplib.VPPUtil import VPPUtil
@ -43,8 +43,8 @@ class CpuUtils(object):
@staticmethod
def is_smt_enabled(cpu_info):
"""Uses CPU mapping to find out if SMT is enabled or not. If SMT is
enabled, the L1d,L1i,L2,L3 setting is the same for two processors. These
two processors are two threads of one core.
enabled, the L1d,L1i,L2,L3 setting is the same for two processors.
These two processors are two threads of one core.
:param cpu_info: CPU info, the output of "lscpu -p".
:type cpu_info: list
@ -53,7 +53,7 @@ class CpuUtils(object):
"""
cpu_mems = [item[-4:] for item in cpu_info]
cpu_mems_len = len(cpu_mems) / CpuUtils.NR_OF_THREADS
cpu_mems_len = len(cpu_mems) // CpuUtils.NR_OF_THREADS
count = 0
for cpu_mem in cpu_mems[:cpu_mems_len]:
if cpu_mem in cpu_mems[cpu_mems_len:]:
@ -137,7 +137,7 @@ class CpuUtils(object):
if smt_enabled and not smt_used:
cpu_list_len = len(cpu_list)
cpu_list = cpu_list[:cpu_list_len / CpuUtils.NR_OF_THREADS]
cpu_list = cpu_list[:cpu_list_len // CpuUtils.NR_OF_THREADS]
return cpu_list
@ -171,8 +171,8 @@ class CpuUtils(object):
cpu_cnt = cpu_list_len - skip_cnt
if smt_used:
cpu_list_0 = cpu_list[:cpu_list_len / CpuUtils.NR_OF_THREADS]
cpu_list_1 = cpu_list[cpu_list_len / CpuUtils.NR_OF_THREADS:]
cpu_list_0 = cpu_list[:cpu_list_len // CpuUtils.NR_OF_THREADS]
cpu_list_1 = cpu_list[cpu_list_len // CpuUtils.NR_OF_THREADS:]
cpu_list = [cpu for cpu in cpu_list_0[skip_cnt:skip_cnt + cpu_cnt]]
cpu_list_ex = [cpu for cpu in
cpu_list_1[skip_cnt:skip_cnt + cpu_cnt]]
@ -236,8 +236,8 @@ class CpuUtils(object):
smt_used=smt_used)
if smt_used:
cpu_list_len = len(cpu_list)
cpu_list_0 = cpu_list[:cpu_list_len / CpuUtils.NR_OF_THREADS]
cpu_list_1 = cpu_list[cpu_list_len / CpuUtils.NR_OF_THREADS:]
cpu_list_0 = cpu_list[:cpu_list_len // CpuUtils.NR_OF_THREADS]
cpu_list_1 = cpu_list[cpu_list_len // CpuUtils.NR_OF_THREADS:]
cpu_range = "{}{}{},{}{}{}".format(cpu_list_0[0], sep,
cpu_list_0[-1],
cpu_list_1[0], sep,

View File

@ -12,6 +12,7 @@
# limitations under the License.
"""QEMU utilities library."""
from __future__ import absolute_import, division
from time import time, sleep
import json
@ -100,8 +101,9 @@ class QemuUtils(object):
:type threads: int
:type sockets: int
"""
self._qemu_opt['smp'] = '-smp {},cores={},threads={},sockets={}'.format(
cpus, cores, threads, sockets)
self._qemu_opt['smp'] = \
'-smp {},cores={},threads={},sockets={}'.format(
cpus, cores, threads, sockets)
def qemu_set_ssh_fwd_port(self, fwd_port):
"""Set host port for guest SSH forwarding.
@ -256,7 +258,8 @@ class QemuUtils(object):
if int(ret_code) != 0:
logging.debug('QMP execute failed {0}'.format(stderr))
raise RuntimeError('QMP execute "{0}"'
' failed on {1}'.format(cmd, self._node['host']))
' failed on {1}'.format(
cmd, self._node['host']))
logging.debug(stdout)
# Skip capabilities negotiation messages.
out_list = stdout.splitlines()
@ -268,7 +271,8 @@ class QemuUtils(object):
def _qemu_qga_flush(self):
"""Flush the QGA parser state
"""
qga_cmd = '(printf "\xFF"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \
qga_cmd = '(printf "\xFF"; sleep 1) | ' \
'sudo -S socat - UNIX-CONNECT:' + \
self._qga_sock
# TODO: probably need something else
(ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd)
@ -298,7 +302,8 @@ class QemuUtils(object):
if int(ret_code) != 0:
logging.debug('QGA execute failed {0}'.format(stderr))
raise RuntimeError('QGA execute "{0}"'
' failed on {1}'.format(cmd, self._node['host']))
' failed on {1}'.format(
cmd, self._node['host']))
logging.debug(stdout)
if not stdout:
return {}
@ -322,7 +327,8 @@ class QemuUtils(object):
self._qemu_qga_flush()
out = self._qemu_qga_exec('guest-ping')
except ValueError:
logging.debug('QGA guest-ping unexpected output {}'.format(out))
logging.debug(
'QGA guest-ping unexpected output {}'.format(out))
# Empty output - VM not booted yet
if not out:
sleep(5)
@ -335,10 +341,12 @@ class QemuUtils(object):
else:
# If there is an unexpected output from QGA guest-info, try
# again until timeout.
logging.debug('QGA guest-ping unexpected output {}'.format(out))
logging.debug(
'QGA guest-ping unexpected output {}'.format(out))
logging.debug('VM {0} booted on {1}'.format(self._qemu_opt['disk_image'],
self._node['host']))
logging.debug(
'VM {0} booted on {1}'.format(self._qemu_opt['disk_image'],
self._node['host']))
def _update_vm_interfaces(self):
"""Update interface names in VM node dict."""
@ -349,8 +357,9 @@ class QemuUtils(object):
interfaces = out.get('return')
mac_name = {}
if not interfaces:
raise RuntimeError('Get VM {0} interface list failed on {1}'.format(
self._qemu_opt['disk_image'], self._node['host']))
raise RuntimeError(
'Get VM {0} interface list failed on {1}'.format(
self._qemu_opt['disk_image'], self._node['host']))
# Create MAC-name dict
for interface in interfaces:
if 'hardware-address' not in interface:
@ -361,7 +370,8 @@ class QemuUtils(object):
mac = interface.get('mac_address')
if_name = mac_name.get(mac)
if if_name is None:
logging.debug('Interface name for MAC {} not found'.format(mac))
logging.debug(
'Interface name for MAC {} not found'.format(mac))
else:
interface['name'] = if_name
@ -380,20 +390,25 @@ class QemuUtils(object):
# If we want to allocate hugepage dynamically
if allocate:
mem_needed = abs((huge_free * huge_size) - (mem_size * 1024))
huge_to_allocate = ((mem_needed / huge_size) * 2) + huge_total
huge_to_allocate = ((mem_needed // huge_size) * 2) + huge_total
max_map_count = huge_to_allocate*4
# Increase maximum number of memory map areas a process may have
cmd = 'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'.format(
# Increase maximum number of memory map areas a
# process may have
cmd = \
'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'.format(
max_map_count)
(ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
# Increase hugepage count
cmd = 'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'.format(
cmd = \
'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'.format(
huge_to_allocate)
(ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd)
if int(ret_code) != 0:
logging.debug('Mount huge pages failed {0}'.format(stderr))
raise RuntimeError('Mount huge pages failed on {0}'.format(
self._node['host']))
logging.debug(
'Mount huge pages failed {0}'.format(stderr))
raise RuntimeError(
'Mount huge pages failed on {0}'.format(
self._node['host']))
# If we do not want to allocate dynamicaly end with error
else:
raise RuntimeError(
@ -467,7 +482,8 @@ class QemuUtils(object):
try:
huge_free = int(out)
except ValueError:
logging.debug('Reading free huge pages information failed')
logging.debug(
'Reading free huge pages information failed')
else:
break
else:
@ -493,7 +509,8 @@ class QemuUtils(object):
try:
huge_total = int(out)
except ValueError:
logging.debug('Reading total huge pages information failed')
logging.debug(
'Reading total huge pages information failed')
else:
break
else:

View File

@ -11,6 +11,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
"""VPP util library"""
import logging
import re
@ -45,12 +47,18 @@ class VPPUtil(object):
stderr=subprocess.PIPE)
with prc.stdout:
for line in iter(prc.stdout.readline, b''):
lines = prc.stdout.readlines()
for line in lines:
if type(line) != str:
line = line.decode()
logging.info(" {}".format(line.strip('\n')))
out += line
with prc.stderr:
for line in iter(prc.stderr.readline, b''):
lines = prc.stderr.readlines()
for line in lines:
if type(line) != str:
line = line.decode()
logging.warn(" {}".format(line.strip('\n')))
err += line
@ -135,8 +143,10 @@ class VPPUtil(object):
sfd.close()
# Add the key
key = requests.get('https://packagecloud.io/fdio/{}/gpgkey'.format(branch))
cmd = 'echo "{}" | apt-key add -'.format(key.content)
key = requests.get(
'https://packagecloud.io/fdio/{}/gpgkey'.format(branch))
cmd = 'echo "{}" | apt-key add -'.format(key.content.decode(key.encoding))
(ret, stdout, stderr) = self.exec_command(cmd)
if ret != 0:
raise RuntimeError('{} failed on node {} {}'.format(
@ -205,28 +215,31 @@ class VPPUtil(object):
stderr))
# Get the file contents
reps = '[fdio_{}]\n'.format(branch)
reps += 'name=fdio_{}\n'.format(branch)
reps += 'baseurl=https://packagecloud.io/fdio/{}/el/7/$basearch\n'.format(branch)
reps += 'repo_gpgcheck=1\n'
reps += 'gpgcheck=0\n'
reps += 'enabled=1\n'
reps += 'gpgkey=https://packagecloud.io/fdio/{}/gpgkey\n'.format(branch)
reps += 'sslverify=1\n'
reps += 'sslcacert=/etc/pki/tls/certs/ca-bundle.crt\n'
reps += 'metadata_expire=300\n'
reps += '\n'
reps += '[fdio_{}-source]\n'.format(branch)
reps += 'name=fdio_{}-source\n'.format(branch)
reps += 'baseurl=https://packagecloud.io/fdio/{}/el/7/SRPMS\n'.format(branch)
reps += 'repo_gpgcheck=1\n'
reps += 'gpgcheck=0\n'
reps += 'enabled=1\n'
reps += 'gpgkey=https://packagecloud.io/fdio/{}/gpgkey\n'.format(branch)
reps += 'sslverify =1\n'
reps += 'sslcacert=/etc/pki/tls/certs/ca-bundle.crt\n'
reps += 'metadata_expire=300\n'
reps = '\n'.join([
'[fdio_{}]'.format(branch),
'name=fdio_{}'.format(branch),
'baseurl=https://packagecloud.io/fdio/{}/el/7/$basearch'.format(
branch),
'repo_gpgcheck=1',
'gpgcheck=0',
'enabled=1',
'gpgkey=https://packagecloud.io/fdio/{}/gpgkey'.format(branch),
'sslverify=1',
'sslcacert=/etc/pki/tls/certs/ca-bundle.crt',
'metadata_expire=300\n',
'[fdio_{}-source]'.format(branch),
'name=fdio_release-{}'.format(branch),
'baseurl=https://packagecloud.io/fdio/{}/el/7/SRPMS'.format(
branch),
'repo_gpgcheck=1',
'gpgcheck=0',
'enabled=1',
'gpgkey=https://packagecloud.io/fdio/{}/gpgkey'.format(branch),
'sslverify =1',
'sslcacert=/etc/pki/tls/certs/ca-bundle.crt',
'metadata_expire=300\n'
])
with open(sfile, 'w') as sfd:
sfd.write(reps)
sfd.close()
@ -240,7 +253,8 @@ class VPPUtil(object):
node['host'],
stderr))
cmd = "yum -q makecache -y --disablerepo='*' --enablerepo='fdio_{}'".format(branch)
cmd = "yum -q makecache -y --disablerepo='*' " \
"--enablerepo='fdio_{}'".format(branch)
(ret, stdout, stderr) = self.exec_command(cmd)
if ret != 0:
logging.debug('{} failed on node {} {}'.format(
@ -425,14 +439,15 @@ class VPPUtil(object):
:param node: VPP node.
:type node: dict
:returns: Dictionary containing a list of VMs and the interfaces that are connected to VPP
:returns: Dictionary containing a list of VMs and the interfaces
that are connected to VPP
:rtype: dictionary
"""
vmdict = {}
print "Need to implement get vms"
print ("Need to implement get vms")
return vmdict
@staticmethod
@ -627,8 +642,10 @@ class VPPUtil(object):
def get_interfaces_numa_node(node, *iface_keys):
"""Get numa node on which are located most of the interfaces.
Return numa node with highest count of interfaces provided as arguments.
Return 0 if the interface does not have numa_node information available.
Return numa node with highest count of interfaces provided as
arguments.
Return 0 if the interface does not have numa_node information
available.
If all interfaces have unknown location (-1), then return 0.
If most of interfaces have unknown location (-1), but there are
some interfaces with known location, then return the second most
@ -702,7 +719,9 @@ class VPPUtil(object):
cmd = 'service vpp stop'
(ret, stdout, stderr) = VPPUtil.exec_command(cmd)
if ret != 0:
logging.debug('{} failed on node {} {} {}'.format(cmd, node['host'], stdout, stderr))
logging.debug('{} failed on node {} {} {}'.
format(cmd, node['host'],
stdout, stderr))
# noinspection RegExpRedundantEscape
@staticmethod
@ -754,11 +773,12 @@ class VPPUtil(object):
distro = platform.linux_distribution()
if distro[0] == 'Ubuntu' or \
distro[0] == 'CentOS Linux' or \
distro[:7] == 'Red Hat':
distro[0] == 'CentOS Linux' or \
distro[:7] == 'Red Hat':
return distro
else:
raise RuntimeError('Linux Distribution {} is not supported'.format(distro[0]))
raise RuntimeError(
'Linux Distribution {} is not supported'.format(distro[0]))
@staticmethod
def version():
@ -810,7 +830,7 @@ class VPPUtil(object):
bridges = []
for line in lines:
if line == 'no bridge-domains in use':
print line
print (line)
return ifaces
if len(line) == 0:
continue
@ -834,5 +854,5 @@ class VPPUtil(object):
ifcidx = {'name': iface[0], 'index': line.split()[1]}
ifaces.append(ifcidx)
print stdout
print (stdout)
return ifaces

View File

@ -101,7 +101,8 @@ class VppGrubUtil(object):
value = cmdline.split('{}='.format(grubcmdline))[1]
value = value.rstrip('"').lstrip('"')
# jadfix intel_pstate=disable sometimes cause networks to hang on reboot
# jadfix intel_pstate=disable sometimes cause networks to
# hang on reboot
# iommu = re.findall(r'iommu=\w+', value)
# pstate = re.findall(r'intel_pstate=\w+', value)
# If there is already some iommu commands set, leave them,

View File

@ -11,6 +11,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
"""VPP Huge Page Utilities"""
import re
@ -95,16 +97,16 @@ class VppHugePageUtil(object):
node = self._node
hugepages = node['hugepages']
print " {:30}: {}".format("Total System Memory",
hugepages['memtotal'])
print " {:30}: {}".format("Total Free Memory",
hugepages['memfree'])
print " {:30}: {}".format("Actual Huge Page Total",
hugepages['actual_total'])
print " {:30}: {}".format("Configured Huge Page Total",
hugepages['total'])
print " {:30}: {}".format("Huge Pages Free", hugepages['free'])
print " {:30}: {}".format("Huge Page Size", hugepages['size'])
print (" {:30}: {}".format("Total System Memory",
hugepages['memtotal']))
print (" {:30}: {}".format("Total Free Memory",
hugepages['memfree']))
print (" {:30}: {}".format("Actual Huge Page Total",
hugepages['actual_total']))
print (" {:30}: {}".format("Configured Huge Page Total",
hugepages['total']))
print (" {:30}: {}".format("Huge Pages Free", hugepages['free']))
print (" {:30}: {}".format("Huge Page Size", hugepages['size']))
def get_huge_page_config(self):
"""

View File

@ -11,6 +11,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
"""VPP PCI Utility libraries"""
import re
@ -44,7 +46,7 @@ class VppPCIUtil(object):
ids = re.findall(PCI_DEV_ID_REGEX, device_string)
descriptions = re.findall(r'\'([\s\S]*?)\'', device_string)
unused = re.findall(r'unused=[\w,]+', device_string)
unused = re.findall(r'unused=\w+|unused=', device_string)
for i, j in enumerate(ids):
device = {'description': descriptions[i]}
@ -268,8 +270,8 @@ class VppPCIUtil(object):
dashseparator = ("-" * (len(header) - 2))
if show_header is True:
print header
print dashseparator
print (header)
print (dashseparator)
for dit in devices.items():
dvid = dit[0]
device = dit[1]
@ -282,11 +284,11 @@ class VppPCIUtil(object):
else:
interface = interfaces[i]
print "{:15} {:25} {:50}".format(
dvid, interface, device['description'])
print ("{:15} {:25} {:50}".format(
dvid, interface, device['description']))
else:
print "{:15} {:50}".format(
dvid, device['description'])
print ("{:15} {:50}".format(
dvid, device['description']))
@staticmethod
def unbind_vpp_device(node, device_id):