Submit initial test framework skeleton.

Change-Id: I1c7cdbbf16c137a6739447d2776595725b798b54
Signed-off-by: Stefan Kobza <skobza@cisco.com>
This commit is contained in:
Stefan Kobza
2015-12-23 17:00:10 +01:00
parent 7d08f5635d
commit b189933c39
20 changed files with 712 additions and 0 deletions

6
test/.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
/env
/outputs
/output.xml
/log.html
/report.html
*.pyc

17
test/README Normal file
View File

@ -0,0 +1,17 @@
# STEPS TO START DEVELOPING TESTS LOCALLY
- install virtualenv
- generate environment using virtualenv:
# cd $ROOT
# virtualenv env
# source env/bin/activate
- install python requirements for this project by executing:
# pip install -r requirements.txt
- make sure user mentioned in topology.py has NOPASSWD sudo access to
vpe_api_test
Done.
# STEPS TO START THE TESTS
export PYTHONPATH=.
pybot -L TRACE -V resources/libraries/python/topology.py tests

105
test/main.py Normal file
View File

@ -0,0 +1,105 @@
#!/usr/bin/env python
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import robot
from robot.run import RobotFramework
from robot.conf.settings import RobotSettings
from robot.running.builder import TestSuiteBuilder
from robot.running.model import TestSuite
def get_suite_list(*datasources, **options):
class _MyRobotFramework(RobotFramework):
def main(self, datasources, **options):
# copied from robot.run.RobotFramework.main
settings = RobotSettings(options)
suite = TestSuiteBuilder(settings['SuiteNames'],
settings['WarnOnSkipped']).build(*datasources)
suite.configure(**settings.suite_config)
return suite
# Options are in robot.conf.settings
suite = _MyRobotFramework().execute(*datasources, **options)
if isinstance(suite, TestSuite):
suites = []
suites.append(suite)
append_new = True
while append_new:
append_new = False
tmp = []
for s in suites:
if len(s.suites._items) > 0:
for i in s.suites._items:
tmp.append(i)
append_new = True
else:
tmp.append(s)
suites = tmp
return suites
else:
# TODO: add from robot.errors typ error
return []
def run_suites(test_dir, suites, out_dir="./outputs", **options):
# TODO: add logic
try:
for f in os.listdir(out_dir):
os.remove('/'.join((out_dir, f)))
except OSError:
pass
if not os.path.exists(out_dir):
os.makedirs(out_dir)
for s in suites:
longname = s.longname
varfile=[]
varfile.append('resources/libraries/python/topology.py')
# TODO: check testcases Tags
with open('{}/{}.out'.format(out_dir, longname), 'w') as out, \
open('{}/{}.log'.format(out_dir, longname), 'w') as debug:
robot.run(test_dir,
suite=[longname],
output='{}/{}.xml'.format(out_dir, longname),
debugfile=debug,
log=None,
report=None,
stdout=out,
variablefile=varfile,
**options)
def parse_outputs(out_dir='./'):
outs = ['/'.join((out_dir, file)) for file in os.listdir(out_dir) if file.endswith('.xml')]
robot.rebot(*outs, merge=True)
if __name__ == "__main__":
i = []
e = []
# i = ['bd', 'ip']
# i = ['hw']
# e = ['hw']
test_dir = "./tests"
out_dir = "./outputs"
suite_list = get_suite_list(test_dir, include=i, exclude=e, output=None, dryrun=True)
run_suites(test_dir, suite_list, include=i, exclude=e, out_dir=out_dir)
parse_outputs(out_dir=out_dir)

3
test/requirements.txt Normal file
View File

@ -0,0 +1,3 @@
robotframework
paramiko
scp

View File

@ -0,0 +1,16 @@
#!/bin/bash
echo vpe process
ps aux | grep vpe
echo Free memory
free -m
echo List vpp packages
dpkg -l \*vpp\*
echo List /proc/meminfo
cat /proc/meminfo

View File

@ -0,0 +1,40 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from robot.api import logger
from topology import NodeType
from ssh import SSH
class DUTSetup(object):
def __init__(self):
pass
def setup_all_duts(self, nodes):
"""Prepare all DUTs in given topology for test execution."""
for node in nodes.values():
if node['type'] == NodeType.DUT:
self.setup_dut(node)
def setup_dut(self, node):
ssh = SSH()
ssh.connect(node)
ssh.scp('resources/libraries/bash/dut_setup.sh', '/tmp/dut_setup.sh')
(ret_code, stdout, stderr) = \
ssh.exec_command('sudo -Sn bash /tmp/dut_setup.sh')
logger.trace(stdout)
if 0 != int(ret_code):
logger.error('DUT {0} setup script failed: "{1}"'.
format(node['host'], stdout + stderr))
raise Exception('DUT test setup script failed at node {}'.
format(node['host']))

View File

@ -0,0 +1,92 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import shlex
from ssh import SSH
from subprocess import Popen, PIPE, call
from tempfile import NamedTemporaryFile
from os.path import basename
from constants import Constants as con
from robot.api import logger
__all__ = ["SetupFramework"]
class SetupFramework(object):
"""Setup suite run on topology nodes.
Many VAT/CLI based tests need the scripts at remote hosts before executing
them. This class packs the whole testing directory and copies it over
to all nodes in topology under /tmp/
"""
def __init__(self):
pass
def __pack_framework_dir(self):
"""Pack the testing WS into temp file, return its name."""
tmpfile = NamedTemporaryFile(suffix=".tgz", prefix="openvpp-testing-")
file_name = tmpfile.name
tmpfile.close()
proc = Popen(shlex.split("tar -zcf {0} .".format(file_name)),
stdout=PIPE, stderr=PIPE)
(stdout, stderr) = proc.communicate()
logger.debug(stdout)
logger.debug(stderr)
return_code = proc.wait()
if 0 != return_code:
raise Exception("Could not pack testing framework.")
return file_name
def __copy_tarball_to_node(self, tarball, node):
logger.console('Copying tarball to {0}'.format(node['host']))
ssh = SSH()
ssh.connect(node)
ssh.scp(tarball, "/tmp/")
def __extract_tarball_at_node(self, tarball, node):
logger.console('Extracting tarball to {0} on {1}'.format(
con.REMOTE_FW_DIR, node['host']))
ssh = SSH()
ssh.connect(node)
cmd = 'rm -rf {1}; mkdir {1} ; sudo -Sn tar -zxf {0} -C {1};'.format(
tarball, con.REMOTE_FW_DIR)
(ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout=30)
if 0 != ret_code:
logger.error('Unpack error: {0}'.format(stderr))
raise Exception('Failed to unpack {0} at node {1}'.format(
tarball, node['host']))
def __delete_local_tarball(self, tarball):
call(shlex.split('sh -c "rm {0} > /dev/null 2>&1"'.format(tarball)))
def setup_framework(self, nodes):
"""Pack the whole directory and extract in temp on each node."""
tarball = self.__pack_framework_dir()
logger.console('Framework packed to {0}'.format(tarball))
remote_tarball = "/tmp/{0}".format(basename(tarball))
for node in nodes.values():
self.__copy_tarball_to_node(tarball, node)
self.__extract_tarball_at_node(remote_tarball, node)
logger.trace('Test framework copied to all topology nodes')
self.__delete_local_tarball(tarball)

View File

@ -0,0 +1,84 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from ssh import SSH
from robot.api import logger
__all__ = []
class VatExecutor(object):
__TMP_DIR = "/tmp/"
__VAT_BIN = "vpe_api_test"
def __init__(self):
self._stdout = None
self._stderr = None
self._ret_code = None
def execute_script(self, local_path, node, timeout=10, json_out=True):
"""Copy local_path script to node, execute it and return result.
Returns (rc, stdout, stderr tuple).
"""
ssh = SSH()
ssh.connect(node)
local_basename = os.path.basename(local_path)
remote_file_path = self.__TMP_DIR + local_basename
remote_file_out = remote_file_path + ".out"
ssh.scp(local_path, remote_file_path)
cmd = "sudo -S {vat} {json} < {input}".format(vat=self.__VAT_BIN,
json="json" if json_out == True else "",
input=remote_file_path)
(ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout)
self._ret_code = ret_code
self._stdout = stdout
self._stderr = stderr
logger.trace("Command '{0}' returned {1}'".format(cmd, self._ret_code))
logger.trace("stdout: '{0}'".format(self._stdout))
logger.trace("stderr: '{0}'".format(self._stderr))
#TODO: download vpe_api_test output file
self._delete_files(node, remote_file_path, remote_file_out)
def _delete_files(self, node, *files):
ssh = SSH()
ssh.connect(node)
files = " ".join([str(x) for x in files])
ssh.exec_command("rm {0}".format(files))
def script_should_have_failed(self):
if self._ret_code is None:
raise Exception("First execute the script!")
if self._ret_code == 0:
raise AssertionError(
"Script execution passed, but failure was expected")
def script_should_have_passed(self):
if self._ret_code is None:
raise Exception("First execute the script!")
if self._ret_code != 0:
raise AssertionError(
"Script execution failed, but success was expected")
def get_script_stdout(self):
return self._stdout
def get_script_stderr(self):
return self._stderr

View File

@ -0,0 +1,15 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class Constants(object):
#OpenVPP testing directory location at topology nodes
REMOTE_FW_DIR = '/tmp/openvpp-testing'

View File

@ -0,0 +1,127 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import paramiko
from scp import SCPClient
from time import time
from robot.api import logger
__all__ = ["exec_cmd"]
# TODO: Attempt to recycle SSH connections
# TODO: load priv key
class SSH(object):
__MAX_RECV_BUF = 10*1024*1024
__existing_connections = {}
def __init__(self):
self._ssh = paramiko.SSHClient()
self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self._hostname = None
def _node_hash(self, node):
return hash(frozenset([node['host'], node['port']]))
def connect(self, node):
"""Connect to node prior to running exec_command or scp.
If there already is a connection to the node, this method reuses it.
"""
self._hostname = node['host']
node_hash = self._node_hash(node)
if node_hash in self.__existing_connections:
self._ssh = self.__existing_connections[node_hash]
else:
start = time()
self._ssh.connect(node['host'], username=node['username'],
password=node['password'])
self.__existing_connections[node_hash] = self._ssh
logger.trace('connect took {} seconds'.format(time() - start))
def exec_command(self, cmd, timeout=10):
"""Execute SSH command on a new channel on the connected Node.
Returns (return_code, stdout, stderr).
"""
start = time()
chan = self._ssh.get_transport().open_session()
if timeout is not None:
chan.settimeout(int(timeout))
chan.exec_command(cmd)
end = time()
logger.trace('exec_command "{0}" on {1} took {2} seconds'.format(cmd,
self._hostname, end-start))
stdout = ""
while True:
buf = chan.recv(self.__MAX_RECV_BUF)
stdout += buf
if not buf:
break
stderr = ""
while True:
buf = chan.recv_stderr(self.__MAX_RECV_BUF)
stderr += buf
if not buf:
break
return_code = chan.recv_exit_status()
logger.trace('chan_recv/_stderr took {} seconds'.format(time()-end))
return (return_code, stdout, stderr)
def scp(self, local_path, remote_path):
"""Copy files from local_path to remote_path.
connect() method has to be called first!
"""
logger.trace('SCP {0} to {1}:{2}'.format(
local_path, self._hostname, remote_path))
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(self._ssh.get_transport())
start = time()
scp.put(local_path, remote_path)
scp.close()
end = time()
logger.trace('SCP took {0} seconds'.format(end-start))
def exec_cmd(node, cmd, timeout=None):
"""Convenience function to ssh/exec/return rc & out.
Returns (rc, stdout).
"""
if node is None:
raise TypeError('Node parameter is None')
if cmd is None:
raise TypeError('Command parameter is None')
if len(cmd) == 0:
raise ValueError('Empty command parameter')
ssh = SSH()
try:
ssh.connect(node)
except Exception, e:
logger.error("Failed to connect to node" + e)
return None
try:
(ret_code, stdout, stderr) = ssh.exec_command(cmd, timeout=timeout)
except Exception, e:
logger.error(e)
return None
return (ret_code, stdout, stderr)

View File

@ -0,0 +1,50 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#Defines nodes and topology structure.
__all__ = ["DICT__nodes"]
class NodeType(object):
DUT = 'DUT'
TG = 'TG'
MOCK_DATA_FOR_NOW = {
'nodes' : {
'DUT1' : {
'type' : NodeType.DUT,
'host' : 'wasa-ucs-14',
'port' : 22,
'username' : '',
'password' : '',
},
'DUT2' : {
'type' : NodeType.DUT,
'host' : 'wasa-ucs-13',
'port' : 22,
'username' : '',
'password' : '',
},
'TG' : {
'type' : NodeType.TG,
'host' : 'wasa-ucs-12',
'port' : 22,
'username' : '',
'password' : '',
},
}
}
DICT__nodes = MOCK_DATA_FOR_NOW['nodes']

View File

@ -0,0 +1,20 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
| Library | resources/libraries/python/DUTSetup.py
*** Keywords ***
| Setup all DUTs before test
| | [Documentation] | Setup all DUTs in topology before test execution
| | Setup All DUTs | ${nodes}

View File

@ -0,0 +1,20 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
| Resource | resources/libraries/robot/vat/interfaces.robot
*** Keywords ***
| VPP reports interfaces on | [Arguments] | ${node}
| | VPP reports interfaces through VAT on | ${node}
#| | VPP reports interfaces through ODL on | ${node}
#| | VPP reports interfaces through DEBUGCLI on | ${node}

View File

@ -0,0 +1,23 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
| Library | resources/libraries/python/VatExecutor.py
*** Variables ***
| ${VAT_DUMP_INTERFACES} | resources/templates/vat/dump_interfaces.vat
*** Keywords ***
| VPP reports interfaces through VAT on
| | [Arguments] | ${node}
| | Execute Script | ${VAT_DUMP_INTERFACES} | ${node}
| | Script Should Have Passed

View File

@ -0,0 +1,3 @@
sw_interface_dump
dump_interface_table
quit

View File

@ -0,0 +1,15 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
| Library | resources/libraries/python/SetupFramework.py
| Suite Setup | Setup Framework | ${nodes}

View File

@ -0,0 +1,26 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
| Resource | resources/libraries/robot/default.robot
| Resource | resources/libraries/robot/interfaces.robot
| Test Setup | Setup all DUTs before test
*** Test Cases ***
| VPP reports interfaces
| | VPP reports interfaces on | ${nodes['DUT1']}
| Loop the test
| | : FOR | ${INDEX} | in range | 10
| | | VPP reports interfaces on | ${nodes['DUT1']}

View File

@ -0,0 +1,14 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
| Documentation | Dummy test suite to test suite execution

View File

@ -0,0 +1,18 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
| Force Tags | vhost-user | hw
*** Test Cases ***
| Create vhost user interface on hw
| | Log | interface created

View File

@ -0,0 +1,18 @@
# Copyright (c) 2015 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*** Settings ***
| Force Tags | vhost-user | virl
*** Test Cases ***
| Create vhost user interface on virl
| | Log | interface created