blob: e566c70e64a1971e653a2922012e5e420a85b193 [file] [log] [blame]
Lucas Bates76b903e2017-06-16 17:22:35 -04001#!/usr/bin/env python3
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01002# SPDX-License-Identifier: GPL-2.0
Lucas Bates76b903e2017-06-16 17:22:35 -04003
4"""
5tdc.py - Linux tc (Traffic Control) unit test driver
6
7Copyright (C) 2017 Lucas Bates <lucasb@mojatatu.com>
8"""
9
10import re
11import os
12import sys
13import argparse
Brenda J. Butler93707cb2018-02-14 14:09:21 -050014import importlib
Lucas Bates76b903e2017-06-16 17:22:35 -040015import json
16import subprocess
Brenda J. Butler93707cb2018-02-14 14:09:21 -050017import time
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -050018import traceback
Lucas Bates76b903e2017-06-16 17:22:35 -040019from collections import OrderedDict
20from string import Template
21
22from tdc_config import *
23from tdc_helper import *
24
Brenda J. Butler93707cb2018-02-14 14:09:21 -050025import TdcPlugin
Lucas Bates915c1582018-12-06 17:42:26 -050026from TdcResults import *
Lucas Bates76b903e2017-06-16 17:22:35 -040027
Lucas Bates489ce2f2019-06-24 21:00:27 -040028class PluginDependencyException(Exception):
29 def __init__(self, missing_pg):
30 self.missing_pg = missing_pg
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -050031
32class PluginMgrTestFail(Exception):
33 def __init__(self, stage, output, message):
34 self.stage = stage
35 self.output = output
36 self.message = message
37
Brenda J. Butler93707cb2018-02-14 14:09:21 -050038class PluginMgr:
39 def __init__(self, argparser):
40 super().__init__()
41 self.plugins = {}
42 self.plugin_instances = []
Lucas Bates489ce2f2019-06-24 21:00:27 -040043 self.failed_plugins = {}
Brenda J. Butler93707cb2018-02-14 14:09:21 -050044 self.argparser = argparser
45
46 # TODO, put plugins in order
47 plugindir = os.getenv('TDC_PLUGIN_DIR', './plugins')
48 for dirpath, dirnames, filenames in os.walk(plugindir):
49 for fn in filenames:
50 if (fn.endswith('.py') and
51 not fn == '__init__.py' and
52 not fn.startswith('#') and
53 not fn.startswith('.#')):
54 mn = fn[0:-3]
55 foo = importlib.import_module('plugins.' + mn)
56 self.plugins[mn] = foo
57 self.plugin_instances.append(foo.SubPlugin())
58
Lucas Bates489ce2f2019-06-24 21:00:27 -040059 def load_plugin(self, pgdir, pgname):
60 pgname = pgname[0:-3]
61 foo = importlib.import_module('{}.{}'.format(pgdir, pgname))
62 self.plugins[pgname] = foo
63 self.plugin_instances.append(foo.SubPlugin())
64 self.plugin_instances[-1].check_args(self.args, None)
65
66 def get_required_plugins(self, testlist):
67 '''
68 Get all required plugins from the list of test cases and return
69 all unique items.
70 '''
71 reqs = []
72 for t in testlist:
73 try:
74 if 'requires' in t['plugins']:
75 if isinstance(t['plugins']['requires'], list):
76 reqs.extend(t['plugins']['requires'])
77 else:
78 reqs.append(t['plugins']['requires'])
79 except KeyError:
80 continue
81 reqs = get_unique_item(reqs)
82 return reqs
83
84 def load_required_plugins(self, reqs, parser, args, remaining):
85 '''
86 Get all required plugins from the list of test cases and load any plugin
87 that is not already enabled.
88 '''
89 pgd = ['plugin-lib', 'plugin-lib-custom']
90 pnf = []
91
92 for r in reqs:
93 if r not in self.plugins:
94 fname = '{}.py'.format(r)
95 source_path = []
96 for d in pgd:
97 pgpath = '{}/{}'.format(d, fname)
98 if os.path.isfile(pgpath):
99 source_path.append(pgpath)
100 if len(source_path) == 0:
101 print('ERROR: unable to find required plugin {}'.format(r))
102 pnf.append(fname)
103 continue
104 elif len(source_path) > 1:
105 print('WARNING: multiple copies of plugin {} found, using version found')
106 print('at {}'.format(source_path[0]))
107 pgdir = source_path[0]
108 pgdir = pgdir.split('/')[0]
109 self.load_plugin(pgdir, fname)
110 if len(pnf) > 0:
111 raise PluginDependencyException(pnf)
112
113 parser = self.call_add_args(parser)
114 (args, remaining) = parser.parse_known_args(args=remaining, namespace=args)
115 return args
116
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500117 def call_pre_suite(self, testcount, testidlist):
118 for pgn_inst in self.plugin_instances:
119 pgn_inst.pre_suite(testcount, testidlist)
120
121 def call_post_suite(self, index):
122 for pgn_inst in reversed(self.plugin_instances):
123 pgn_inst.post_suite(index)
124
Lucas Batesa7d50a02019-07-08 21:34:26 -0400125 def call_pre_case(self, caseinfo, *, test_skip=False):
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500126 for pgn_inst in self.plugin_instances:
127 try:
Lucas Batesa7d50a02019-07-08 21:34:26 -0400128 pgn_inst.pre_case(caseinfo, test_skip)
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500129 except Exception as ee:
130 print('exception {} in call to pre_case for {} plugin'.
131 format(ee, pgn_inst.__class__))
132 print('test_ordinal is {}'.format(test_ordinal))
Lucas Batesa7d50a02019-07-08 21:34:26 -0400133 print('testid is {}'.format(caseinfo['id']))
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500134 raise
135
136 def call_post_case(self):
137 for pgn_inst in reversed(self.plugin_instances):
138 pgn_inst.post_case()
139
140 def call_pre_execute(self):
141 for pgn_inst in self.plugin_instances:
142 pgn_inst.pre_execute()
143
144 def call_post_execute(self):
145 for pgn_inst in reversed(self.plugin_instances):
146 pgn_inst.post_execute()
147
148 def call_add_args(self, parser):
149 for pgn_inst in self.plugin_instances:
150 parser = pgn_inst.add_args(parser)
151 return parser
152
153 def call_check_args(self, args, remaining):
154 for pgn_inst in self.plugin_instances:
155 pgn_inst.check_args(args, remaining)
156
157 def call_adjust_command(self, stage, command):
158 for pgn_inst in self.plugin_instances:
159 command = pgn_inst.adjust_command(stage, command)
160 return command
161
Lucas Bates489ce2f2019-06-24 21:00:27 -0400162 def set_args(self, args):
163 self.args = args
164
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500165 @staticmethod
166 def _make_argparser(args):
167 self.argparser = argparse.ArgumentParser(
168 description='Linux TC unit tests')
169
Lucas Bates76b903e2017-06-16 17:22:35 -0400170def replace_keywords(cmd):
171 """
172 For a given executable command, substitute any known
173 variables contained within NAMES with the correct values
174 """
175 tcmd = Template(cmd)
176 subcmd = tcmd.safe_substitute(NAMES)
177 return subcmd
178
179
Brenda J. Butlera13fedb2018-02-14 14:09:23 -0500180def exec_cmd(args, pm, stage, command):
Lucas Bates76b903e2017-06-16 17:22:35 -0400181 """
182 Perform any required modifications on an executable command, then run
183 it in a subprocess and return the results.
184 """
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500185 if len(command.strip()) == 0:
186 return None, None
Lucas Bates76b903e2017-06-16 17:22:35 -0400187 if '$' in command:
188 command = replace_keywords(command)
189
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500190 command = pm.call_adjust_command(stage, command)
191 if args.verbose > 0:
192 print('command "{}"'.format(command))
Lucas Bates76b903e2017-06-16 17:22:35 -0400193 proc = subprocess.Popen(command,
194 shell=True,
195 stdout=subprocess.PIPE,
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500196 stderr=subprocess.PIPE,
197 env=ENVIR)
Lucas Bates76b903e2017-06-16 17:22:35 -0400198
Lucas Batesd37e56d2018-12-06 17:42:24 -0500199 try:
200 (rawout, serr) = proc.communicate(timeout=NAMES['TIMEOUT'])
201 if proc.returncode != 0 and len(serr) > 0:
202 foutput = serr.decode("utf-8", errors="ignore")
203 else:
204 foutput = rawout.decode("utf-8", errors="ignore")
205 except subprocess.TimeoutExpired:
206 foutput = "Command \"{}\" timed out\n".format(command)
207 proc.returncode = 255
Lucas Bates76b903e2017-06-16 17:22:35 -0400208
209 proc.stdout.close()
210 proc.stderr.close()
211 return proc, foutput
212
213
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500214def prepare_env(args, pm, stage, prefix, cmdlist, output = None):
Lucas Bates76b903e2017-06-16 17:22:35 -0400215 """
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500216 Execute the setup/teardown commands for a test case.
217 Optionally terminate test execution if the command fails.
Lucas Bates76b903e2017-06-16 17:22:35 -0400218 """
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500219 if args.verbose > 0:
220 print('{}'.format(prefix))
Lucas Bates76b903e2017-06-16 17:22:35 -0400221 for cmdinfo in cmdlist:
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500222 if isinstance(cmdinfo, list):
Lucas Bates76b903e2017-06-16 17:22:35 -0400223 exit_codes = cmdinfo[1:]
224 cmd = cmdinfo[0]
225 else:
226 exit_codes = [0]
227 cmd = cmdinfo
228
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500229 if not cmd:
Lucas Bates76b903e2017-06-16 17:22:35 -0400230 continue
231
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500232 (proc, foutput) = exec_cmd(args, pm, stage, cmd)
Lucas Bates76b903e2017-06-16 17:22:35 -0400233
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500234 if proc and (proc.returncode not in exit_codes):
235 print('', file=sys.stderr)
236 print("{} *** Could not execute: \"{}\"".format(prefix, cmd),
237 file=sys.stderr)
238 print("\n{} *** Error message: \"{}\"".format(prefix, foutput),
239 file=sys.stderr)
Brenda J. Butlerc6cecf42018-11-16 17:37:56 -0500240 print("returncode {}; expected {}".format(proc.returncode,
241 exit_codes))
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500242 print("\n{} *** Aborting test run.".format(prefix), file=sys.stderr)
243 print("\n\n{} *** stdout ***".format(proc.stdout), file=sys.stderr)
244 print("\n\n{} *** stderr ***".format(proc.stderr), file=sys.stderr)
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500245 raise PluginMgrTestFail(
246 stage, output,
247 '"{}" did not complete successfully'.format(prefix))
Lucas Bates76b903e2017-06-16 17:22:35 -0400248
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500249def run_one_test(pm, args, index, tidx):
Brenda J. Butler75291f32018-03-08 15:27:44 -0500250 global NAMES
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500251 result = True
252 tresult = ""
253 tap = ""
Lucas Bates915c1582018-12-06 17:42:26 -0500254 res = TestResult(tidx['id'], tidx['name'])
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500255 if args.verbose > 0:
256 print("\t====================\n=====> ", end="")
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500257 print("Test " + tidx["id"] + ": " + tidx["name"])
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500258
Lucas Bates255c1c72019-02-28 17:38:40 -0500259 if 'skip' in tidx:
260 if tidx['skip'] == 'yes':
261 res = TestResult(tidx['id'], tidx['name'])
262 res.set_result(ResultState.skip)
263 res.set_errormsg('Test case designated as skipped.')
Lucas Batesa7d50a02019-07-08 21:34:26 -0400264 pm.call_pre_case(tidx, test_skip=True)
Lucas Bates255c1c72019-02-28 17:38:40 -0500265 pm.call_post_execute()
266 return res
267
Brenda J. Butler75291f32018-03-08 15:27:44 -0500268 # populate NAMES with TESTID for this test
269 NAMES['TESTID'] = tidx['id']
270
Lucas Batesa7d50a02019-07-08 21:34:26 -0400271 pm.call_pre_case(tidx)
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500272 prepare_env(args, pm, 'setup', "-----> prepare stage", tidx["setup"])
273
274 if (args.verbose > 0):
275 print('-----> execute stage')
276 pm.call_pre_execute()
277 (p, procout) = exec_cmd(args, pm, 'execute', tidx["cmdUnderTest"])
Brenda J. Butlerc6cecf42018-11-16 17:37:56 -0500278 if p:
279 exit_code = p.returncode
280 else:
281 exit_code = None
282
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500283 pm.call_post_execute()
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500284
Brenda J. Butlerc6cecf42018-11-16 17:37:56 -0500285 if (exit_code is None or exit_code != int(tidx["expExitCode"])):
Brenda J. Butlerc6cecf42018-11-16 17:37:56 -0500286 print("exit: {!r}".format(exit_code))
287 print("exit: {}".format(int(tidx["expExitCode"])))
288 #print("exit: {!r} {}".format(exit_code, int(tidx["expExitCode"])))
Lucas Bates915c1582018-12-06 17:42:26 -0500289 res.set_result(ResultState.fail)
290 res.set_failmsg('Command exited with {}, expected {}\n{}'.format(exit_code, tidx["expExitCode"], procout))
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500291 print(procout)
292 else:
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500293 if args.verbose > 0:
294 print('-----> verify stage')
295 match_pattern = re.compile(
296 str(tidx["matchPattern"]), re.DOTALL | re.MULTILINE)
297 (p, procout) = exec_cmd(args, pm, 'verify', tidx["verifyCmd"])
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500298 if procout:
299 match_index = re.findall(match_pattern, procout)
300 if len(match_index) != int(tidx["matchCount"]):
Lucas Bates915c1582018-12-06 17:42:26 -0500301 res.set_result(ResultState.fail)
302 res.set_failmsg('Could not match regex pattern. Verify command output:\n{}'.format(procout))
303 else:
304 res.set_result(ResultState.success)
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500305 elif int(tidx["matchCount"]) != 0:
Lucas Bates915c1582018-12-06 17:42:26 -0500306 res.set_result(ResultState.fail)
307 res.set_failmsg('No output generated by verify command.')
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500308 else:
Lucas Bates915c1582018-12-06 17:42:26 -0500309 res.set_result(ResultState.success)
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500310
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500311 prepare_env(args, pm, 'teardown', '-----> teardown stage', tidx['teardown'], procout)
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500312 pm.call_post_case()
313
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500314 index += 1
315
Brenda J. Butler75291f32018-03-08 15:27:44 -0500316 # remove TESTID from NAMES
317 del(NAMES['TESTID'])
Lucas Bates915c1582018-12-06 17:42:26 -0500318 return res
Lucas Bates76b903e2017-06-16 17:22:35 -0400319
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500320def test_runner(pm, args, filtered_tests):
Lucas Bates76b903e2017-06-16 17:22:35 -0400321 """
322 Driver function for the unit tests.
323
324 Prints information about the tests being run, executes the setup and
325 teardown commands and the command under test itself. Also determines
326 success/failure based on the information in the test case and generates
327 TAP output accordingly.
328 """
329 testlist = filtered_tests
330 tcount = len(testlist)
331 index = 1
Brenda J. Butlerefab1632018-02-28 15:36:19 -0500332 tap = ''
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500333 badtest = None
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500334 stage = None
335 emergency_exit = False
336 emergency_exit_message = ''
Lucas Bates76b903e2017-06-16 17:22:35 -0400337
Lucas Bates915c1582018-12-06 17:42:26 -0500338 tsr = TestSuiteReport()
339
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500340 try:
341 pm.call_pre_suite(tcount, [tidx['id'] for tidx in testlist])
342 except Exception as ee:
343 ex_type, ex, ex_tb = sys.exc_info()
344 print('Exception {} {} (caught in pre_suite).'.
345 format(ex_type, ex))
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500346 traceback.print_tb(ex_tb)
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500347 emergency_exit_message = 'EMERGENCY EXIT, call_pre_suite failed with exception {} {}\n'.format(ex_type, ex)
348 emergency_exit = True
349 stage = 'pre-SUITE'
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500350
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500351 if emergency_exit:
352 pm.call_post_suite(index)
353 return emergency_exit_message
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500354 if args.verbose > 1:
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500355 print('give test rig 2 seconds to stabilize')
356 time.sleep(2)
Lucas Bates76b903e2017-06-16 17:22:35 -0400357 for tidx in testlist:
Chris Mi31c26112017-10-26 18:24:43 -0700358 if "flower" in tidx["category"] and args.device == None:
Lucas Bates0eba31e2019-07-29 19:18:12 -0400359 errmsg = "Tests using the DEV2 variable must define the name of a "
360 errmsg += "physical NIC with the -d option when running tdc.\n"
361 errmsg += "Test has been skipped."
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500362 if args.verbose > 1:
Lucas Bates0eba31e2019-07-29 19:18:12 -0400363 print(errmsg)
Lucas Bates915c1582018-12-06 17:42:26 -0500364 res = TestResult(tidx['id'], tidx['name'])
365 res.set_result(ResultState.skip)
Lucas Bates0eba31e2019-07-29 19:18:12 -0400366 res.set_errormsg(errmsg)
Lucas Bates915c1582018-12-06 17:42:26 -0500367 tsr.add_resultdata(res)
Chris Mi31c26112017-10-26 18:24:43 -0700368 continue
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500369 try:
370 badtest = tidx # in case it goes bad
Lucas Bates915c1582018-12-06 17:42:26 -0500371 res = run_one_test(pm, args, index, tidx)
372 tsr.add_resultdata(res)
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500373 except PluginMgrTestFail as pmtf:
374 ex_type, ex, ex_tb = sys.exc_info()
375 stage = pmtf.stage
376 message = pmtf.message
377 output = pmtf.output
Lucas Bates915c1582018-12-06 17:42:26 -0500378 res = TestResult(tidx['id'], tidx['name'])
379 res.set_result(ResultState.skip)
380 res.set_errormsg(pmtf.message)
381 res.set_failmsg(pmtf.output)
382 tsr.add_resultdata(res)
383 index += 1
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500384 print(message)
385 print('Exception {} {} (caught in test_runner, running test {} {} {} stage {})'.
386 format(ex_type, ex, index, tidx['id'], tidx['name'], stage))
387 print('---------------')
388 print('traceback')
389 traceback.print_tb(ex_tb)
390 print('---------------')
391 if stage == 'teardown':
392 print('accumulated output for this test:')
393 if pmtf.output:
394 print(pmtf.output)
395 print('---------------')
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500396 break
Lucas Bates76b903e2017-06-16 17:22:35 -0400397 index += 1
398
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500399 # if we failed in setup or teardown,
Brenda J. Butlerf9b63a12018-02-23 12:16:38 -0500400 # fill in the remaining tests with ok-skipped
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500401 count = index
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500402
Lucas Bates915c1582018-12-06 17:42:26 -0500403 if tcount + 1 != count:
404 for tidx in testlist[count - 1:]:
405 res = TestResult(tidx['id'], tidx['name'])
406 res.set_result(ResultState.skip)
407 msg = 'skipped - previous {} failed {} {}'.format(stage,
408 index, badtest.get('id', '--Unknown--'))
409 res.set_errormsg(msg)
410 tsr.add_resultdata(res)
411 count += 1
Brenda J. Butler2b3905d2018-03-08 15:28:03 -0500412
413 if args.pause:
414 print('Want to pause\nPress enter to continue ...')
415 if input(sys.stdin):
416 print('got something on stdin')
417
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500418 pm.call_post_suite(index)
Brenda J. Butler6fac7332018-02-14 14:09:20 -0500419
Lucas Bates915c1582018-12-06 17:42:26 -0500420 return tsr
Lucas Bates76b903e2017-06-16 17:22:35 -0400421
Lucas Bates76b903e2017-06-16 17:22:35 -0400422def has_blank_ids(idlist):
423 """
424 Search the list for empty ID fields and return true/false accordingly.
425 """
426 return not(all(k for k in idlist))
427
428
429def load_from_file(filename):
430 """
Brenda J. Butler170b8ff2017-10-31 14:29:03 -0400431 Open the JSON file containing the test cases and return them
432 as list of ordered dictionary objects.
Lucas Bates76b903e2017-06-16 17:22:35 -0400433 """
Brenda J. Butler170b8ff2017-10-31 14:29:03 -0400434 try:
435 with open(filename) as test_data:
436 testlist = json.load(test_data, object_pairs_hook=OrderedDict)
437 except json.JSONDecodeError as jde:
438 print('IGNORING test case file {}\n\tBECAUSE: {}'.format(filename, jde))
439 testlist = list()
440 else:
441 idlist = get_id_list(testlist)
442 if (has_blank_ids(idlist)):
443 for k in testlist:
444 k['filename'] = filename
Lucas Bates76b903e2017-06-16 17:22:35 -0400445 return testlist
446
447
448def args_parse():
449 """
450 Create the argument parser.
451 """
452 parser = argparse.ArgumentParser(description='Linux TC unit tests')
453 return parser
454
455
456def set_args(parser):
457 """
458 Set the command line arguments for tdc.
459 """
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500460 parser.add_argument(
Lucas Bates915c1582018-12-06 17:42:26 -0500461 '--outfile', type=str,
462 help='Path to the file in which results should be saved. ' +
463 'Default target is the current directory.')
464 parser.add_argument(
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500465 '-p', '--path', type=str,
466 help='The full path to the tc executable to use')
467 sg = parser.add_argument_group(
468 'selection', 'select which test cases: ' +
469 'files plus directories; filtered by categories plus testids')
470 ag = parser.add_argument_group(
471 'action', 'select action to perform on selected test cases')
472
473 sg.add_argument(
474 '-D', '--directory', nargs='+', metavar='DIR',
475 help='Collect tests from the specified directory(ies) ' +
476 '(default [tc-tests])')
477 sg.add_argument(
478 '-f', '--file', nargs='+', metavar='FILE',
479 help='Run tests from the specified file(s)')
480 sg.add_argument(
481 '-c', '--category', nargs='*', metavar='CATG', default=['+c'],
482 help='Run tests only from the specified category/ies, ' +
483 'or if no category/ies is/are specified, list known categories.')
484 sg.add_argument(
485 '-e', '--execute', nargs='+', metavar='ID',
486 help='Execute the specified test cases with specified IDs')
487 ag.add_argument(
488 '-l', '--list', action='store_true',
489 help='List all test cases, or those only within the specified category')
490 ag.add_argument(
491 '-s', '--show', action='store_true', dest='showID',
492 help='Display the selected test cases')
493 ag.add_argument(
494 '-i', '--id', action='store_true', dest='gen_id',
495 help='Generate ID numbers for new test cases')
496 parser.add_argument(
497 '-v', '--verbose', action='count', default=0,
498 help='Show the commands that are being run')
Brenda J. Butlerefab1632018-02-28 15:36:19 -0500499 parser.add_argument(
Lucas Bates915c1582018-12-06 17:42:26 -0500500 '--format', default='tap', const='tap', nargs='?',
501 choices=['none', 'xunit', 'tap'],
502 help='Specify the format for test results. (Default: TAP)')
Chris Mi31c26112017-10-26 18:24:43 -0700503 parser.add_argument('-d', '--device',
Lucas Bates0eba31e2019-07-29 19:18:12 -0400504 help='Execute test cases that use a physical device, ' +
505 'where DEVICE is its name. (If not defined, tests ' +
506 'that require a physical device will be skipped)')
Brenda J. Butler2b3905d2018-03-08 15:28:03 -0500507 parser.add_argument(
508 '-P', '--pause', action='store_true',
509 help='Pause execution just before post-suite stage')
Lucas Bates76b903e2017-06-16 17:22:35 -0400510 return parser
511
512
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500513def check_default_settings(args, remaining, pm):
Lucas Bates76b903e2017-06-16 17:22:35 -0400514 """
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500515 Process any arguments overriding the default settings,
516 and ensure the settings are correct.
Lucas Bates76b903e2017-06-16 17:22:35 -0400517 """
518 # Allow for overriding specific settings
519 global NAMES
520
521 if args.path != None:
Brenda J. Butler6a7b75f2018-02-23 12:16:37 -0500522 NAMES['TC'] = args.path
Chris Mi31c26112017-10-26 18:24:43 -0700523 if args.device != None:
Brenda J. Butler6a7b75f2018-02-23 12:16:37 -0500524 NAMES['DEV2'] = args.device
Lucas Batesd37e56d2018-12-06 17:42:24 -0500525 if 'TIMEOUT' not in NAMES:
526 NAMES['TIMEOUT'] = None
Lucas Bates76b903e2017-06-16 17:22:35 -0400527 if not os.path.isfile(NAMES['TC']):
528 print("The specified tc path " + NAMES['TC'] + " does not exist.")
529 exit(1)
530
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500531 pm.call_check_args(args, remaining)
532
Lucas Bates76b903e2017-06-16 17:22:35 -0400533
534def get_id_list(alltests):
535 """
536 Generate a list of all IDs in the test cases.
537 """
538 return [x["id"] for x in alltests]
539
540
541def check_case_id(alltests):
542 """
543 Check for duplicate test case IDs.
544 """
545 idl = get_id_list(alltests)
546 return [x for x in idl if idl.count(x) > 1]
547
548
549def does_id_exist(alltests, newid):
550 """
551 Check if a given ID already exists in the list of test cases.
552 """
553 idl = get_id_list(alltests)
554 return (any(newid == x for x in idl))
555
556
557def generate_case_ids(alltests):
558 """
559 If a test case has a blank ID field, generate a random hex ID for it
560 and then write the test cases back to disk.
561 """
562 import random
563 for c in alltests:
564 if (c["id"] == ""):
565 while True:
BTaskaya3adc1c62018-02-23 22:57:35 +0300566 newid = str('{:04x}'.format(random.randrange(16**4)))
Lucas Bates76b903e2017-06-16 17:22:35 -0400567 if (does_id_exist(alltests, newid)):
568 continue
569 else:
570 c['id'] = newid
571 break
572
573 ufilename = []
574 for c in alltests:
575 if ('filename' in c):
576 ufilename.append(c['filename'])
577 ufilename = get_unique_item(ufilename)
578 for f in ufilename:
579 testlist = []
580 for t in alltests:
581 if 'filename' in t:
582 if t['filename'] == f:
583 del t['filename']
584 testlist.append(t)
585 outfile = open(f, "w")
586 json.dump(testlist, outfile, indent=4)
Lucas Batesc0b6ede2018-03-29 15:58:10 -0400587 outfile.write("\n")
Lucas Bates76b903e2017-06-16 17:22:35 -0400588 outfile.close()
589
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500590def filter_tests_by_id(args, testlist):
591 '''
592 Remove tests from testlist that are not in the named id list.
593 If id list is empty, return empty list.
594 '''
595 newlist = list()
596 if testlist and args.execute:
597 target_ids = args.execute
598
599 if isinstance(target_ids, list) and (len(target_ids) > 0):
600 newlist = list(filter(lambda x: x['id'] in target_ids, testlist))
601 return newlist
602
603def filter_tests_by_category(args, testlist):
604 '''
605 Remove tests from testlist that are not in a named category.
606 '''
607 answer = list()
608 if args.category and testlist:
609 test_ids = list()
610 for catg in set(args.category):
611 if catg == '+c':
612 continue
613 print('considering category {}'.format(catg))
614 for tc in testlist:
615 if catg in tc['category'] and tc['id'] not in test_ids:
616 answer.append(tc)
617 test_ids.append(tc['id'])
618
619 return answer
Lucas Bates76b903e2017-06-16 17:22:35 -0400620
Lucas Bates489ce2f2019-06-24 21:00:27 -0400621
Lucas Bates76b903e2017-06-16 17:22:35 -0400622def get_test_cases(args):
623 """
624 If a test case file is specified, retrieve tests from that file.
625 Otherwise, glob for all json files in subdirectories and load from
626 each one.
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500627 Also, if requested, filter by category, and add tests matching
628 certain ids.
Lucas Bates76b903e2017-06-16 17:22:35 -0400629 """
630 import fnmatch
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500631
632 flist = []
633 testdirs = ['tc-tests']
634
635 if args.file:
636 # at least one file was specified - remove the default directory
637 testdirs = []
638
639 for ff in args.file:
640 if not os.path.isfile(ff):
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500641 print("IGNORING file " + ff + "\n\tBECAUSE does not exist.")
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500642 else:
643 flist.append(os.path.abspath(ff))
644
645 if args.directory:
646 testdirs = args.directory
647
648 for testdir in testdirs:
649 for root, dirnames, filenames in os.walk(testdir):
Lucas Bates76b903e2017-06-16 17:22:35 -0400650 for filename in fnmatch.filter(filenames, '*.json'):
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500651 candidate = os.path.abspath(os.path.join(root, filename))
652 if candidate not in testdirs:
653 flist.append(candidate)
654
655 alltestcases = list()
Lucas Bates76b903e2017-06-16 17:22:35 -0400656 for casefile in flist:
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500657 alltestcases = alltestcases + (load_from_file(casefile))
658
659 allcatlist = get_test_categories(alltestcases)
660 allidlist = get_id_list(alltestcases)
661
662 testcases_by_cats = get_categorized_testlist(alltestcases, allcatlist)
663 idtestcases = filter_tests_by_id(args, alltestcases)
664 cattestcases = filter_tests_by_category(args, alltestcases)
665
666 cat_ids = [x['id'] for x in cattestcases]
667 if args.execute:
668 if args.category:
669 alltestcases = cattestcases + [x for x in idtestcases if x['id'] not in cat_ids]
670 else:
671 alltestcases = idtestcases
672 else:
673 if cat_ids:
674 alltestcases = cattestcases
675 else:
676 # just accept the existing value of alltestcases,
677 # which has been filtered by file/directory
678 pass
679
680 return allcatlist, allidlist, testcases_by_cats, alltestcases
Lucas Bates76b903e2017-06-16 17:22:35 -0400681
682
Lucas Bates489ce2f2019-06-24 21:00:27 -0400683def set_operation_mode(pm, parser, args, remaining):
Lucas Bates76b903e2017-06-16 17:22:35 -0400684 """
685 Load the test case data and process remaining arguments to determine
686 what the script should do for this run, and call the appropriate
687 function.
688 """
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500689 ucat, idlist, testcases, alltests = get_test_cases(args)
Lucas Bates76b903e2017-06-16 17:22:35 -0400690
691 if args.gen_id:
Lucas Bates76b903e2017-06-16 17:22:35 -0400692 if (has_blank_ids(idlist)):
693 alltests = generate_case_ids(alltests)
694 else:
695 print("No empty ID fields found in test files.")
696 exit(0)
697
698 duplicate_ids = check_case_id(alltests)
699 if (len(duplicate_ids) > 0):
700 print("The following test case IDs are not unique:")
701 print(str(set(duplicate_ids)))
702 print("Please correct them before continuing.")
703 exit(1)
704
Lucas Bates76b903e2017-06-16 17:22:35 -0400705 if args.showID:
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500706 for atest in alltests:
707 print_test_case(atest)
Lucas Bates76b903e2017-06-16 17:22:35 -0400708 exit(0)
709
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500710 if isinstance(args.category, list) and (len(args.category) == 0):
711 print("Available categories:")
712 print_sll(ucat)
713 exit(0)
Lucas Bates76b903e2017-06-16 17:22:35 -0400714
715 if args.list:
Brenda J. Butlerf87c7f62018-02-14 14:09:19 -0500716 if args.list:
Lucas Bates76b903e2017-06-16 17:22:35 -0400717 list_test_cases(alltests)
718 exit(0)
Lucas Bates76b903e2017-06-16 17:22:35 -0400719
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500720 if len(alltests):
Lucas Bates489ce2f2019-06-24 21:00:27 -0400721 req_plugins = pm.get_required_plugins(alltests)
722 try:
723 args = pm.load_required_plugins(req_plugins, parser, args, remaining)
724 except PluginDependencyException as pde:
725 print('The following plugins were not found:')
726 print('{}'.format(pde.missing_pg))
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500727 catresults = test_runner(pm, args, alltests)
Lucas Bates915c1582018-12-06 17:42:26 -0500728 if args.format == 'none':
729 print('Test results output suppression requested\n')
730 else:
731 print('\nAll test results: \n')
732 if args.format == 'xunit':
733 suffix = 'xml'
734 res = catresults.format_xunit()
735 elif args.format == 'tap':
736 suffix = 'tap'
737 res = catresults.format_tap()
738 print(res)
739 print('\n\n')
740 if not args.outfile:
741 fname = 'test-results.{}'.format(suffix)
742 else:
743 fname = args.outfile
744 with open(fname, 'w') as fh:
745 fh.write(res)
746 fh.close()
747 if os.getenv('SUDO_UID') is not None:
748 os.chown(fname, uid=int(os.getenv('SUDO_UID')),
749 gid=int(os.getenv('SUDO_GID')))
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500750 else:
Lucas Bates915c1582018-12-06 17:42:26 -0500751 print('No tests found\n')
Lucas Bates76b903e2017-06-16 17:22:35 -0400752
Lucas Bates76b903e2017-06-16 17:22:35 -0400753def main():
754 """
755 Start of execution; set up argument parser and get the arguments,
756 and start operations.
757 """
758 parser = args_parse()
759 parser = set_args(parser)
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500760 pm = PluginMgr(parser)
761 parser = pm.call_add_args(parser)
Lucas Bates76b903e2017-06-16 17:22:35 -0400762 (args, remaining) = parser.parse_known_args()
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500763 args.NAMES = NAMES
Lucas Bates489ce2f2019-06-24 21:00:27 -0400764 pm.set_args(args)
Brenda J. Butler93707cb2018-02-14 14:09:21 -0500765 check_default_settings(args, remaining, pm)
766 if args.verbose > 2:
767 print('args is {}'.format(args))
Lucas Bates76b903e2017-06-16 17:22:35 -0400768
Lucas Bates489ce2f2019-06-24 21:00:27 -0400769 set_operation_mode(pm, parser, args, remaining)
Lucas Bates76b903e2017-06-16 17:22:35 -0400770
771 exit(0)
772
773
774if __name__ == "__main__":
775 main()