Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Thomas Gleixner | 4f19048 | 2019-05-27 08:55:14 +0200 | [diff] [blame] | 2 | # SPDX-License-Identifier: GPL-2.0-only |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 3 | |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 4 | """Find Kconfig symbols that are referenced but not defined.""" |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 5 | |
Valentin Rothberg | 8e8e333 | 2017-01-18 13:08:19 +0100 | [diff] [blame] | 6 | # (c) 2014-2017 Valentin Rothberg <valentinrothberg@gmail.com> |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 7 | # (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de> |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 8 | # |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 9 | |
| 10 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 11 | import argparse |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 12 | import difflib |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 13 | import os |
| 14 | import re |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 15 | import signal |
Valentin Rothberg | f175ba1 | 2016-08-27 10:59:07 +0200 | [diff] [blame] | 16 | import subprocess |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 17 | import sys |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 18 | from multiprocessing import Pool, cpu_count |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 19 | |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 20 | |
| 21 | # regex expressions |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 22 | OPERATORS = r"&|\(|\)|\||\!" |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 23 | SYMBOL = r"(?:\w*[A-Z0-9]\w*){2,}" |
| 24 | DEF = r"^\s*(?:menu){,1}config\s+(" + SYMBOL + r")\s*" |
| 25 | EXPR = r"(?:" + OPERATORS + r"|\s|" + SYMBOL + r")+" |
Valentin Rothberg | 0bd38ae | 2015-07-27 12:33:05 +0200 | [diff] [blame] | 26 | DEFAULT = r"default\s+.*?(?:if\s.+){,1}" |
Valentin Rothberg | 3b28f4f | 2017-02-02 18:00:44 +0100 | [diff] [blame] | 27 | STMT = r"^\s*(?:if|select|imply|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 28 | SOURCE_SYMBOL = r"(?:\W|\b)+[D]{,1}CONFIG_(" + SYMBOL + r")" |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 29 | |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 30 | # regex objects |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 31 | REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$") |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 32 | REGEX_SYMBOL = re.compile(r'(?!\B)' + SYMBOL + r'(?!\B)') |
| 33 | REGEX_SOURCE_SYMBOL = re.compile(SOURCE_SYMBOL) |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 34 | REGEX_KCONFIG_DEF = re.compile(DEF) |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 35 | REGEX_KCONFIG_EXPR = re.compile(EXPR) |
| 36 | REGEX_KCONFIG_STMT = re.compile(STMT) |
| 37 | REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$") |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 38 | REGEX_FILTER_SYMBOLS = re.compile(r"[A-Za-z0-9]$") |
Valentin Rothberg | 0bd38ae | 2015-07-27 12:33:05 +0200 | [diff] [blame] | 39 | REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+") |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 40 | REGEX_QUOTES = re.compile("(\"(.*?)\")") |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 41 | |
| 42 | |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 43 | def parse_options(): |
| 44 | """The user interface of this module.""" |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 45 | usage = "Run this tool to detect Kconfig symbols that are referenced but " \ |
| 46 | "not defined in Kconfig. If no option is specified, " \ |
| 47 | "checkkconfigsymbols defaults to check your current tree. " \ |
| 48 | "Please note that specifying commits will 'git reset --hard\' " \ |
| 49 | "your current tree! You may save uncommitted changes to avoid " \ |
| 50 | "losing data." |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 51 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 52 | parser = argparse.ArgumentParser(description=usage) |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 53 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 54 | parser.add_argument('-c', '--commit', dest='commit', action='store', |
| 55 | default="", |
| 56 | help="check if the specified commit (hash) introduces " |
| 57 | "undefined Kconfig symbols") |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 58 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 59 | parser.add_argument('-d', '--diff', dest='diff', action='store', |
| 60 | default="", |
| 61 | help="diff undefined symbols between two commits " |
| 62 | "(e.g., -d commmit1..commit2)") |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 63 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 64 | parser.add_argument('-f', '--find', dest='find', action='store_true', |
| 65 | default=False, |
| 66 | help="find and show commits that may cause symbols to be " |
| 67 | "missing (required to run with --diff)") |
Valentin Rothberg | a42fa92 | 2015-06-01 16:00:19 +0200 | [diff] [blame] | 68 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 69 | parser.add_argument('-i', '--ignore', dest='ignore', action='store', |
| 70 | default="", |
| 71 | help="ignore files matching this Python regex " |
| 72 | "(e.g., -i '.*defconfig')") |
Valentin Rothberg | cf132e4 | 2015-04-29 16:58:27 +0200 | [diff] [blame] | 73 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 74 | parser.add_argument('-s', '--sim', dest='sim', action='store', default="", |
| 75 | help="print a list of max. 10 string-similar symbols") |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 76 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 77 | parser.add_argument('--force', dest='force', action='store_true', |
| 78 | default=False, |
| 79 | help="reset current Git tree even when it's dirty") |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 80 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 81 | parser.add_argument('--no-color', dest='color', action='store_false', |
| 82 | default=True, |
| 83 | help="don't print colored output (default when not " |
| 84 | "outputting to a terminal)") |
Andrew Donnellan | 4c73c08 | 2016-07-05 17:47:37 +1000 | [diff] [blame] | 85 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 86 | args = parser.parse_args() |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 87 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 88 | if args.commit and args.diff: |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 89 | sys.exit("Please specify only one option at once.") |
| 90 | |
Valentin Rothberg | 0d18c19 | 2016-10-27 14:34:57 +0200 | [diff] [blame] | 91 | if args.diff and not re.match(r"^[\w\-\.\^]+\.\.[\w\-\.\^]+$", args.diff): |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 92 | sys.exit("Please specify valid input in the following format: " |
Andreas Ziegler | 38cbfe4 | 2016-03-31 09:24:29 +0200 | [diff] [blame] | 93 | "\'commit1..commit2\'") |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 94 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 95 | if args.commit or args.diff: |
| 96 | if not args.force and tree_is_dirty(): |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 97 | sys.exit("The current Git tree is dirty (see 'git status'). " |
| 98 | "Running this script may\ndelete important data since it " |
| 99 | "calls 'git reset --hard' for some performance\nreasons. " |
| 100 | " Please run this script in a clean Git tree or pass " |
| 101 | "'--force' if you\nwant to ignore this warning and " |
| 102 | "continue.") |
| 103 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 104 | if args.commit: |
| 105 | args.find = False |
Valentin Rothberg | a42fa92 | 2015-06-01 16:00:19 +0200 | [diff] [blame] | 106 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 107 | if args.ignore: |
Valentin Rothberg | cf132e4 | 2015-04-29 16:58:27 +0200 | [diff] [blame] | 108 | try: |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 109 | re.match(args.ignore, "this/is/just/a/test.c") |
Valentin Rothberg | cf132e4 | 2015-04-29 16:58:27 +0200 | [diff] [blame] | 110 | except: |
| 111 | sys.exit("Please specify a valid Python regex.") |
| 112 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 113 | return args |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 114 | |
| 115 | |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 116 | def main(): |
| 117 | """Main function of this module.""" |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 118 | args = parse_options() |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 119 | |
Valentin Rothberg | 36c79c7f | 2016-08-28 08:51:30 +0200 | [diff] [blame] | 120 | global COLOR |
| 121 | COLOR = args.color and sys.stdout.isatty() |
Andrew Donnellan | 4c73c08 | 2016-07-05 17:47:37 +1000 | [diff] [blame] | 122 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 123 | if args.sim and not args.commit and not args.diff: |
| 124 | sims = find_sims(args.sim, args.ignore) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 125 | if sims: |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 126 | print("%s: %s" % (yel("Similar symbols"), ', '.join(sims))) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 127 | else: |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 128 | print("%s: no similar symbols found" % yel("Similar symbols")) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 129 | sys.exit(0) |
| 130 | |
| 131 | # dictionary of (un)defined symbols |
| 132 | defined = {} |
| 133 | undefined = {} |
| 134 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 135 | if args.commit or args.diff: |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 136 | head = get_head() |
| 137 | |
| 138 | # get commit range |
| 139 | commit_a = None |
| 140 | commit_b = None |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 141 | if args.commit: |
| 142 | commit_a = args.commit + "~" |
| 143 | commit_b = args.commit |
| 144 | elif args.diff: |
| 145 | split = args.diff.split("..") |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 146 | commit_a = split[0] |
| 147 | commit_b = split[1] |
| 148 | undefined_a = {} |
| 149 | undefined_b = {} |
| 150 | |
| 151 | # get undefined items before the commit |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 152 | reset(commit_a) |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 153 | undefined_a, _ = check_symbols(args.ignore) |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 154 | |
| 155 | # get undefined items for the commit |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 156 | reset(commit_b) |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 157 | undefined_b, defined = check_symbols(args.ignore) |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 158 | |
| 159 | # report cases that are present for the commit but not before |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 160 | for symbol in sorted(undefined_b): |
| 161 | # symbol has not been undefined before |
| 162 | if symbol not in undefined_a: |
| 163 | files = sorted(undefined_b.get(symbol)) |
| 164 | undefined[symbol] = files |
| 165 | # check if there are new files that reference the undefined symbol |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 166 | else: |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 167 | files = sorted(undefined_b.get(symbol) - |
| 168 | undefined_a.get(symbol)) |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 169 | if files: |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 170 | undefined[symbol] = files |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 171 | |
| 172 | # reset to head |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 173 | reset(head) |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 174 | |
| 175 | # default to check the entire tree |
| 176 | else: |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 177 | undefined, defined = check_symbols(args.ignore) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 178 | |
| 179 | # now print the output |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 180 | for symbol in sorted(undefined): |
| 181 | print(red(symbol)) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 182 | |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 183 | files = sorted(undefined.get(symbol)) |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 184 | print("%s: %s" % (yel("Referencing files"), ", ".join(files))) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 185 | |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 186 | sims = find_sims(symbol, args.ignore, defined) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 187 | sims_out = yel("Similar symbols") |
| 188 | if sims: |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 189 | print("%s: %s" % (sims_out, ', '.join(sims))) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 190 | else: |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 191 | print("%s: %s" % (sims_out, "no similar symbols found")) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 192 | |
Valentin Rothberg | 14390e3 | 2016-08-28 08:51:29 +0200 | [diff] [blame] | 193 | if args.find: |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 194 | print("%s:" % yel("Commits changing symbol")) |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 195 | commits = find_commits(symbol, args.diff) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 196 | if commits: |
| 197 | for commit in commits: |
| 198 | commit = commit.split(" ", 1) |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 199 | print("\t- %s (\"%s\")" % (yel(commit[0]), commit[1])) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 200 | else: |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 201 | print("\t- no commit found") |
Valentin Rothberg | 36c79c7f | 2016-08-28 08:51:30 +0200 | [diff] [blame] | 202 | print() # new line |
Valentin Rothberg | c745566 | 2015-06-01 16:00:20 +0200 | [diff] [blame] | 203 | |
| 204 | |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 205 | def reset(commit): |
| 206 | """Reset current git tree to %commit.""" |
| 207 | execute(["git", "reset", "--hard", commit]) |
| 208 | |
| 209 | |
Valentin Rothberg | c745566 | 2015-06-01 16:00:20 +0200 | [diff] [blame] | 210 | def yel(string): |
| 211 | """ |
| 212 | Color %string yellow. |
| 213 | """ |
Valentin Rothberg | 36c79c7f | 2016-08-28 08:51:30 +0200 | [diff] [blame] | 214 | return "\033[33m%s\033[0m" % string if COLOR else string |
Valentin Rothberg | c745566 | 2015-06-01 16:00:20 +0200 | [diff] [blame] | 215 | |
| 216 | |
| 217 | def red(string): |
| 218 | """ |
| 219 | Color %string red. |
| 220 | """ |
Valentin Rothberg | 36c79c7f | 2016-08-28 08:51:30 +0200 | [diff] [blame] | 221 | return "\033[31m%s\033[0m" % string if COLOR else string |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 222 | |
| 223 | |
| 224 | def execute(cmd): |
| 225 | """Execute %cmd and return stdout. Exit in case of error.""" |
Valentin Rothberg | f175ba1 | 2016-08-27 10:59:07 +0200 | [diff] [blame] | 226 | try: |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 227 | stdout = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False) |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 228 | stdout = stdout.decode(errors='replace') |
Valentin Rothberg | f175ba1 | 2016-08-27 10:59:07 +0200 | [diff] [blame] | 229 | except subprocess.CalledProcessError as fail: |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 230 | exit(fail) |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 231 | return stdout |
| 232 | |
| 233 | |
Valentin Rothberg | a42fa92 | 2015-06-01 16:00:19 +0200 | [diff] [blame] | 234 | def find_commits(symbol, diff): |
| 235 | """Find commits changing %symbol in the given range of %diff.""" |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 236 | commits = execute(["git", "log", "--pretty=oneline", |
| 237 | "--abbrev-commit", "-G", |
| 238 | symbol, diff]) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 239 | return [x for x in commits.split("\n") if x] |
Valentin Rothberg | a42fa92 | 2015-06-01 16:00:19 +0200 | [diff] [blame] | 240 | |
| 241 | |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 242 | def tree_is_dirty(): |
| 243 | """Return true if the current working tree is dirty (i.e., if any file has |
| 244 | been added, deleted, modified, renamed or copied but not committed).""" |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 245 | stdout = execute(["git", "status", "--porcelain"]) |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 246 | for line in stdout: |
| 247 | if re.findall(r"[URMADC]{1}", line[:2]): |
| 248 | return True |
| 249 | return False |
| 250 | |
| 251 | |
| 252 | def get_head(): |
| 253 | """Return commit hash of current HEAD.""" |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 254 | stdout = execute(["git", "rev-parse", "HEAD"]) |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 255 | return stdout.strip('\n') |
| 256 | |
| 257 | |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 258 | def partition(lst, size): |
| 259 | """Partition list @lst into eveni-sized lists of size @size.""" |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 260 | return [lst[i::size] for i in range(size)] |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 261 | |
| 262 | |
| 263 | def init_worker(): |
| 264 | """Set signal handler to ignore SIGINT.""" |
| 265 | signal.signal(signal.SIGINT, signal.SIG_IGN) |
| 266 | |
| 267 | |
Valentin Rothberg | 36c79c7f | 2016-08-28 08:51:30 +0200 | [diff] [blame] | 268 | def find_sims(symbol, ignore, defined=[]): |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 269 | """Return a list of max. ten Kconfig symbols that are string-similar to |
| 270 | @symbol.""" |
| 271 | if defined: |
Valentin Rothberg | 8e8e333 | 2017-01-18 13:08:19 +0100 | [diff] [blame] | 272 | return difflib.get_close_matches(symbol, set(defined), 10) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 273 | |
| 274 | pool = Pool(cpu_count(), init_worker) |
| 275 | kfiles = [] |
| 276 | for gitfile in get_files(): |
| 277 | if REGEX_FILE_KCONFIG.match(gitfile): |
| 278 | kfiles.append(gitfile) |
| 279 | |
| 280 | arglist = [] |
| 281 | for part in partition(kfiles, cpu_count()): |
| 282 | arglist.append((part, ignore)) |
| 283 | |
| 284 | for res in pool.map(parse_kconfig_files, arglist): |
| 285 | defined.extend(res[0]) |
| 286 | |
Valentin Rothberg | 8e8e333 | 2017-01-18 13:08:19 +0100 | [diff] [blame] | 287 | return difflib.get_close_matches(symbol, set(defined), 10) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 288 | |
| 289 | |
| 290 | def get_files(): |
| 291 | """Return a list of all files in the current git directory.""" |
| 292 | # use 'git ls-files' to get the worklist |
Valentin Rothberg | 2f9cc12 | 2016-08-28 08:51:32 +0200 | [diff] [blame] | 293 | stdout = execute(["git", "ls-files"]) |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 294 | if len(stdout) > 0 and stdout[-1] == "\n": |
| 295 | stdout = stdout[:-1] |
| 296 | |
| 297 | files = [] |
| 298 | for gitfile in stdout.rsplit("\n"): |
| 299 | if ".git" in gitfile or "ChangeLog" in gitfile or \ |
| 300 | ".log" in gitfile or os.path.isdir(gitfile) or \ |
| 301 | gitfile.startswith("tools/"): |
| 302 | continue |
| 303 | files.append(gitfile) |
| 304 | return files |
| 305 | |
| 306 | |
Valentin Rothberg | cf132e4 | 2015-04-29 16:58:27 +0200 | [diff] [blame] | 307 | def check_symbols(ignore): |
Valentin Rothberg | b1a3f24 | 2015-03-16 12:16:14 +0100 | [diff] [blame] | 308 | """Find undefined Kconfig symbols and return a dict with the symbol as key |
Valentin Rothberg | cf132e4 | 2015-04-29 16:58:27 +0200 | [diff] [blame] | 309 | and a list of referencing files as value. Files matching %ignore are not |
| 310 | checked for undefined symbols.""" |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 311 | pool = Pool(cpu_count(), init_worker) |
| 312 | try: |
| 313 | return check_symbols_helper(pool, ignore) |
| 314 | except KeyboardInterrupt: |
| 315 | pool.terminate() |
| 316 | pool.join() |
| 317 | sys.exit(1) |
| 318 | |
| 319 | |
| 320 | def check_symbols_helper(pool, ignore): |
| 321 | """Helper method for check_symbols(). Used to catch keyboard interrupts in |
| 322 | check_symbols() in order to properly terminate running worker processes.""" |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 323 | source_files = [] |
| 324 | kconfig_files = [] |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 325 | defined_symbols = [] |
| 326 | referenced_symbols = dict() # {file: [symbols]} |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 327 | |
Valentin Rothberg | 1b2c841 | 2015-11-26 14:17:15 +0100 | [diff] [blame] | 328 | for gitfile in get_files(): |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 329 | if REGEX_FILE_KCONFIG.match(gitfile): |
| 330 | kconfig_files.append(gitfile) |
| 331 | else: |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 332 | if ignore and not re.match(ignore, gitfile): |
| 333 | continue |
| 334 | # add source files that do not match the ignore pattern |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 335 | source_files.append(gitfile) |
| 336 | |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 337 | # parse source files |
| 338 | arglist = partition(source_files, cpu_count()) |
| 339 | for res in pool.map(parse_source_files, arglist): |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 340 | referenced_symbols.update(res) |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 341 | |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 342 | # parse kconfig files |
| 343 | arglist = [] |
| 344 | for part in partition(kconfig_files, cpu_count()): |
| 345 | arglist.append((part, ignore)) |
| 346 | for res in pool.map(parse_kconfig_files, arglist): |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 347 | defined_symbols.extend(res[0]) |
| 348 | referenced_symbols.update(res[1]) |
| 349 | defined_symbols = set(defined_symbols) |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 350 | |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 351 | # inverse mapping of referenced_symbols to dict(symbol: [files]) |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 352 | inv_map = dict() |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 353 | for _file, symbols in referenced_symbols.items(): |
| 354 | for symbol in symbols: |
| 355 | inv_map[symbol] = inv_map.get(symbol, set()) |
| 356 | inv_map[symbol].add(_file) |
| 357 | referenced_symbols = inv_map |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 358 | |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 359 | undefined = {} # {symbol: [files]} |
| 360 | for symbol in sorted(referenced_symbols): |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 361 | # filter some false positives |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 362 | if symbol == "FOO" or symbol == "BAR" or \ |
| 363 | symbol == "FOO_BAR" or symbol == "XXX": |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 364 | continue |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 365 | if symbol not in defined_symbols: |
| 366 | if symbol.endswith("_MODULE"): |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 367 | # avoid false positives for kernel modules |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 368 | if symbol[:-len("_MODULE")] in defined_symbols: |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 369 | continue |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 370 | undefined[symbol] = referenced_symbols.get(symbol) |
| 371 | return undefined, defined_symbols |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 372 | |
| 373 | |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 374 | def parse_source_files(source_files): |
| 375 | """Parse each source file in @source_files and return dictionary with source |
| 376 | files as keys and lists of references Kconfig symbols as values.""" |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 377 | referenced_symbols = dict() |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 378 | for sfile in source_files: |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 379 | referenced_symbols[sfile] = parse_source_file(sfile) |
| 380 | return referenced_symbols |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 381 | |
| 382 | |
| 383 | def parse_source_file(sfile): |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 384 | """Parse @sfile and return a list of referenced Kconfig symbols.""" |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 385 | lines = [] |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 386 | references = [] |
| 387 | |
| 388 | if not os.path.exists(sfile): |
| 389 | return references |
| 390 | |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 391 | with open(sfile, "r", encoding='utf-8', errors='replace') as stream: |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 392 | lines = stream.readlines() |
| 393 | |
| 394 | for line in lines: |
Valentin Rothberg | 36c79c7f | 2016-08-28 08:51:30 +0200 | [diff] [blame] | 395 | if "CONFIG_" not in line: |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 396 | continue |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 397 | symbols = REGEX_SOURCE_SYMBOL.findall(line) |
| 398 | for symbol in symbols: |
| 399 | if not REGEX_FILTER_SYMBOLS.search(symbol): |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 400 | continue |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 401 | references.append(symbol) |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 402 | |
| 403 | return references |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 404 | |
| 405 | |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 406 | def get_symbols_in_line(line): |
| 407 | """Return mentioned Kconfig symbols in @line.""" |
| 408 | return REGEX_SYMBOL.findall(line) |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 409 | |
| 410 | |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 411 | def parse_kconfig_files(args): |
| 412 | """Parse kconfig files and return tuple of defined and references Kconfig |
| 413 | symbols. Note, @args is a tuple of a list of files and the @ignore |
| 414 | pattern.""" |
| 415 | kconfig_files = args[0] |
| 416 | ignore = args[1] |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 417 | defined_symbols = [] |
| 418 | referenced_symbols = dict() |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 419 | |
| 420 | for kfile in kconfig_files: |
| 421 | defined, references = parse_kconfig_file(kfile) |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 422 | defined_symbols.extend(defined) |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 423 | if ignore and re.match(ignore, kfile): |
| 424 | # do not collect references for files that match the ignore pattern |
| 425 | continue |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 426 | referenced_symbols[kfile] = references |
| 427 | return (defined_symbols, referenced_symbols) |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 428 | |
| 429 | |
| 430 | def parse_kconfig_file(kfile): |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 431 | """Parse @kfile and update symbol definitions and references.""" |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 432 | lines = [] |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 433 | defined = [] |
| 434 | references = [] |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 435 | skip = False |
| 436 | |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 437 | if not os.path.exists(kfile): |
| 438 | return defined, references |
| 439 | |
Valentin Rothberg | 7c5227a | 2016-08-28 08:51:28 +0200 | [diff] [blame] | 440 | with open(kfile, "r", encoding='utf-8', errors='replace') as stream: |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 441 | lines = stream.readlines() |
| 442 | |
| 443 | for i in range(len(lines)): |
| 444 | line = lines[i] |
| 445 | line = line.strip('\n') |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 446 | line = line.split("#")[0] # ignore comments |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 447 | |
| 448 | if REGEX_KCONFIG_DEF.match(line): |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 449 | symbol_def = REGEX_KCONFIG_DEF.findall(line) |
| 450 | defined.append(symbol_def[0]) |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 451 | skip = False |
| 452 | elif REGEX_KCONFIG_HELP.match(line): |
| 453 | skip = True |
| 454 | elif skip: |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 455 | # ignore content of help messages |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 456 | pass |
| 457 | elif REGEX_KCONFIG_STMT.match(line): |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 458 | line = REGEX_QUOTES.sub("", line) |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 459 | symbols = get_symbols_in_line(line) |
Valentin Rothberg | cc641d55 | 2014-11-08 20:56:35 +0100 | [diff] [blame] | 460 | # multi-line statements |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 461 | while line.endswith("\\"): |
| 462 | i += 1 |
| 463 | line = lines[i] |
| 464 | line = line.strip('\n') |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 465 | symbols.extend(get_symbols_in_line(line)) |
| 466 | for symbol in set(symbols): |
| 467 | if REGEX_NUMERIC.match(symbol): |
Valentin Rothberg | 0bd38ae | 2015-07-27 12:33:05 +0200 | [diff] [blame] | 468 | # ignore numeric values |
| 469 | continue |
Valentin Rothberg | ef3f554 | 2016-08-28 08:51:31 +0200 | [diff] [blame] | 470 | references.append(symbol) |
Valentin Rothberg | e2042a8 | 2015-10-15 10:37:47 +0200 | [diff] [blame] | 471 | |
| 472 | return defined, references |
Valentin Rothberg | 24fe1f0 | 2014-09-27 16:30:45 +0200 | [diff] [blame] | 473 | |
| 474 | |
| 475 | if __name__ == "__main__": |
| 476 | main() |