Thomas Gleixner | 4317cf95 | 2019-05-31 01:09:38 -0700 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 2 | /* |
Shile Zhang | 1091670 | 2019-12-04 08:46:31 +0800 | [diff] [blame] | 3 | * sorttable.h |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 4 | * |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 5 | * Added ORC unwind tables sort support and other updates: |
| 6 | * Copyright (C) 1999-2019 Alibaba Group Holding Limited. by: |
| 7 | * Shile Zhang <shile.zhang@linux.alibaba.com> |
| 8 | * |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 9 | * Copyright 2011 - 2012 Cavium, Inc. |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 10 | * |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 11 | * Some of code was taken out of arch/x86/kernel/unwind_orc.c, written by: |
| 12 | * Copyright (C) 2017 Josh Poimboeuf <jpoimboe@redhat.com> |
| 13 | * |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 14 | * Some of this code was taken out of recordmcount.h written by: |
| 15 | * |
Shile Zhang | 6402e14 | 2019-12-04 08:46:28 +0800 | [diff] [blame] | 16 | * Copyright 2009 John F. Reiser <jreiser@BitWagon.com>. All rights reserved. |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 17 | * Copyright 2010 Steven Rostedt <srostedt@redhat.com>, Red Hat Inc. |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 18 | */ |
| 19 | |
| 20 | #undef extable_ent_size |
| 21 | #undef compare_extable |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 22 | #undef get_mcount_loc |
| 23 | #undef sort_mcount_loc |
| 24 | #undef elf_mcount_loc |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 25 | #undef do_sort |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 26 | #undef Elf_Addr |
| 27 | #undef Elf_Ehdr |
| 28 | #undef Elf_Shdr |
| 29 | #undef Elf_Rel |
| 30 | #undef Elf_Rela |
| 31 | #undef Elf_Sym |
| 32 | #undef ELF_R_SYM |
| 33 | #undef Elf_r_sym |
| 34 | #undef ELF_R_INFO |
| 35 | #undef Elf_r_info |
| 36 | #undef ELF_ST_BIND |
| 37 | #undef ELF_ST_TYPE |
| 38 | #undef fn_ELF_R_SYM |
| 39 | #undef fn_ELF_R_INFO |
| 40 | #undef uint_t |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 41 | #undef _r |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 42 | #undef _w |
| 43 | |
Shile Zhang | 1091670 | 2019-12-04 08:46:31 +0800 | [diff] [blame] | 44 | #ifdef SORTTABLE_64 |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 45 | # define extable_ent_size 16 |
| 46 | # define compare_extable compare_extable_64 |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 47 | # define get_mcount_loc get_mcount_loc_64 |
| 48 | # define sort_mcount_loc sort_mcount_loc_64 |
| 49 | # define elf_mcount_loc elf_mcount_loc_64 |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 50 | # define do_sort do_sort_64 |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 51 | # define Elf_Addr Elf64_Addr |
| 52 | # define Elf_Ehdr Elf64_Ehdr |
| 53 | # define Elf_Shdr Elf64_Shdr |
| 54 | # define Elf_Rel Elf64_Rel |
| 55 | # define Elf_Rela Elf64_Rela |
| 56 | # define Elf_Sym Elf64_Sym |
| 57 | # define ELF_R_SYM ELF64_R_SYM |
| 58 | # define Elf_r_sym Elf64_r_sym |
| 59 | # define ELF_R_INFO ELF64_R_INFO |
| 60 | # define Elf_r_info Elf64_r_info |
| 61 | # define ELF_ST_BIND ELF64_ST_BIND |
| 62 | # define ELF_ST_TYPE ELF64_ST_TYPE |
| 63 | # define fn_ELF_R_SYM fn_ELF64_R_SYM |
| 64 | # define fn_ELF_R_INFO fn_ELF64_R_INFO |
| 65 | # define uint_t uint64_t |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 66 | # define _r r8 |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 67 | # define _w w8 |
| 68 | #else |
| 69 | # define extable_ent_size 8 |
| 70 | # define compare_extable compare_extable_32 |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 71 | # define get_mcount_loc get_mcount_loc_32 |
| 72 | # define sort_mcount_loc sort_mcount_loc_32 |
| 73 | # define elf_mcount_loc elf_mcount_loc_32 |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 74 | # define do_sort do_sort_32 |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 75 | # define Elf_Addr Elf32_Addr |
| 76 | # define Elf_Ehdr Elf32_Ehdr |
| 77 | # define Elf_Shdr Elf32_Shdr |
| 78 | # define Elf_Rel Elf32_Rel |
| 79 | # define Elf_Rela Elf32_Rela |
| 80 | # define Elf_Sym Elf32_Sym |
| 81 | # define ELF_R_SYM ELF32_R_SYM |
| 82 | # define Elf_r_sym Elf32_r_sym |
| 83 | # define ELF_R_INFO ELF32_R_INFO |
| 84 | # define Elf_r_info Elf32_r_info |
| 85 | # define ELF_ST_BIND ELF32_ST_BIND |
| 86 | # define ELF_ST_TYPE ELF32_ST_TYPE |
| 87 | # define fn_ELF_R_SYM fn_ELF32_R_SYM |
| 88 | # define fn_ELF_R_INFO fn_ELF32_R_INFO |
| 89 | # define uint_t uint32_t |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 90 | # define _r r |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 91 | # define _w w |
| 92 | #endif |
| 93 | |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 94 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
| 95 | /* ORC unwinder only support X86_64 */ |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 96 | #include <asm/orc_types.h> |
| 97 | |
| 98 | #define ERRSTR_MAXSZ 256 |
| 99 | |
| 100 | char g_err[ERRSTR_MAXSZ]; |
| 101 | int *g_orc_ip_table; |
| 102 | struct orc_entry *g_orc_table; |
| 103 | |
| 104 | pthread_t orc_sort_thread; |
| 105 | |
| 106 | static inline unsigned long orc_ip(const int *ip) |
| 107 | { |
| 108 | return (unsigned long)ip + *ip; |
| 109 | } |
| 110 | |
| 111 | static int orc_sort_cmp(const void *_a, const void *_b) |
| 112 | { |
| 113 | struct orc_entry *orc_a; |
| 114 | const int *a = g_orc_ip_table + *(int *)_a; |
| 115 | const int *b = g_orc_ip_table + *(int *)_b; |
| 116 | unsigned long a_val = orc_ip(a); |
| 117 | unsigned long b_val = orc_ip(b); |
| 118 | |
| 119 | if (a_val > b_val) |
| 120 | return 1; |
| 121 | if (a_val < b_val) |
| 122 | return -1; |
| 123 | |
| 124 | /* |
| 125 | * The "weak" section terminator entries need to always be on the left |
| 126 | * to ensure the lookup code skips them in favor of real entries. |
| 127 | * These terminator entries exist to handle any gaps created by |
| 128 | * whitelisted .o files which didn't get objtool generation. |
| 129 | */ |
| 130 | orc_a = g_orc_table + (a - g_orc_ip_table); |
Josh Poimboeuf | fb799447 | 2023-03-01 07:13:12 -0800 | [diff] [blame] | 131 | return orc_a->type == ORC_TYPE_UNDEFINED ? -1 : 1; |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 132 | } |
| 133 | |
| 134 | static void *sort_orctable(void *arg) |
| 135 | { |
| 136 | int i; |
| 137 | int *idxs = NULL; |
| 138 | int *tmp_orc_ip_table = NULL; |
| 139 | struct orc_entry *tmp_orc_table = NULL; |
| 140 | unsigned int *orc_ip_size = (unsigned int *)arg; |
| 141 | unsigned int num_entries = *orc_ip_size / sizeof(int); |
| 142 | unsigned int orc_size = num_entries * sizeof(struct orc_entry); |
| 143 | |
| 144 | idxs = (int *)malloc(*orc_ip_size); |
| 145 | if (!idxs) { |
| 146 | snprintf(g_err, ERRSTR_MAXSZ, "malloc idxs: %s", |
| 147 | strerror(errno)); |
| 148 | pthread_exit(g_err); |
| 149 | } |
| 150 | |
| 151 | tmp_orc_ip_table = (int *)malloc(*orc_ip_size); |
| 152 | if (!tmp_orc_ip_table) { |
| 153 | snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_ip_table: %s", |
| 154 | strerror(errno)); |
| 155 | pthread_exit(g_err); |
| 156 | } |
| 157 | |
| 158 | tmp_orc_table = (struct orc_entry *)malloc(orc_size); |
| 159 | if (!tmp_orc_table) { |
| 160 | snprintf(g_err, ERRSTR_MAXSZ, "malloc tmp_orc_table: %s", |
| 161 | strerror(errno)); |
| 162 | pthread_exit(g_err); |
| 163 | } |
| 164 | |
| 165 | /* initialize indices array, convert ip_table to absolute address */ |
| 166 | for (i = 0; i < num_entries; i++) { |
| 167 | idxs[i] = i; |
| 168 | tmp_orc_ip_table[i] = g_orc_ip_table[i] + i * sizeof(int); |
| 169 | } |
| 170 | memcpy(tmp_orc_table, g_orc_table, orc_size); |
| 171 | |
| 172 | qsort(idxs, num_entries, sizeof(int), orc_sort_cmp); |
| 173 | |
| 174 | for (i = 0; i < num_entries; i++) { |
| 175 | if (idxs[i] == i) |
| 176 | continue; |
| 177 | |
| 178 | /* convert back to relative address */ |
| 179 | g_orc_ip_table[i] = tmp_orc_ip_table[idxs[i]] - i * sizeof(int); |
| 180 | g_orc_table[i] = tmp_orc_table[idxs[i]]; |
| 181 | } |
| 182 | |
| 183 | free(idxs); |
| 184 | free(tmp_orc_ip_table); |
| 185 | free(tmp_orc_table); |
| 186 | pthread_exit(NULL); |
| 187 | } |
| 188 | #endif |
| 189 | |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 190 | static int compare_extable(const void *a, const void *b) |
| 191 | { |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 192 | Elf_Addr av = _r(a); |
| 193 | Elf_Addr bv = _r(b); |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 194 | |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 195 | if (av < bv) |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 196 | return -1; |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 197 | if (av > bv) |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 198 | return 1; |
| 199 | return 0; |
| 200 | } |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 201 | #ifdef MCOUNT_SORT_ENABLED |
Yinan Liu | 35140d3 | 2022-01-18 14:52:41 +0800 | [diff] [blame] | 202 | pthread_t mcount_sort_thread; |
| 203 | |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 204 | struct elf_mcount_loc { |
| 205 | Elf_Ehdr *ehdr; |
| 206 | Elf_Shdr *init_data_sec; |
| 207 | uint_t start_mcount_loc; |
| 208 | uint_t stop_mcount_loc; |
| 209 | }; |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 210 | |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 211 | /* Sort the addresses stored between __start_mcount_loc to __stop_mcount_loc in vmlinux */ |
| 212 | static void *sort_mcount_loc(void *arg) |
| 213 | { |
| 214 | struct elf_mcount_loc *emloc = (struct elf_mcount_loc *)arg; |
| 215 | uint_t offset = emloc->start_mcount_loc - _r(&(emloc->init_data_sec)->sh_addr) |
| 216 | + _r(&(emloc->init_data_sec)->sh_offset); |
| 217 | uint_t count = emloc->stop_mcount_loc - emloc->start_mcount_loc; |
| 218 | unsigned char *start_loc = (void *)emloc->ehdr + offset; |
| 219 | |
| 220 | qsort(start_loc, count/sizeof(uint_t), sizeof(uint_t), compare_extable); |
| 221 | return NULL; |
| 222 | } |
| 223 | |
| 224 | /* Get the address of __start_mcount_loc and __stop_mcount_loc in System.map */ |
| 225 | static void get_mcount_loc(uint_t *_start, uint_t *_stop) |
| 226 | { |
| 227 | FILE *file_start, *file_stop; |
| 228 | char start_buff[20]; |
| 229 | char stop_buff[20]; |
| 230 | int len = 0; |
| 231 | |
| 232 | file_start = popen(" grep start_mcount System.map | awk '{print $1}' ", "r"); |
| 233 | if (!file_start) { |
| 234 | fprintf(stderr, "get start_mcount_loc error!"); |
| 235 | return; |
| 236 | } |
| 237 | |
| 238 | file_stop = popen(" grep stop_mcount System.map | awk '{print $1}' ", "r"); |
| 239 | if (!file_stop) { |
| 240 | fprintf(stderr, "get stop_mcount_loc error!"); |
| 241 | pclose(file_start); |
| 242 | return; |
| 243 | } |
| 244 | |
| 245 | while (fgets(start_buff, sizeof(start_buff), file_start) != NULL) { |
| 246 | len = strlen(start_buff); |
| 247 | start_buff[len - 1] = '\0'; |
| 248 | } |
| 249 | *_start = strtoul(start_buff, NULL, 16); |
| 250 | |
| 251 | while (fgets(stop_buff, sizeof(stop_buff), file_stop) != NULL) { |
| 252 | len = strlen(stop_buff); |
| 253 | stop_buff[len - 1] = '\0'; |
| 254 | } |
| 255 | *_stop = strtoul(stop_buff, NULL, 16); |
| 256 | |
| 257 | pclose(file_start); |
| 258 | pclose(file_stop); |
| 259 | } |
| 260 | #endif |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 261 | static int do_sort(Elf_Ehdr *ehdr, |
Shile Zhang | 6402e14 | 2019-12-04 08:46:28 +0800 | [diff] [blame] | 262 | char const *const fname, |
| 263 | table_sort_t custom_sort) |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 264 | { |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 265 | int rc = -1; |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 266 | Elf_Shdr *s, *shdr = (Elf_Shdr *)((char *)ehdr + _r(&ehdr->e_shoff)); |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 267 | Elf_Shdr *strtab_sec = NULL; |
| 268 | Elf_Shdr *symtab_sec = NULL; |
| 269 | Elf_Shdr *extab_sec = NULL; |
| 270 | Elf_Sym *sym; |
Jamie Iles | 59c3645 | 2013-11-12 15:06:51 -0800 | [diff] [blame] | 271 | const Elf_Sym *symtab; |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 272 | Elf32_Word *symtab_shndx = NULL; |
| 273 | Elf_Sym *sort_needed_sym = NULL; |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 274 | Elf_Shdr *sort_needed_sec; |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 275 | Elf_Rel *relocs = NULL; |
Tim Gardner | 7cbc0ea7 | 2014-10-13 15:54:20 -0700 | [diff] [blame] | 276 | int relocs_size = 0; |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 277 | uint32_t *sort_needed_loc; |
| 278 | const char *secstrings; |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 279 | const char *strtab; |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 280 | char *extab_image; |
| 281 | int extab_index = 0; |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 282 | int i; |
| 283 | int idx; |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 284 | unsigned int shnum; |
| 285 | unsigned int shstrndx; |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 286 | #ifdef MCOUNT_SORT_ENABLED |
Yinan Liu | 35140d3 | 2022-01-18 14:52:41 +0800 | [diff] [blame] | 287 | struct elf_mcount_loc mstruct = {0}; |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 288 | uint_t _start_mcount_loc = 0; |
| 289 | uint_t _stop_mcount_loc = 0; |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 290 | #endif |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 291 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
| 292 | unsigned int orc_ip_size = 0; |
| 293 | unsigned int orc_size = 0; |
| 294 | unsigned int orc_num_entries = 0; |
| 295 | #endif |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 296 | |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 297 | shstrndx = r2(&ehdr->e_shstrndx); |
| 298 | if (shstrndx == SHN_XINDEX) |
| 299 | shstrndx = r(&shdr[0].sh_link); |
| 300 | secstrings = (const char *)ehdr + _r(&shdr[shstrndx].sh_offset); |
Jamie Iles | 59c3645 | 2013-11-12 15:06:51 -0800 | [diff] [blame] | 301 | |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 302 | shnum = r2(&ehdr->e_shnum); |
| 303 | if (shnum == SHN_UNDEF) |
| 304 | shnum = _r(&shdr[0].sh_size); |
Jamie Iles | 59c3645 | 2013-11-12 15:06:51 -0800 | [diff] [blame] | 305 | |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 306 | for (i = 0, s = shdr; s < shdr + shnum; i++, s++) { |
| 307 | idx = r(&s->sh_name); |
| 308 | if (!strcmp(secstrings + idx, "__ex_table")) { |
| 309 | extab_sec = s; |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 310 | extab_index = i; |
| 311 | } |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 312 | if (!strcmp(secstrings + idx, ".symtab")) |
| 313 | symtab_sec = s; |
| 314 | if (!strcmp(secstrings + idx, ".strtab")) |
| 315 | strtab_sec = s; |
| 316 | |
| 317 | if ((r(&s->sh_type) == SHT_REL || |
| 318 | r(&s->sh_type) == SHT_RELA) && |
| 319 | r(&s->sh_info) == extab_index) { |
| 320 | relocs = (void *)ehdr + _r(&s->sh_offset); |
| 321 | relocs_size = _r(&s->sh_size); |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 322 | } |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 323 | if (r(&s->sh_type) == SHT_SYMTAB_SHNDX) |
| 324 | symtab_shndx = (Elf32_Word *)((const char *)ehdr + |
| 325 | _r(&s->sh_offset)); |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 326 | |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 327 | #ifdef MCOUNT_SORT_ENABLED |
| 328 | /* locate the .init.data section in vmlinux */ |
| 329 | if (!strcmp(secstrings + idx, ".init.data")) { |
| 330 | get_mcount_loc(&_start_mcount_loc, &_stop_mcount_loc); |
| 331 | mstruct.ehdr = ehdr; |
| 332 | mstruct.init_data_sec = s; |
| 333 | mstruct.start_mcount_loc = _start_mcount_loc; |
| 334 | mstruct.stop_mcount_loc = _stop_mcount_loc; |
| 335 | } |
| 336 | #endif |
| 337 | |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 338 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
| 339 | /* locate the ORC unwind tables */ |
| 340 | if (!strcmp(secstrings + idx, ".orc_unwind_ip")) { |
| 341 | orc_ip_size = s->sh_size; |
| 342 | g_orc_ip_table = (int *)((void *)ehdr + |
| 343 | s->sh_offset); |
| 344 | } |
| 345 | if (!strcmp(secstrings + idx, ".orc_unwind")) { |
| 346 | orc_size = s->sh_size; |
| 347 | g_orc_table = (struct orc_entry *)((void *)ehdr + |
| 348 | s->sh_offset); |
| 349 | } |
| 350 | #endif |
| 351 | } /* for loop */ |
| 352 | |
| 353 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
| 354 | if (!g_orc_ip_table || !g_orc_table) { |
| 355 | fprintf(stderr, |
| 356 | "incomplete ORC unwind tables in file: %s\n", fname); |
| 357 | goto out; |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 358 | } |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 359 | |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 360 | orc_num_entries = orc_ip_size / sizeof(int); |
| 361 | if (orc_ip_size % sizeof(int) != 0 || |
| 362 | orc_size % sizeof(struct orc_entry) != 0 || |
| 363 | orc_num_entries != orc_size / sizeof(struct orc_entry)) { |
| 364 | fprintf(stderr, |
| 365 | "inconsistent ORC unwind table entries in file: %s\n", |
| 366 | fname); |
| 367 | goto out; |
| 368 | } |
| 369 | |
| 370 | /* create thread to sort ORC unwind tables concurrently */ |
| 371 | if (pthread_create(&orc_sort_thread, NULL, |
| 372 | sort_orctable, &orc_ip_size)) { |
| 373 | fprintf(stderr, |
| 374 | "pthread_create orc_sort_thread failed '%s': %s\n", |
| 375 | strerror(errno), fname); |
| 376 | goto out; |
| 377 | } |
| 378 | #endif |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 379 | |
| 380 | #ifdef MCOUNT_SORT_ENABLED |
| 381 | if (!mstruct.init_data_sec || !_start_mcount_loc || !_stop_mcount_loc) { |
| 382 | fprintf(stderr, |
| 383 | "incomplete mcount's sort in file: %s\n", |
| 384 | fname); |
| 385 | goto out; |
| 386 | } |
| 387 | |
| 388 | /* create thread to sort mcount_loc concurrently */ |
| 389 | if (pthread_create(&mcount_sort_thread, NULL, &sort_mcount_loc, &mstruct)) { |
| 390 | fprintf(stderr, |
| 391 | "pthread_create mcount_sort_thread failed '%s': %s\n", |
| 392 | strerror(errno), fname); |
| 393 | goto out; |
| 394 | } |
| 395 | #endif |
Shile Zhang | 6402e14 | 2019-12-04 08:46:28 +0800 | [diff] [blame] | 396 | if (!extab_sec) { |
| 397 | fprintf(stderr, "no __ex_table in file: %s\n", fname); |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 398 | goto out; |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 399 | } |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 400 | |
| 401 | if (!symtab_sec) { |
| 402 | fprintf(stderr, "no .symtab in file: %s\n", fname); |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 403 | goto out; |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 404 | } |
| 405 | |
| 406 | if (!strtab_sec) { |
| 407 | fprintf(stderr, "no .strtab in file: %s\n", fname); |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 408 | goto out; |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 409 | } |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 410 | |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 411 | extab_image = (void *)ehdr + _r(&extab_sec->sh_offset); |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 412 | strtab = (const char *)ehdr + _r(&strtab_sec->sh_offset); |
| 413 | symtab = (const Elf_Sym *)((const char *)ehdr + |
| 414 | _r(&symtab_sec->sh_offset)); |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 415 | |
| 416 | if (custom_sort) { |
| 417 | custom_sort(extab_image, _r(&extab_sec->sh_size)); |
| 418 | } else { |
| 419 | int num_entries = _r(&extab_sec->sh_size) / extable_ent_size; |
| 420 | qsort(extab_image, num_entries, |
| 421 | extable_ent_size, compare_extable); |
| 422 | } |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 423 | |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 424 | /* If there were relocations, we no longer need them. */ |
| 425 | if (relocs) |
| 426 | memset(relocs, 0, relocs_size); |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 427 | |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 428 | /* find the flag main_extable_sort_needed */ |
| 429 | for (sym = (void *)ehdr + _r(&symtab_sec->sh_offset); |
| 430 | sym < sym + _r(&symtab_sec->sh_size) / sizeof(Elf_Sym); |
| 431 | sym++) { |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 432 | if (ELF_ST_TYPE(sym->st_info) != STT_OBJECT) |
| 433 | continue; |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 434 | if (!strcmp(strtab + r(&sym->st_name), |
| 435 | "main_extable_sort_needed")) { |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 436 | sort_needed_sym = sym; |
| 437 | break; |
| 438 | } |
| 439 | } |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 440 | |
Shile Zhang | 6402e14 | 2019-12-04 08:46:28 +0800 | [diff] [blame] | 441 | if (!sort_needed_sym) { |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 442 | fprintf(stderr, |
Shile Zhang | 6402e14 | 2019-12-04 08:46:28 +0800 | [diff] [blame] | 443 | "no main_extable_sort_needed symbol in file: %s\n", |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 444 | fname); |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 445 | goto out; |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 446 | } |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 447 | |
Jamie Iles | 59c3645 | 2013-11-12 15:06:51 -0800 | [diff] [blame] | 448 | sort_needed_sec = &shdr[get_secindex(r2(&sym->st_shndx), |
| 449 | sort_needed_sym - symtab, |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 450 | symtab_shndx)]; |
| 451 | sort_needed_loc = (void *)ehdr + |
David Daney | d59a168 | 2012-04-24 11:23:14 -0700 | [diff] [blame] | 452 | _r(&sort_needed_sec->sh_offset) + |
| 453 | _r(&sort_needed_sym->st_value) - |
| 454 | _r(&sort_needed_sec->sh_addr); |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 455 | |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 456 | /* extable has been sorted, clear the flag */ |
| 457 | w(0, sort_needed_loc); |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 458 | rc = 0; |
Shile Zhang | 57cafdf | 2019-12-04 08:46:30 +0800 | [diff] [blame] | 459 | |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 460 | out: |
| 461 | #if defined(SORTTABLE_64) && defined(UNWINDER_ORC_ENABLED) |
| 462 | if (orc_sort_thread) { |
| 463 | void *retval = NULL; |
| 464 | /* wait for ORC tables sort done */ |
| 465 | rc = pthread_join(orc_sort_thread, &retval); |
Yinan Liu | c8a7ff1 | 2021-12-07 23:13:47 +0800 | [diff] [blame] | 466 | if (rc) { |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 467 | fprintf(stderr, |
| 468 | "pthread_join failed '%s': %s\n", |
| 469 | strerror(errno), fname); |
Yinan Liu | c8a7ff1 | 2021-12-07 23:13:47 +0800 | [diff] [blame] | 470 | } else if (retval) { |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 471 | rc = -1; |
| 472 | fprintf(stderr, |
| 473 | "failed to sort ORC tables '%s': %s\n", |
| 474 | (char *)retval, fname); |
| 475 | } |
| 476 | } |
| 477 | #endif |
Yinan Liu | 72b3942 | 2021-12-12 19:33:58 +0800 | [diff] [blame] | 478 | |
| 479 | #ifdef MCOUNT_SORT_ENABLED |
| 480 | if (mcount_sort_thread) { |
| 481 | void *retval = NULL; |
| 482 | /* wait for mcount sort done */ |
| 483 | rc = pthread_join(mcount_sort_thread, &retval); |
| 484 | if (rc) { |
| 485 | fprintf(stderr, |
| 486 | "pthread_join failed '%s': %s\n", |
| 487 | strerror(errno), fname); |
| 488 | } else if (retval) { |
| 489 | rc = -1; |
| 490 | fprintf(stderr, |
| 491 | "failed to sort mcount '%s': %s\n", |
| 492 | (char *)retval, fname); |
| 493 | } |
| 494 | } |
| 495 | #endif |
Shile Zhang | 57fa189 | 2019-12-04 08:46:32 +0800 | [diff] [blame] | 496 | return rc; |
David Daney | a79f248 | 2012-04-19 14:59:55 -0700 | [diff] [blame] | 497 | } |