Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2007 Jens Axboe <jens.axboe@oracle.com> |
| 3 | * |
| 4 | * Scatterlist handling helpers. |
| 5 | * |
| 6 | * This source code is licensed under the GNU General Public License, |
| 7 | * Version 2. See the file COPYING for more details. |
| 8 | */ |
Paul Gortmaker | 8bc3bcc | 2011-11-16 21:29:17 -0500 | [diff] [blame] | 9 | #include <linux/export.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 10 | #include <linux/slab.h> |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 11 | #include <linux/scatterlist.h> |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 12 | #include <linux/highmem.h> |
Chris Wilson | b94de9b | 2010-07-28 22:59:02 +0100 | [diff] [blame] | 13 | #include <linux/kmemleak.h> |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 14 | |
| 15 | /** |
| 16 | * sg_next - return the next scatterlist entry in a list |
| 17 | * @sg: The current sg entry |
| 18 | * |
| 19 | * Description: |
| 20 | * Usually the next entry will be @sg@ + 1, but if this sg element is part |
| 21 | * of a chained scatterlist, it could jump to the start of a new |
| 22 | * scatterlist array. |
| 23 | * |
| 24 | **/ |
| 25 | struct scatterlist *sg_next(struct scatterlist *sg) |
| 26 | { |
| 27 | #ifdef CONFIG_DEBUG_SG |
| 28 | BUG_ON(sg->sg_magic != SG_MAGIC); |
| 29 | #endif |
| 30 | if (sg_is_last(sg)) |
| 31 | return NULL; |
| 32 | |
| 33 | sg++; |
| 34 | if (unlikely(sg_is_chain(sg))) |
| 35 | sg = sg_chain_ptr(sg); |
| 36 | |
| 37 | return sg; |
| 38 | } |
| 39 | EXPORT_SYMBOL(sg_next); |
| 40 | |
| 41 | /** |
Maxim Levitsky | 2e48461 | 2012-09-27 12:45:28 +0200 | [diff] [blame] | 42 | * sg_nents - return total count of entries in scatterlist |
| 43 | * @sg: The scatterlist |
| 44 | * |
| 45 | * Description: |
| 46 | * Allows to know how many entries are in sg, taking into acount |
| 47 | * chaining as well |
| 48 | * |
| 49 | **/ |
| 50 | int sg_nents(struct scatterlist *sg) |
| 51 | { |
Maxim Levitsky | 232f1b5 | 2012-09-28 10:38:15 +0200 | [diff] [blame] | 52 | int nents; |
| 53 | for (nents = 0; sg; sg = sg_next(sg)) |
Maxim Levitsky | 2e48461 | 2012-09-27 12:45:28 +0200 | [diff] [blame] | 54 | nents++; |
Maxim Levitsky | 2e48461 | 2012-09-27 12:45:28 +0200 | [diff] [blame] | 55 | return nents; |
| 56 | } |
| 57 | EXPORT_SYMBOL(sg_nents); |
| 58 | |
Tom Lendacky | cfaed10 | 2015-06-01 11:15:25 -0500 | [diff] [blame] | 59 | /** |
| 60 | * sg_nents_for_len - return total count of entries in scatterlist |
| 61 | * needed to satisfy the supplied length |
| 62 | * @sg: The scatterlist |
| 63 | * @len: The total required length |
| 64 | * |
| 65 | * Description: |
| 66 | * Determines the number of entries in sg that are required to meet |
| 67 | * the supplied length, taking into acount chaining as well |
| 68 | * |
| 69 | * Returns: |
| 70 | * the number of sg entries needed, negative error on failure |
| 71 | * |
| 72 | **/ |
| 73 | int sg_nents_for_len(struct scatterlist *sg, u64 len) |
| 74 | { |
| 75 | int nents; |
| 76 | u64 total; |
| 77 | |
| 78 | if (!len) |
| 79 | return 0; |
| 80 | |
| 81 | for (nents = 0, total = 0; sg; sg = sg_next(sg)) { |
| 82 | nents++; |
| 83 | total += sg->length; |
| 84 | if (total >= len) |
| 85 | return nents; |
| 86 | } |
| 87 | |
| 88 | return -EINVAL; |
| 89 | } |
| 90 | EXPORT_SYMBOL(sg_nents_for_len); |
Maxim Levitsky | 2e48461 | 2012-09-27 12:45:28 +0200 | [diff] [blame] | 91 | |
| 92 | /** |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 93 | * sg_last - return the last scatterlist entry in a list |
| 94 | * @sgl: First entry in the scatterlist |
| 95 | * @nents: Number of entries in the scatterlist |
| 96 | * |
| 97 | * Description: |
| 98 | * Should only be used casually, it (currently) scans the entire list |
| 99 | * to get the last entry. |
| 100 | * |
| 101 | * Note that the @sgl@ pointer passed in need not be the first one, |
| 102 | * the important bit is that @nents@ denotes the number of entries that |
| 103 | * exist from @sgl@. |
| 104 | * |
| 105 | **/ |
| 106 | struct scatterlist *sg_last(struct scatterlist *sgl, unsigned int nents) |
| 107 | { |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 108 | struct scatterlist *sg, *ret = NULL; |
| 109 | unsigned int i; |
| 110 | |
| 111 | for_each_sg(sgl, sg, nents, i) |
| 112 | ret = sg; |
| 113 | |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 114 | #ifdef CONFIG_DEBUG_SG |
| 115 | BUG_ON(sgl[0].sg_magic != SG_MAGIC); |
| 116 | BUG_ON(!sg_is_last(ret)); |
| 117 | #endif |
| 118 | return ret; |
| 119 | } |
| 120 | EXPORT_SYMBOL(sg_last); |
| 121 | |
| 122 | /** |
| 123 | * sg_init_table - Initialize SG table |
| 124 | * @sgl: The SG table |
| 125 | * @nents: Number of entries in table |
| 126 | * |
| 127 | * Notes: |
| 128 | * If this is part of a chained sg table, sg_mark_end() should be |
| 129 | * used only on the last table part. |
| 130 | * |
| 131 | **/ |
| 132 | void sg_init_table(struct scatterlist *sgl, unsigned int nents) |
| 133 | { |
| 134 | memset(sgl, 0, sizeof(*sgl) * nents); |
| 135 | #ifdef CONFIG_DEBUG_SG |
| 136 | { |
| 137 | unsigned int i; |
| 138 | for (i = 0; i < nents; i++) |
| 139 | sgl[i].sg_magic = SG_MAGIC; |
| 140 | } |
| 141 | #endif |
| 142 | sg_mark_end(&sgl[nents - 1]); |
| 143 | } |
| 144 | EXPORT_SYMBOL(sg_init_table); |
| 145 | |
| 146 | /** |
| 147 | * sg_init_one - Initialize a single entry sg list |
| 148 | * @sg: SG entry |
| 149 | * @buf: Virtual address for IO |
| 150 | * @buflen: IO length |
| 151 | * |
| 152 | **/ |
| 153 | void sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen) |
| 154 | { |
| 155 | sg_init_table(sg, 1); |
| 156 | sg_set_buf(sg, buf, buflen); |
| 157 | } |
| 158 | EXPORT_SYMBOL(sg_init_one); |
| 159 | |
| 160 | /* |
| 161 | * The default behaviour of sg_alloc_table() is to use these kmalloc/kfree |
| 162 | * helpers. |
| 163 | */ |
| 164 | static struct scatterlist *sg_kmalloc(unsigned int nents, gfp_t gfp_mask) |
| 165 | { |
Chris Wilson | b94de9b | 2010-07-28 22:59:02 +0100 | [diff] [blame] | 166 | if (nents == SG_MAX_SINGLE_ALLOC) { |
| 167 | /* |
| 168 | * Kmemleak doesn't track page allocations as they are not |
| 169 | * commonly used (in a raw form) for kernel data structures. |
| 170 | * As we chain together a list of pages and then a normal |
| 171 | * kmalloc (tracked by kmemleak), in order to for that last |
| 172 | * allocation not to become decoupled (and thus a |
| 173 | * false-positive) we need to inform kmemleak of all the |
| 174 | * intermediate allocations. |
| 175 | */ |
| 176 | void *ptr = (void *) __get_free_page(gfp_mask); |
| 177 | kmemleak_alloc(ptr, PAGE_SIZE, 1, gfp_mask); |
| 178 | return ptr; |
| 179 | } else |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 180 | return kmalloc(nents * sizeof(struct scatterlist), gfp_mask); |
| 181 | } |
| 182 | |
| 183 | static void sg_kfree(struct scatterlist *sg, unsigned int nents) |
| 184 | { |
Chris Wilson | b94de9b | 2010-07-28 22:59:02 +0100 | [diff] [blame] | 185 | if (nents == SG_MAX_SINGLE_ALLOC) { |
| 186 | kmemleak_free(sg); |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 187 | free_page((unsigned long) sg); |
Chris Wilson | b94de9b | 2010-07-28 22:59:02 +0100 | [diff] [blame] | 188 | } else |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 189 | kfree(sg); |
| 190 | } |
| 191 | |
| 192 | /** |
| 193 | * __sg_free_table - Free a previously mapped sg table |
| 194 | * @table: The sg table header to use |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 195 | * @max_ents: The maximum number of entries per single scatterlist |
Christoph Hellwig | c53c6d6 | 2014-04-15 14:38:31 +0200 | [diff] [blame] | 196 | * @skip_first_chunk: don't free the (preallocated) first scatterlist chunk |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 197 | * @free_fn: Free function |
| 198 | * |
| 199 | * Description: |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 200 | * Free an sg table previously allocated and setup with |
| 201 | * __sg_alloc_table(). The @max_ents value must be identical to |
| 202 | * that previously used with __sg_alloc_table(). |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 203 | * |
| 204 | **/ |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 205 | void __sg_free_table(struct sg_table *table, unsigned int max_ents, |
Christoph Hellwig | c53c6d6 | 2014-04-15 14:38:31 +0200 | [diff] [blame] | 206 | bool skip_first_chunk, sg_free_fn *free_fn) |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 207 | { |
| 208 | struct scatterlist *sgl, *next; |
| 209 | |
| 210 | if (unlikely(!table->sgl)) |
| 211 | return; |
| 212 | |
| 213 | sgl = table->sgl; |
| 214 | while (table->orig_nents) { |
| 215 | unsigned int alloc_size = table->orig_nents; |
| 216 | unsigned int sg_size; |
| 217 | |
| 218 | /* |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 219 | * If we have more than max_ents segments left, |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 220 | * then assign 'next' to the sg table after the current one. |
| 221 | * sg_size is then one less than alloc size, since the last |
| 222 | * element is the chain pointer. |
| 223 | */ |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 224 | if (alloc_size > max_ents) { |
| 225 | next = sg_chain_ptr(&sgl[max_ents - 1]); |
| 226 | alloc_size = max_ents; |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 227 | sg_size = alloc_size - 1; |
| 228 | } else { |
| 229 | sg_size = alloc_size; |
| 230 | next = NULL; |
| 231 | } |
| 232 | |
| 233 | table->orig_nents -= sg_size; |
Tony Battersby | c21e59d | 2014-10-23 15:10:21 -0400 | [diff] [blame] | 234 | if (skip_first_chunk) |
Christoph Hellwig | c53c6d6 | 2014-04-15 14:38:31 +0200 | [diff] [blame] | 235 | skip_first_chunk = false; |
Tony Battersby | c21e59d | 2014-10-23 15:10:21 -0400 | [diff] [blame] | 236 | else |
| 237 | free_fn(sgl, alloc_size); |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 238 | sgl = next; |
| 239 | } |
| 240 | |
| 241 | table->sgl = NULL; |
| 242 | } |
| 243 | EXPORT_SYMBOL(__sg_free_table); |
| 244 | |
| 245 | /** |
| 246 | * sg_free_table - Free a previously allocated sg table |
| 247 | * @table: The mapped sg table header |
| 248 | * |
| 249 | **/ |
| 250 | void sg_free_table(struct sg_table *table) |
| 251 | { |
Christoph Hellwig | c53c6d6 | 2014-04-15 14:38:31 +0200 | [diff] [blame] | 252 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree); |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 253 | } |
| 254 | EXPORT_SYMBOL(sg_free_table); |
| 255 | |
| 256 | /** |
| 257 | * __sg_alloc_table - Allocate and initialize an sg table with given allocator |
| 258 | * @table: The sg table header to use |
| 259 | * @nents: Number of entries in sg list |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 260 | * @max_ents: The maximum number of entries the allocator returns per call |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 261 | * @gfp_mask: GFP allocation mask |
| 262 | * @alloc_fn: Allocator to use |
| 263 | * |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 264 | * Description: |
| 265 | * This function returns a @table @nents long. The allocator is |
| 266 | * defined to return scatterlist chunks of maximum size @max_ents. |
| 267 | * Thus if @nents is bigger than @max_ents, the scatterlists will be |
| 268 | * chained in units of @max_ents. |
| 269 | * |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 270 | * Notes: |
| 271 | * If this function returns non-0 (eg failure), the caller must call |
| 272 | * __sg_free_table() to cleanup any leftover allocations. |
| 273 | * |
| 274 | **/ |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 275 | int __sg_alloc_table(struct sg_table *table, unsigned int nents, |
Christoph Hellwig | c53c6d6 | 2014-04-15 14:38:31 +0200 | [diff] [blame] | 276 | unsigned int max_ents, struct scatterlist *first_chunk, |
| 277 | gfp_t gfp_mask, sg_alloc_fn *alloc_fn) |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 278 | { |
| 279 | struct scatterlist *sg, *prv; |
| 280 | unsigned int left; |
| 281 | |
Dan Carpenter | 27daabd | 2013-07-08 16:01:58 -0700 | [diff] [blame] | 282 | memset(table, 0, sizeof(*table)); |
| 283 | |
| 284 | if (nents == 0) |
| 285 | return -EINVAL; |
Laura Abbott | 308c09f | 2014-08-08 14:23:25 -0700 | [diff] [blame] | 286 | #ifndef CONFIG_ARCH_HAS_SG_CHAIN |
Nick Bowler | 6fd59a8 | 2012-12-17 16:05:20 -0800 | [diff] [blame] | 287 | if (WARN_ON_ONCE(nents > max_ents)) |
| 288 | return -EINVAL; |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 289 | #endif |
| 290 | |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 291 | left = nents; |
| 292 | prv = NULL; |
| 293 | do { |
| 294 | unsigned int sg_size, alloc_size = left; |
| 295 | |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 296 | if (alloc_size > max_ents) { |
| 297 | alloc_size = max_ents; |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 298 | sg_size = alloc_size - 1; |
| 299 | } else |
| 300 | sg_size = alloc_size; |
| 301 | |
| 302 | left -= sg_size; |
| 303 | |
Christoph Hellwig | c53c6d6 | 2014-04-15 14:38:31 +0200 | [diff] [blame] | 304 | if (first_chunk) { |
| 305 | sg = first_chunk; |
| 306 | first_chunk = NULL; |
| 307 | } else { |
| 308 | sg = alloc_fn(alloc_size, gfp_mask); |
| 309 | } |
Jeffrey Carlyle | edce682 | 2010-08-30 19:55:09 +0200 | [diff] [blame] | 310 | if (unlikely(!sg)) { |
| 311 | /* |
| 312 | * Adjust entry count to reflect that the last |
| 313 | * entry of the previous table won't be used for |
| 314 | * linkage. Without this, sg_kfree() may get |
| 315 | * confused. |
| 316 | */ |
| 317 | if (prv) |
| 318 | table->nents = ++table->orig_nents; |
| 319 | |
| 320 | return -ENOMEM; |
| 321 | } |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 322 | |
| 323 | sg_init_table(sg, alloc_size); |
| 324 | table->nents = table->orig_nents += sg_size; |
| 325 | |
| 326 | /* |
| 327 | * If this is the first mapping, assign the sg table header. |
| 328 | * If this is not the first mapping, chain previous part. |
| 329 | */ |
| 330 | if (prv) |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 331 | sg_chain(prv, max_ents, sg); |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 332 | else |
| 333 | table->sgl = sg; |
| 334 | |
| 335 | /* |
| 336 | * If no more entries after this one, mark the end |
| 337 | */ |
| 338 | if (!left) |
| 339 | sg_mark_end(&sg[sg_size - 1]); |
| 340 | |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 341 | prv = sg; |
| 342 | } while (left); |
| 343 | |
| 344 | return 0; |
| 345 | } |
| 346 | EXPORT_SYMBOL(__sg_alloc_table); |
| 347 | |
| 348 | /** |
| 349 | * sg_alloc_table - Allocate and initialize an sg table |
| 350 | * @table: The sg table header to use |
| 351 | * @nents: Number of entries in sg list |
| 352 | * @gfp_mask: GFP allocation mask |
| 353 | * |
| 354 | * Description: |
| 355 | * Allocate and initialize an sg table. If @nents@ is larger than |
| 356 | * SG_MAX_SINGLE_ALLOC a chained sg table will be setup. |
| 357 | * |
| 358 | **/ |
| 359 | int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) |
| 360 | { |
| 361 | int ret; |
| 362 | |
James Bottomley | 7cedb1f | 2008-01-13 14:15:28 -0600 | [diff] [blame] | 363 | ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, |
Christoph Hellwig | c53c6d6 | 2014-04-15 14:38:31 +0200 | [diff] [blame] | 364 | NULL, gfp_mask, sg_kmalloc); |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 365 | if (unlikely(ret)) |
Christoph Hellwig | c53c6d6 | 2014-04-15 14:38:31 +0200 | [diff] [blame] | 366 | __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree); |
Jens Axboe | 0db9299 | 2007-11-30 09:16:50 +0100 | [diff] [blame] | 367 | |
| 368 | return ret; |
| 369 | } |
| 370 | EXPORT_SYMBOL(sg_alloc_table); |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 371 | |
| 372 | /** |
Tvrtko Ursulin | 89d8589 | 2017-08-03 10:13:51 +0100 | [diff] [blame] | 373 | * __sg_alloc_table_from_pages - Allocate and initialize an sg table from |
| 374 | * an array of pages |
| 375 | * @sgt: The sg table header to use |
| 376 | * @pages: Pointer to an array of page pointers |
| 377 | * @n_pages: Number of pages in the pages array |
| 378 | * @offset: Offset from start of the first page to the start of a buffer |
| 379 | * @size: Number of valid bytes in the buffer (after offset) |
| 380 | * @max_segment: Maximum size of a scatterlist node in bytes (page aligned) |
| 381 | * @gfp_mask: GFP allocation mask |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 382 | * |
| 383 | * Description: |
| 384 | * Allocate and initialize an sg table from a list of pages. Contiguous |
Tvrtko Ursulin | 89d8589 | 2017-08-03 10:13:51 +0100 | [diff] [blame] | 385 | * ranges of the pages are squashed into a single scatterlist node up to the |
| 386 | * maximum size specified in @max_segment. An user may provide an offset at a |
| 387 | * start and a size of valid data in a buffer specified by the page array. |
| 388 | * The returned sg table is released by sg_free_table. |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 389 | * |
| 390 | * Returns: |
| 391 | * 0 on success, negative error on failure |
| 392 | */ |
Tvrtko Ursulin | 89d8589 | 2017-08-03 10:13:51 +0100 | [diff] [blame] | 393 | int __sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, |
| 394 | unsigned int n_pages, unsigned int offset, |
| 395 | unsigned long size, unsigned int max_segment, |
| 396 | gfp_t gfp_mask) |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 397 | { |
Tvrtko Ursulin | c125906 | 2017-08-03 10:13:12 +0100 | [diff] [blame] | 398 | unsigned int chunks, cur_page, seg_len, i; |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 399 | int ret; |
| 400 | struct scatterlist *s; |
| 401 | |
Tvrtko Ursulin | 89d8589 | 2017-08-03 10:13:51 +0100 | [diff] [blame] | 402 | if (WARN_ON(!max_segment || offset_in_page(max_segment))) |
| 403 | return -EINVAL; |
| 404 | |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 405 | /* compute number of contiguous chunks */ |
| 406 | chunks = 1; |
Tvrtko Ursulin | c125906 | 2017-08-03 10:13:12 +0100 | [diff] [blame] | 407 | seg_len = 0; |
| 408 | for (i = 1; i < n_pages; i++) { |
| 409 | seg_len += PAGE_SIZE; |
| 410 | if (seg_len >= max_segment || |
| 411 | page_to_pfn(pages[i]) != page_to_pfn(pages[i - 1]) + 1) { |
| 412 | chunks++; |
| 413 | seg_len = 0; |
| 414 | } |
| 415 | } |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 416 | |
| 417 | ret = sg_alloc_table(sgt, chunks, gfp_mask); |
| 418 | if (unlikely(ret)) |
| 419 | return ret; |
| 420 | |
| 421 | /* merging chunks and putting them into the scatterlist */ |
| 422 | cur_page = 0; |
| 423 | for_each_sg(sgt->sgl, s, sgt->orig_nents, i) { |
Tvrtko Ursulin | c125906 | 2017-08-03 10:13:12 +0100 | [diff] [blame] | 424 | unsigned int j, chunk_size; |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 425 | |
| 426 | /* look for the end of the current chunk */ |
Tvrtko Ursulin | c125906 | 2017-08-03 10:13:12 +0100 | [diff] [blame] | 427 | seg_len = 0; |
| 428 | for (j = cur_page + 1; j < n_pages; j++) { |
| 429 | seg_len += PAGE_SIZE; |
| 430 | if (seg_len >= max_segment || |
| 431 | page_to_pfn(pages[j]) != |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 432 | page_to_pfn(pages[j - 1]) + 1) |
| 433 | break; |
Tvrtko Ursulin | c125906 | 2017-08-03 10:13:12 +0100 | [diff] [blame] | 434 | } |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 435 | |
| 436 | chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset; |
Tvrtko Ursulin | c125906 | 2017-08-03 10:13:12 +0100 | [diff] [blame] | 437 | sg_set_page(s, pages[cur_page], |
| 438 | min_t(unsigned long, size, chunk_size), offset); |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 439 | size -= chunk_size; |
| 440 | offset = 0; |
| 441 | cur_page = j; |
| 442 | } |
| 443 | |
| 444 | return 0; |
| 445 | } |
Tvrtko Ursulin | 89d8589 | 2017-08-03 10:13:51 +0100 | [diff] [blame] | 446 | EXPORT_SYMBOL(__sg_alloc_table_from_pages); |
| 447 | |
| 448 | /** |
| 449 | * sg_alloc_table_from_pages - Allocate and initialize an sg table from |
| 450 | * an array of pages |
| 451 | * @sgt: The sg table header to use |
| 452 | * @pages: Pointer to an array of page pointers |
| 453 | * @n_pages: Number of pages in the pages array |
| 454 | * @offset: Offset from start of the first page to the start of a buffer |
| 455 | * @size: Number of valid bytes in the buffer (after offset) |
| 456 | * @gfp_mask: GFP allocation mask |
| 457 | * |
| 458 | * Description: |
| 459 | * Allocate and initialize an sg table from a list of pages. Contiguous |
| 460 | * ranges of the pages are squashed into a single scatterlist node. A user |
| 461 | * may provide an offset at a start and a size of valid data in a buffer |
| 462 | * specified by the page array. The returned sg table is released by |
| 463 | * sg_free_table. |
| 464 | * |
| 465 | * Returns: |
| 466 | * 0 on success, negative error on failure |
| 467 | */ |
| 468 | int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, |
| 469 | unsigned int n_pages, unsigned int offset, |
| 470 | unsigned long size, gfp_t gfp_mask) |
| 471 | { |
| 472 | return __sg_alloc_table_from_pages(sgt, pages, n_pages, offset, size, |
| 473 | SCATTERLIST_MAX_SEGMENT, gfp_mask); |
| 474 | } |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 475 | EXPORT_SYMBOL(sg_alloc_table_from_pages); |
| 476 | |
Bart Van Assche | e80a0af | 2018-01-05 08:26:46 -0800 | [diff] [blame] | 477 | #ifdef CONFIG_SGL_ALLOC |
| 478 | |
| 479 | /** |
| 480 | * sgl_alloc_order - allocate a scatterlist and its pages |
| 481 | * @length: Length in bytes of the scatterlist. Must be at least one |
| 482 | * @order: Second argument for alloc_pages() |
| 483 | * @chainable: Whether or not to allocate an extra element in the scatterlist |
| 484 | * for scatterlist chaining purposes |
| 485 | * @gfp: Memory allocation flags |
| 486 | * @nent_p: [out] Number of entries in the scatterlist that have pages |
| 487 | * |
| 488 | * Returns: A pointer to an initialized scatterlist or %NULL upon failure. |
| 489 | */ |
| 490 | struct scatterlist *sgl_alloc_order(unsigned long long length, |
| 491 | unsigned int order, bool chainable, |
| 492 | gfp_t gfp, unsigned int *nent_p) |
| 493 | { |
| 494 | struct scatterlist *sgl, *sg; |
| 495 | struct page *page; |
| 496 | unsigned int nent, nalloc; |
| 497 | u32 elem_len; |
| 498 | |
| 499 | nent = round_up(length, PAGE_SIZE << order) >> (PAGE_SHIFT + order); |
| 500 | /* Check for integer overflow */ |
| 501 | if (length > (nent << (PAGE_SHIFT + order))) |
| 502 | return NULL; |
| 503 | nalloc = nent; |
| 504 | if (chainable) { |
| 505 | /* Check for integer overflow */ |
| 506 | if (nalloc + 1 < nalloc) |
| 507 | return NULL; |
| 508 | nalloc++; |
| 509 | } |
| 510 | sgl = kmalloc_array(nalloc, sizeof(struct scatterlist), |
| 511 | (gfp & ~GFP_DMA) | __GFP_ZERO); |
| 512 | if (!sgl) |
| 513 | return NULL; |
| 514 | |
Bart Van Assche | 8c7a8d1 | 2018-01-19 11:00:54 -0800 | [diff] [blame] | 515 | sg_init_table(sgl, nalloc); |
Bart Van Assche | e80a0af | 2018-01-05 08:26:46 -0800 | [diff] [blame] | 516 | sg = sgl; |
| 517 | while (length) { |
| 518 | elem_len = min_t(u64, length, PAGE_SIZE << order); |
| 519 | page = alloc_pages(gfp, order); |
| 520 | if (!page) { |
| 521 | sgl_free(sgl); |
| 522 | return NULL; |
| 523 | } |
| 524 | |
| 525 | sg_set_page(sg, page, elem_len, 0); |
| 526 | length -= elem_len; |
| 527 | sg = sg_next(sg); |
| 528 | } |
Bart Van Assche | 8c7a8d1 | 2018-01-19 11:00:54 -0800 | [diff] [blame] | 529 | WARN_ONCE(length, "length = %lld\n", length); |
Bart Van Assche | e80a0af | 2018-01-05 08:26:46 -0800 | [diff] [blame] | 530 | if (nent_p) |
| 531 | *nent_p = nent; |
| 532 | return sgl; |
| 533 | } |
| 534 | EXPORT_SYMBOL(sgl_alloc_order); |
| 535 | |
| 536 | /** |
| 537 | * sgl_alloc - allocate a scatterlist and its pages |
| 538 | * @length: Length in bytes of the scatterlist |
| 539 | * @gfp: Memory allocation flags |
| 540 | * @nent_p: [out] Number of entries in the scatterlist |
| 541 | * |
| 542 | * Returns: A pointer to an initialized scatterlist or %NULL upon failure. |
| 543 | */ |
| 544 | struct scatterlist *sgl_alloc(unsigned long long length, gfp_t gfp, |
| 545 | unsigned int *nent_p) |
| 546 | { |
| 547 | return sgl_alloc_order(length, 0, false, gfp, nent_p); |
| 548 | } |
| 549 | EXPORT_SYMBOL(sgl_alloc); |
| 550 | |
| 551 | /** |
Bart Van Assche | 8c7a8d1 | 2018-01-19 11:00:54 -0800 | [diff] [blame] | 552 | * sgl_free_n_order - free a scatterlist and its pages |
| 553 | * @sgl: Scatterlist with one or more elements |
| 554 | * @nents: Maximum number of elements to free |
| 555 | * @order: Second argument for __free_pages() |
| 556 | * |
| 557 | * Notes: |
| 558 | * - If several scatterlists have been chained and each chain element is |
| 559 | * freed separately then it's essential to set nents correctly to avoid that a |
| 560 | * page would get freed twice. |
| 561 | * - All pages in a chained scatterlist can be freed at once by setting @nents |
| 562 | * to a high number. |
| 563 | */ |
| 564 | void sgl_free_n_order(struct scatterlist *sgl, int nents, int order) |
| 565 | { |
| 566 | struct scatterlist *sg; |
| 567 | struct page *page; |
| 568 | int i; |
| 569 | |
| 570 | for_each_sg(sgl, sg, nents, i) { |
| 571 | if (!sg) |
| 572 | break; |
| 573 | page = sg_page(sg); |
| 574 | if (page) |
| 575 | __free_pages(page, order); |
| 576 | } |
| 577 | kfree(sgl); |
| 578 | } |
| 579 | EXPORT_SYMBOL(sgl_free_n_order); |
| 580 | |
| 581 | /** |
Bart Van Assche | e80a0af | 2018-01-05 08:26:46 -0800 | [diff] [blame] | 582 | * sgl_free_order - free a scatterlist and its pages |
| 583 | * @sgl: Scatterlist with one or more elements |
| 584 | * @order: Second argument for __free_pages() |
| 585 | */ |
| 586 | void sgl_free_order(struct scatterlist *sgl, int order) |
| 587 | { |
Bart Van Assche | 8c7a8d1 | 2018-01-19 11:00:54 -0800 | [diff] [blame] | 588 | sgl_free_n_order(sgl, INT_MAX, order); |
Bart Van Assche | e80a0af | 2018-01-05 08:26:46 -0800 | [diff] [blame] | 589 | } |
| 590 | EXPORT_SYMBOL(sgl_free_order); |
| 591 | |
| 592 | /** |
| 593 | * sgl_free - free a scatterlist and its pages |
| 594 | * @sgl: Scatterlist with one or more elements |
| 595 | */ |
| 596 | void sgl_free(struct scatterlist *sgl) |
| 597 | { |
| 598 | sgl_free_order(sgl, 0); |
| 599 | } |
| 600 | EXPORT_SYMBOL(sgl_free); |
| 601 | |
| 602 | #endif /* CONFIG_SGL_ALLOC */ |
| 603 | |
Imre Deak | a321e91 | 2013-02-27 17:02:56 -0800 | [diff] [blame] | 604 | void __sg_page_iter_start(struct sg_page_iter *piter, |
| 605 | struct scatterlist *sglist, unsigned int nents, |
| 606 | unsigned long pgoffset) |
| 607 | { |
| 608 | piter->__pg_advance = 0; |
| 609 | piter->__nents = nents; |
| 610 | |
Imre Deak | a321e91 | 2013-02-27 17:02:56 -0800 | [diff] [blame] | 611 | piter->sg = sglist; |
| 612 | piter->sg_pgoffset = pgoffset; |
| 613 | } |
| 614 | EXPORT_SYMBOL(__sg_page_iter_start); |
| 615 | |
| 616 | static int sg_page_count(struct scatterlist *sg) |
| 617 | { |
| 618 | return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT; |
| 619 | } |
| 620 | |
| 621 | bool __sg_page_iter_next(struct sg_page_iter *piter) |
| 622 | { |
| 623 | if (!piter->__nents || !piter->sg) |
| 624 | return false; |
| 625 | |
| 626 | piter->sg_pgoffset += piter->__pg_advance; |
| 627 | piter->__pg_advance = 1; |
| 628 | |
| 629 | while (piter->sg_pgoffset >= sg_page_count(piter->sg)) { |
| 630 | piter->sg_pgoffset -= sg_page_count(piter->sg); |
| 631 | piter->sg = sg_next(piter->sg); |
| 632 | if (!--piter->__nents || !piter->sg) |
| 633 | return false; |
| 634 | } |
Imre Deak | a321e91 | 2013-02-27 17:02:56 -0800 | [diff] [blame] | 635 | |
| 636 | return true; |
| 637 | } |
| 638 | EXPORT_SYMBOL(__sg_page_iter_next); |
| 639 | |
Tomasz Stanislawski | efc42bc | 2012-06-18 09:25:01 +0200 | [diff] [blame] | 640 | /** |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 641 | * sg_miter_start - start mapping iteration over a sg list |
| 642 | * @miter: sg mapping iter to be started |
| 643 | * @sgl: sg list to iterate over |
| 644 | * @nents: number of sg entries |
| 645 | * |
| 646 | * Description: |
| 647 | * Starts mapping iterator @miter. |
| 648 | * |
| 649 | * Context: |
| 650 | * Don't care. |
| 651 | */ |
| 652 | void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl, |
| 653 | unsigned int nents, unsigned int flags) |
| 654 | { |
| 655 | memset(miter, 0, sizeof(struct sg_mapping_iter)); |
| 656 | |
Imre Deak | 4225fc8 | 2013-02-27 17:02:57 -0800 | [diff] [blame] | 657 | __sg_page_iter_start(&miter->piter, sgl, nents, 0); |
Sebastian Andrzej Siewior | 6de7e356 | 2009-06-18 10:19:12 +0200 | [diff] [blame] | 658 | WARN_ON(!(flags & (SG_MITER_TO_SG | SG_MITER_FROM_SG))); |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 659 | miter->__flags = flags; |
| 660 | } |
| 661 | EXPORT_SYMBOL(sg_miter_start); |
| 662 | |
Akinobu Mita | 1105200 | 2013-07-08 16:01:52 -0700 | [diff] [blame] | 663 | static bool sg_miter_get_next_page(struct sg_mapping_iter *miter) |
| 664 | { |
| 665 | if (!miter->__remaining) { |
| 666 | struct scatterlist *sg; |
| 667 | unsigned long pgoffset; |
| 668 | |
| 669 | if (!__sg_page_iter_next(&miter->piter)) |
| 670 | return false; |
| 671 | |
| 672 | sg = miter->piter.sg; |
| 673 | pgoffset = miter->piter.sg_pgoffset; |
| 674 | |
| 675 | miter->__offset = pgoffset ? 0 : sg->offset; |
| 676 | miter->__remaining = sg->offset + sg->length - |
| 677 | (pgoffset << PAGE_SHIFT) - miter->__offset; |
| 678 | miter->__remaining = min_t(unsigned long, miter->__remaining, |
| 679 | PAGE_SIZE - miter->__offset); |
| 680 | } |
| 681 | |
| 682 | return true; |
| 683 | } |
| 684 | |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 685 | /** |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 686 | * sg_miter_skip - reposition mapping iterator |
| 687 | * @miter: sg mapping iter to be skipped |
| 688 | * @offset: number of bytes to plus the current location |
| 689 | * |
| 690 | * Description: |
| 691 | * Sets the offset of @miter to its current location plus @offset bytes. |
| 692 | * If mapping iterator @miter has been proceeded by sg_miter_next(), this |
| 693 | * stops @miter. |
| 694 | * |
| 695 | * Context: |
| 696 | * Don't care if @miter is stopped, or not proceeded yet. |
| 697 | * Otherwise, preemption disabled if the SG_MITER_ATOMIC is set. |
| 698 | * |
| 699 | * Returns: |
| 700 | * true if @miter contains the valid mapping. false if end of sg |
| 701 | * list is reached. |
| 702 | */ |
Ming Lei | 0d6077f | 2013-11-26 12:43:37 +0800 | [diff] [blame] | 703 | bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset) |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 704 | { |
| 705 | sg_miter_stop(miter); |
| 706 | |
| 707 | while (offset) { |
| 708 | off_t consumed; |
| 709 | |
| 710 | if (!sg_miter_get_next_page(miter)) |
| 711 | return false; |
| 712 | |
| 713 | consumed = min_t(off_t, offset, miter->__remaining); |
| 714 | miter->__offset += consumed; |
| 715 | miter->__remaining -= consumed; |
| 716 | offset -= consumed; |
| 717 | } |
| 718 | |
| 719 | return true; |
| 720 | } |
Ming Lei | 0d6077f | 2013-11-26 12:43:37 +0800 | [diff] [blame] | 721 | EXPORT_SYMBOL(sg_miter_skip); |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 722 | |
| 723 | /** |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 724 | * sg_miter_next - proceed mapping iterator to the next mapping |
| 725 | * @miter: sg mapping iter to proceed |
| 726 | * |
| 727 | * Description: |
Tejun Heo | 8290e2d | 2012-10-04 17:13:28 -0700 | [diff] [blame] | 728 | * Proceeds @miter to the next mapping. @miter should have been started |
| 729 | * using sg_miter_start(). On successful return, @miter->page, |
| 730 | * @miter->addr and @miter->length point to the current mapping. |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 731 | * |
| 732 | * Context: |
Tejun Heo | 8290e2d | 2012-10-04 17:13:28 -0700 | [diff] [blame] | 733 | * Preemption disabled if SG_MITER_ATOMIC. Preemption must stay disabled |
| 734 | * till @miter is stopped. May sleep if !SG_MITER_ATOMIC. |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 735 | * |
| 736 | * Returns: |
| 737 | * true if @miter contains the next mapping. false if end of sg |
| 738 | * list is reached. |
| 739 | */ |
| 740 | bool sg_miter_next(struct sg_mapping_iter *miter) |
| 741 | { |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 742 | sg_miter_stop(miter); |
| 743 | |
Imre Deak | 4225fc8 | 2013-02-27 17:02:57 -0800 | [diff] [blame] | 744 | /* |
| 745 | * Get to the next page if necessary. |
| 746 | * __remaining, __offset is adjusted by sg_miter_stop |
| 747 | */ |
Akinobu Mita | 1105200 | 2013-07-08 16:01:52 -0700 | [diff] [blame] | 748 | if (!sg_miter_get_next_page(miter)) |
| 749 | return false; |
Imre Deak | 4225fc8 | 2013-02-27 17:02:57 -0800 | [diff] [blame] | 750 | |
Imre Deak | 2db76d7 | 2013-03-26 15:14:18 +0200 | [diff] [blame] | 751 | miter->page = sg_page_iter_page(&miter->piter); |
Imre Deak | 4225fc8 | 2013-02-27 17:02:57 -0800 | [diff] [blame] | 752 | miter->consumed = miter->length = miter->__remaining; |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 753 | |
| 754 | if (miter->__flags & SG_MITER_ATOMIC) |
Imre Deak | 4225fc8 | 2013-02-27 17:02:57 -0800 | [diff] [blame] | 755 | miter->addr = kmap_atomic(miter->page) + miter->__offset; |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 756 | else |
Imre Deak | 4225fc8 | 2013-02-27 17:02:57 -0800 | [diff] [blame] | 757 | miter->addr = kmap(miter->page) + miter->__offset; |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 758 | |
| 759 | return true; |
| 760 | } |
| 761 | EXPORT_SYMBOL(sg_miter_next); |
| 762 | |
| 763 | /** |
| 764 | * sg_miter_stop - stop mapping iteration |
| 765 | * @miter: sg mapping iter to be stopped |
| 766 | * |
| 767 | * Description: |
| 768 | * Stops mapping iterator @miter. @miter should have been started |
Masahiro Yamada | 4ba6a2b | 2016-02-08 16:09:08 +0900 | [diff] [blame] | 769 | * using sg_miter_start(). A stopped iteration can be resumed by |
| 770 | * calling sg_miter_next() on it. This is useful when resources (kmap) |
| 771 | * need to be released during iteration. |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 772 | * |
| 773 | * Context: |
Tejun Heo | 8290e2d | 2012-10-04 17:13:28 -0700 | [diff] [blame] | 774 | * Preemption disabled if the SG_MITER_ATOMIC is set. Don't care |
| 775 | * otherwise. |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 776 | */ |
| 777 | void sg_miter_stop(struct sg_mapping_iter *miter) |
| 778 | { |
| 779 | WARN_ON(miter->consumed > miter->length); |
| 780 | |
| 781 | /* drop resources from the last iteration */ |
| 782 | if (miter->addr) { |
| 783 | miter->__offset += miter->consumed; |
Imre Deak | 4225fc8 | 2013-02-27 17:02:57 -0800 | [diff] [blame] | 784 | miter->__remaining -= miter->consumed; |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 785 | |
Ming Lei | 3d77b50 | 2013-10-31 16:34:17 -0700 | [diff] [blame] | 786 | if ((miter->__flags & SG_MITER_TO_SG) && |
| 787 | !PageSlab(miter->page)) |
Sebastian Andrzej Siewior | 6de7e356 | 2009-06-18 10:19:12 +0200 | [diff] [blame] | 788 | flush_kernel_dcache_page(miter->page); |
| 789 | |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 790 | if (miter->__flags & SG_MITER_ATOMIC) { |
Tejun Heo | 8290e2d | 2012-10-04 17:13:28 -0700 | [diff] [blame] | 791 | WARN_ON_ONCE(preemptible()); |
Cong Wang | c3eede8 | 2011-11-25 23:14:39 +0800 | [diff] [blame] | 792 | kunmap_atomic(miter->addr); |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 793 | } else |
Arjan van de Ven | f652c52 | 2008-11-19 15:36:19 -0800 | [diff] [blame] | 794 | kunmap(miter->page); |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 795 | |
| 796 | miter->page = NULL; |
| 797 | miter->addr = NULL; |
| 798 | miter->length = 0; |
| 799 | miter->consumed = 0; |
| 800 | } |
| 801 | } |
| 802 | EXPORT_SYMBOL(sg_miter_stop); |
| 803 | |
| 804 | /** |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 805 | * sg_copy_buffer - Copy data between a linear buffer and an SG list |
| 806 | * @sgl: The SG list |
| 807 | * @nents: Number of SG entries |
| 808 | * @buf: Where to copy from |
| 809 | * @buflen: The number of bytes to copy |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 810 | * @skip: Number of bytes to skip before copying |
| 811 | * @to_buffer: transfer direction (true == from an sg list to a |
| 812 | * buffer, false == from a buffer to an sg list |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 813 | * |
| 814 | * Returns the number of copied bytes. |
| 815 | * |
| 816 | **/ |
Dave Gordon | 386ecb1 | 2015-06-30 14:58:57 -0700 | [diff] [blame] | 817 | size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf, |
| 818 | size_t buflen, off_t skip, bool to_buffer) |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 819 | { |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 820 | unsigned int offset = 0; |
| 821 | struct sg_mapping_iter miter; |
Sebastian Andrzej Siewior | 6de7e356 | 2009-06-18 10:19:12 +0200 | [diff] [blame] | 822 | unsigned int sg_flags = SG_MITER_ATOMIC; |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 823 | |
Sebastian Andrzej Siewior | 6de7e356 | 2009-06-18 10:19:12 +0200 | [diff] [blame] | 824 | if (to_buffer) |
| 825 | sg_flags |= SG_MITER_FROM_SG; |
| 826 | else |
| 827 | sg_flags |= SG_MITER_TO_SG; |
| 828 | |
| 829 | sg_miter_start(&miter, sgl, nents, sg_flags); |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 830 | |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 831 | if (!sg_miter_skip(&miter, skip)) |
| 832 | return false; |
| 833 | |
Gilad Ben-Yossef | 1d5210ef | 2017-02-27 14:28:27 -0800 | [diff] [blame] | 834 | while ((offset < buflen) && sg_miter_next(&miter)) { |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 835 | unsigned int len; |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 836 | |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 837 | len = min(miter.length, buflen - offset); |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 838 | |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 839 | if (to_buffer) |
| 840 | memcpy(buf + offset, miter.addr, len); |
Sebastian Andrzej Siewior | 6de7e356 | 2009-06-18 10:19:12 +0200 | [diff] [blame] | 841 | else |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 842 | memcpy(miter.addr, buf + offset, len); |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 843 | |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 844 | offset += len; |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 845 | } |
| 846 | |
Tejun Heo | 137d3ed | 2008-07-19 23:03:35 +0900 | [diff] [blame] | 847 | sg_miter_stop(&miter); |
| 848 | |
| 849 | return offset; |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 850 | } |
Dave Gordon | 386ecb1 | 2015-06-30 14:58:57 -0700 | [diff] [blame] | 851 | EXPORT_SYMBOL(sg_copy_buffer); |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 852 | |
| 853 | /** |
| 854 | * sg_copy_from_buffer - Copy from a linear buffer to an SG list |
| 855 | * @sgl: The SG list |
| 856 | * @nents: Number of SG entries |
| 857 | * @buf: Where to copy from |
| 858 | * @buflen: The number of bytes to copy |
| 859 | * |
| 860 | * Returns the number of copied bytes. |
| 861 | * |
| 862 | **/ |
| 863 | size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, |
Dave Gordon | 2a1bf8f | 2015-06-30 14:58:54 -0700 | [diff] [blame] | 864 | const void *buf, size_t buflen) |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 865 | { |
Dave Gordon | 2a1bf8f | 2015-06-30 14:58:54 -0700 | [diff] [blame] | 866 | return sg_copy_buffer(sgl, nents, (void *)buf, buflen, 0, false); |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 867 | } |
| 868 | EXPORT_SYMBOL(sg_copy_from_buffer); |
| 869 | |
| 870 | /** |
| 871 | * sg_copy_to_buffer - Copy from an SG list to a linear buffer |
| 872 | * @sgl: The SG list |
| 873 | * @nents: Number of SG entries |
| 874 | * @buf: Where to copy to |
| 875 | * @buflen: The number of bytes to copy |
| 876 | * |
| 877 | * Returns the number of copied bytes. |
| 878 | * |
| 879 | **/ |
| 880 | size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents, |
| 881 | void *buf, size_t buflen) |
| 882 | { |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 883 | return sg_copy_buffer(sgl, nents, buf, buflen, 0, true); |
FUJITA Tomonori | b1adaf6 | 2008-03-18 00:15:03 +0900 | [diff] [blame] | 884 | } |
| 885 | EXPORT_SYMBOL(sg_copy_to_buffer); |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 886 | |
| 887 | /** |
| 888 | * sg_pcopy_from_buffer - Copy from a linear buffer to an SG list |
| 889 | * @sgl: The SG list |
| 890 | * @nents: Number of SG entries |
| 891 | * @buf: Where to copy from |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 892 | * @buflen: The number of bytes to copy |
Dave Gordon | 4dc7daf | 2015-06-30 14:58:52 -0700 | [diff] [blame] | 893 | * @skip: Number of bytes to skip before copying |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 894 | * |
| 895 | * Returns the number of copied bytes. |
| 896 | * |
| 897 | **/ |
| 898 | size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents, |
Dave Gordon | 2a1bf8f | 2015-06-30 14:58:54 -0700 | [diff] [blame] | 899 | const void *buf, size_t buflen, off_t skip) |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 900 | { |
Dave Gordon | 2a1bf8f | 2015-06-30 14:58:54 -0700 | [diff] [blame] | 901 | return sg_copy_buffer(sgl, nents, (void *)buf, buflen, skip, false); |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 902 | } |
| 903 | EXPORT_SYMBOL(sg_pcopy_from_buffer); |
| 904 | |
| 905 | /** |
| 906 | * sg_pcopy_to_buffer - Copy from an SG list to a linear buffer |
| 907 | * @sgl: The SG list |
| 908 | * @nents: Number of SG entries |
| 909 | * @buf: Where to copy to |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 910 | * @buflen: The number of bytes to copy |
Dave Gordon | 4dc7daf | 2015-06-30 14:58:52 -0700 | [diff] [blame] | 911 | * @skip: Number of bytes to skip before copying |
Akinobu Mita | df642ce | 2013-07-08 16:01:54 -0700 | [diff] [blame] | 912 | * |
| 913 | * Returns the number of copied bytes. |
| 914 | * |
| 915 | **/ |
| 916 | size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents, |
| 917 | void *buf, size_t buflen, off_t skip) |
| 918 | { |
| 919 | return sg_copy_buffer(sgl, nents, buf, buflen, skip, true); |
| 920 | } |
| 921 | EXPORT_SYMBOL(sg_pcopy_to_buffer); |
Johannes Thumshirn | 0945e56 | 2017-06-07 11:45:28 +0200 | [diff] [blame] | 922 | |
| 923 | /** |
| 924 | * sg_zero_buffer - Zero-out a part of a SG list |
| 925 | * @sgl: The SG list |
| 926 | * @nents: Number of SG entries |
| 927 | * @buflen: The number of bytes to zero out |
| 928 | * @skip: Number of bytes to skip before zeroing |
| 929 | * |
| 930 | * Returns the number of bytes zeroed. |
| 931 | **/ |
| 932 | size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents, |
| 933 | size_t buflen, off_t skip) |
| 934 | { |
| 935 | unsigned int offset = 0; |
| 936 | struct sg_mapping_iter miter; |
| 937 | unsigned int sg_flags = SG_MITER_ATOMIC | SG_MITER_TO_SG; |
| 938 | |
| 939 | sg_miter_start(&miter, sgl, nents, sg_flags); |
| 940 | |
| 941 | if (!sg_miter_skip(&miter, skip)) |
| 942 | return false; |
| 943 | |
| 944 | while (offset < buflen && sg_miter_next(&miter)) { |
| 945 | unsigned int len; |
| 946 | |
| 947 | len = min(miter.length, buflen - offset); |
| 948 | memset(miter.addr, 0, len); |
| 949 | |
| 950 | offset += len; |
| 951 | } |
| 952 | |
| 953 | sg_miter_stop(&miter); |
| 954 | return offset; |
| 955 | } |
| 956 | EXPORT_SYMBOL(sg_zero_buffer); |