blob: 5b78f3b1b918a7f5600c4222b89288481ed985c2 [file] [log] [blame]
Bjorn Helgaas7328c8f2018-01-26 11:45:16 -06001// SPDX-License-Identifier: GPL-2.0
Joerg Roedeldb3c33c2011-09-27 15:57:13 +02002/*
Bjorn Helgaasdf62ab52018-03-09 16:36:33 -06003 * PCI Express I/O Virtualization (IOV) support
Joerg Roedeldb3c33c2011-09-27 15:57:13 +02004 * Address Translation Service 1.0
Joerg Roedelc320b972011-09-27 15:57:15 +02005 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
Joerg Roedel086ac112011-09-27 15:57:16 +02006 * PASID support added by Joerg Roedel <joerg.roedel@amd.com>
Bjorn Helgaasdf62ab52018-03-09 16:36:33 -06007 *
8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
9 * Copyright (C) 2011 Advanced Micro Devices,
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020010 */
11
Paul Gortmaker363c75d2011-05-27 09:37:25 -040012#include <linux/export.h>
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020013#include <linux/pci-ats.h>
14#include <linux/pci.h>
James Bottomley8c451942011-11-29 19:20:23 +000015#include <linux/slab.h>
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020016
17#include "pci.h"
18
Bjorn Helgaasafdd5962015-07-17 15:35:18 -050019void pci_ats_init(struct pci_dev *dev)
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020020{
21 int pos;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020022
Gil Kupfercef74402018-05-10 17:56:02 -050023 if (pci_ats_disabled())
24 return;
25
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020026 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
27 if (!pos)
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050028 return;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020029
Bjorn Helgaasd544d752015-07-17 15:15:19 -050030 dev->ats_cap = pos;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020031}
32
33/**
34 * pci_enable_ats - enable the ATS capability
35 * @dev: the PCI device
36 * @ps: the IOMMU page shift
37 *
38 * Returns 0 on success, or negative on failure.
39 */
40int pci_enable_ats(struct pci_dev *dev, int ps)
41{
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020042 u16 ctrl;
Bjorn Helgaasc39127d2015-07-17 15:38:13 -050043 struct pci_dev *pdev;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020044
Bjorn Helgaasd544d752015-07-17 15:15:19 -050045 if (!dev->ats_cap)
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050046 return -EINVAL;
47
Bjorn Helgaasf7ef1342015-07-20 09:23:37 -050048 if (WARN_ON(dev->ats_enabled))
Bjorn Helgaasa021f302015-07-17 15:43:27 -050049 return -EBUSY;
50
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020051 if (ps < PCI_ATS_MIN_STU)
52 return -EINVAL;
53
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050054 /*
55 * Note that enabling ATS on a VF fails unless it's already enabled
56 * with the same STU on the PF.
57 */
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020058 ctrl = PCI_ATS_CTRL_ENABLE;
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050059 if (dev->is_virtfn) {
Bjorn Helgaasc39127d2015-07-17 15:38:13 -050060 pdev = pci_physfn(dev);
Bjorn Helgaasd544d752015-07-17 15:15:19 -050061 if (pdev->ats_stu != ps)
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050062 return -EINVAL;
63
Bjorn Helgaasd544d752015-07-17 15:15:19 -050064 atomic_inc(&pdev->ats_ref_cnt); /* count enabled VFs */
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050065 } else {
Bjorn Helgaasd544d752015-07-17 15:15:19 -050066 dev->ats_stu = ps;
67 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050068 }
Bjorn Helgaasd544d752015-07-17 15:15:19 -050069 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020070
Bjorn Helgaasd544d752015-07-17 15:15:19 -050071 dev->ats_enabled = 1;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020072 return 0;
73}
Joerg Roedeld4c06362011-09-27 15:57:14 +020074EXPORT_SYMBOL_GPL(pci_enable_ats);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020075
76/**
77 * pci_disable_ats - disable the ATS capability
78 * @dev: the PCI device
79 */
80void pci_disable_ats(struct pci_dev *dev)
81{
Bjorn Helgaasc39127d2015-07-17 15:38:13 -050082 struct pci_dev *pdev;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020083 u16 ctrl;
84
Bjorn Helgaasf7ef1342015-07-20 09:23:37 -050085 if (WARN_ON(!dev->ats_enabled))
Bjorn Helgaasa021f302015-07-17 15:43:27 -050086 return;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020087
Bjorn Helgaasd544d752015-07-17 15:15:19 -050088 if (atomic_read(&dev->ats_ref_cnt))
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050089 return; /* VFs still enabled */
90
91 if (dev->is_virtfn) {
Bjorn Helgaasc39127d2015-07-17 15:38:13 -050092 pdev = pci_physfn(dev);
Bjorn Helgaasd544d752015-07-17 15:15:19 -050093 atomic_dec(&pdev->ats_ref_cnt);
Bjorn Helgaasedc90fe2015-07-17 15:05:46 -050094 }
95
Bjorn Helgaasd544d752015-07-17 15:15:19 -050096 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020097 ctrl &= ~PCI_ATS_CTRL_ENABLE;
Bjorn Helgaasd544d752015-07-17 15:15:19 -050098 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +020099
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500100 dev->ats_enabled = 0;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200101}
Joerg Roedeld4c06362011-09-27 15:57:14 +0200102EXPORT_SYMBOL_GPL(pci_disable_ats);
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200103
Hao, Xudong1900ca12011-12-17 21:24:40 +0800104void pci_restore_ats_state(struct pci_dev *dev)
105{
106 u16 ctrl;
107
Bjorn Helgaasf7ef1342015-07-20 09:23:37 -0500108 if (!dev->ats_enabled)
Hao, Xudong1900ca12011-12-17 21:24:40 +0800109 return;
Hao, Xudong1900ca12011-12-17 21:24:40 +0800110
111 ctrl = PCI_ATS_CTRL_ENABLE;
112 if (!dev->is_virtfn)
Bjorn Helgaasd544d752015-07-17 15:15:19 -0500113 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
114 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
Hao, Xudong1900ca12011-12-17 21:24:40 +0800115}
116EXPORT_SYMBOL_GPL(pci_restore_ats_state);
117
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200118/**
119 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
120 * @dev: the PCI device
121 *
122 * Returns the queue depth on success, or negative on failure.
123 *
124 * The ATS spec uses 0 in the Invalidate Queue Depth field to
125 * indicate that the function can accept 32 Invalidate Request.
126 * But here we use the `real' values (i.e. 1~32) for the Queue
127 * Depth; and 0 indicates the function shares the Queue with
128 * other functions (doesn't exclusively own a Queue).
129 */
130int pci_ats_queue_depth(struct pci_dev *dev)
131{
Bjorn Helgaasa71f9382015-07-20 09:24:32 -0500132 u16 cap;
133
Bjorn Helgaas3c765392015-07-17 15:30:26 -0500134 if (!dev->ats_cap)
135 return -EINVAL;
136
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200137 if (dev->is_virtfn)
138 return 0;
139
Bjorn Helgaasa71f9382015-07-20 09:24:32 -0500140 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
141 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;
Joerg Roedeldb3c33c2011-09-27 15:57:13 +0200142}
Joerg Roedeld4c06362011-09-27 15:57:14 +0200143EXPORT_SYMBOL_GPL(pci_ats_queue_depth);
Joerg Roedelc320b972011-09-27 15:57:15 +0200144
145#ifdef CONFIG_PCI_PRI
146/**
147 * pci_enable_pri - Enable PRI capability
148 * @ pdev: PCI device structure
149 *
150 * Returns 0 on success, negative value on error
151 */
152int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
153{
154 u16 control, status;
155 u32 max_requests;
156 int pos;
157
Jean-Philippe Bruckera4f4fa62017-05-30 09:25:48 -0700158 if (WARN_ON(pdev->pri_enabled))
159 return -EBUSY;
160
Alex Williamson69166fb2011-11-02 14:07:15 -0600161 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
Joerg Roedelc320b972011-09-27 15:57:15 +0200162 if (!pos)
163 return -EINVAL;
164
Alex Williamson91f57d52011-11-11 10:07:36 -0700165 pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
CQ Tang4ebeb1e2017-05-30 09:25:49 -0700166 if (!(status & PCI_PRI_STATUS_STOPPED))
Joerg Roedelc320b972011-09-27 15:57:15 +0200167 return -EBUSY;
168
Alex Williamson91f57d52011-11-11 10:07:36 -0700169 pci_read_config_dword(pdev, pos + PCI_PRI_MAX_REQ, &max_requests);
Joerg Roedelc320b972011-09-27 15:57:15 +0200170 reqs = min(max_requests, reqs);
CQ Tang4ebeb1e2017-05-30 09:25:49 -0700171 pdev->pri_reqs_alloc = reqs;
Alex Williamson91f57d52011-11-11 10:07:36 -0700172 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs);
Joerg Roedelc320b972011-09-27 15:57:15 +0200173
CQ Tang4ebeb1e2017-05-30 09:25:49 -0700174 control = PCI_PRI_CTRL_ENABLE;
Alex Williamson91f57d52011-11-11 10:07:36 -0700175 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
Joerg Roedelc320b972011-09-27 15:57:15 +0200176
Jean-Philippe Bruckera4f4fa62017-05-30 09:25:48 -0700177 pdev->pri_enabled = 1;
178
Joerg Roedelc320b972011-09-27 15:57:15 +0200179 return 0;
180}
181EXPORT_SYMBOL_GPL(pci_enable_pri);
182
183/**
184 * pci_disable_pri - Disable PRI capability
185 * @pdev: PCI device structure
186 *
187 * Only clears the enabled-bit, regardless of its former value
188 */
189void pci_disable_pri(struct pci_dev *pdev)
190{
191 u16 control;
192 int pos;
193
Jean-Philippe Bruckera4f4fa62017-05-30 09:25:48 -0700194 if (WARN_ON(!pdev->pri_enabled))
195 return;
196
Alex Williamson69166fb2011-11-02 14:07:15 -0600197 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
Joerg Roedelc320b972011-09-27 15:57:15 +0200198 if (!pos)
199 return;
200
Alex Williamson91f57d52011-11-11 10:07:36 -0700201 pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
202 control &= ~PCI_PRI_CTRL_ENABLE;
203 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
Jean-Philippe Bruckera4f4fa62017-05-30 09:25:48 -0700204
205 pdev->pri_enabled = 0;
Joerg Roedelc320b972011-09-27 15:57:15 +0200206}
207EXPORT_SYMBOL_GPL(pci_disable_pri);
208
209/**
CQ Tang4ebeb1e2017-05-30 09:25:49 -0700210 * pci_restore_pri_state - Restore PRI
211 * @pdev: PCI device structure
212 */
213void pci_restore_pri_state(struct pci_dev *pdev)
214{
215 u16 control = PCI_PRI_CTRL_ENABLE;
216 u32 reqs = pdev->pri_reqs_alloc;
217 int pos;
218
219 if (!pdev->pri_enabled)
220 return;
221
222 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
223 if (!pos)
224 return;
225
226 pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs);
227 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
228}
229EXPORT_SYMBOL_GPL(pci_restore_pri_state);
230
231/**
Joerg Roedelc320b972011-09-27 15:57:15 +0200232 * pci_reset_pri - Resets device's PRI state
233 * @pdev: PCI device structure
234 *
235 * The PRI capability must be disabled before this function is called.
236 * Returns 0 on success, negative value on error.
237 */
238int pci_reset_pri(struct pci_dev *pdev)
239{
240 u16 control;
241 int pos;
242
Jean-Philippe Bruckera4f4fa62017-05-30 09:25:48 -0700243 if (WARN_ON(pdev->pri_enabled))
244 return -EBUSY;
245
Alex Williamson69166fb2011-11-02 14:07:15 -0600246 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
Joerg Roedelc320b972011-09-27 15:57:15 +0200247 if (!pos)
248 return -EINVAL;
249
CQ Tang4ebeb1e2017-05-30 09:25:49 -0700250 control = PCI_PRI_CTRL_RESET;
Alex Williamson91f57d52011-11-11 10:07:36 -0700251 pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
Joerg Roedelc320b972011-09-27 15:57:15 +0200252
253 return 0;
254}
255EXPORT_SYMBOL_GPL(pci_reset_pri);
Joerg Roedelc320b972011-09-27 15:57:15 +0200256#endif /* CONFIG_PCI_PRI */
Joerg Roedel086ac112011-09-27 15:57:16 +0200257
258#ifdef CONFIG_PCI_PASID
259/**
260 * pci_enable_pasid - Enable the PASID capability
261 * @pdev: PCI device structure
262 * @features: Features to enable
263 *
264 * Returns 0 on success, negative value on error. This function checks
265 * whether the features are actually supported by the device and returns
266 * an error if not.
267 */
268int pci_enable_pasid(struct pci_dev *pdev, int features)
269{
270 u16 control, supported;
271 int pos;
272
Jean-Philippe Bruckera4f4fa62017-05-30 09:25:48 -0700273 if (WARN_ON(pdev->pasid_enabled))
274 return -EBUSY;
275
Sinan Kaya7ce3f912018-06-30 11:24:24 -0400276 if (!pdev->eetlp_prefix_path)
277 return -EINVAL;
278
Alex Williamson69166fb2011-11-02 14:07:15 -0600279 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200280 if (!pos)
281 return -EINVAL;
282
Alex Williamson91f57d52011-11-11 10:07:36 -0700283 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
Alex Williamson91f57d52011-11-11 10:07:36 -0700284 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
Joerg Roedel086ac112011-09-27 15:57:16 +0200285
286 /* User wants to enable anything unsupported? */
287 if ((supported & features) != features)
288 return -EINVAL;
289
Alex Williamson91f57d52011-11-11 10:07:36 -0700290 control = PCI_PASID_CTRL_ENABLE | features;
CQ Tang4ebeb1e2017-05-30 09:25:49 -0700291 pdev->pasid_features = features;
Joerg Roedel086ac112011-09-27 15:57:16 +0200292
Alex Williamson91f57d52011-11-11 10:07:36 -0700293 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
Joerg Roedel086ac112011-09-27 15:57:16 +0200294
Jean-Philippe Bruckera4f4fa62017-05-30 09:25:48 -0700295 pdev->pasid_enabled = 1;
296
Joerg Roedel086ac112011-09-27 15:57:16 +0200297 return 0;
298}
299EXPORT_SYMBOL_GPL(pci_enable_pasid);
300
301/**
302 * pci_disable_pasid - Disable the PASID capability
303 * @pdev: PCI device structure
Joerg Roedel086ac112011-09-27 15:57:16 +0200304 */
305void pci_disable_pasid(struct pci_dev *pdev)
306{
307 u16 control = 0;
308 int pos;
309
Jean-Philippe Bruckera4f4fa62017-05-30 09:25:48 -0700310 if (WARN_ON(!pdev->pasid_enabled))
311 return;
312
Alex Williamson69166fb2011-11-02 14:07:15 -0600313 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200314 if (!pos)
315 return;
316
Alex Williamson91f57d52011-11-11 10:07:36 -0700317 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
Jean-Philippe Bruckera4f4fa62017-05-30 09:25:48 -0700318
319 pdev->pasid_enabled = 0;
Joerg Roedel086ac112011-09-27 15:57:16 +0200320}
321EXPORT_SYMBOL_GPL(pci_disable_pasid);
322
323/**
CQ Tang4ebeb1e2017-05-30 09:25:49 -0700324 * pci_restore_pasid_state - Restore PASID capabilities
325 * @pdev: PCI device structure
326 */
327void pci_restore_pasid_state(struct pci_dev *pdev)
328{
329 u16 control;
330 int pos;
331
332 if (!pdev->pasid_enabled)
333 return;
334
335 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
336 if (!pos)
337 return;
338
339 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features;
340 pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
341}
342EXPORT_SYMBOL_GPL(pci_restore_pasid_state);
343
344/**
Joerg Roedel086ac112011-09-27 15:57:16 +0200345 * pci_pasid_features - Check which PASID features are supported
346 * @pdev: PCI device structure
347 *
348 * Returns a negative value when no PASI capability is present.
349 * Otherwise is returns a bitmask with supported features. Current
350 * features reported are:
Alex Williamson91f57d52011-11-11 10:07:36 -0700351 * PCI_PASID_CAP_EXEC - Execute permission supported
Bjorn Helgaasf7625982013-11-14 11:28:18 -0700352 * PCI_PASID_CAP_PRIV - Privileged mode supported
Joerg Roedel086ac112011-09-27 15:57:16 +0200353 */
354int pci_pasid_features(struct pci_dev *pdev)
355{
356 u16 supported;
357 int pos;
358
Alex Williamson69166fb2011-11-02 14:07:15 -0600359 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200360 if (!pos)
361 return -EINVAL;
362
Alex Williamson91f57d52011-11-11 10:07:36 -0700363 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
Joerg Roedel086ac112011-09-27 15:57:16 +0200364
Alex Williamson91f57d52011-11-11 10:07:36 -0700365 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
Joerg Roedel086ac112011-09-27 15:57:16 +0200366
367 return supported;
368}
369EXPORT_SYMBOL_GPL(pci_pasid_features);
370
371#define PASID_NUMBER_SHIFT 8
372#define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT)
373/**
374 * pci_max_pasid - Get maximum number of PASIDs supported by device
375 * @pdev: PCI device structure
376 *
377 * Returns negative value when PASID capability is not present.
378 * Otherwise it returns the numer of supported PASIDs.
379 */
380int pci_max_pasids(struct pci_dev *pdev)
381{
382 u16 supported;
383 int pos;
384
Alex Williamson69166fb2011-11-02 14:07:15 -0600385 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
Joerg Roedel086ac112011-09-27 15:57:16 +0200386 if (!pos)
387 return -EINVAL;
388
Alex Williamson91f57d52011-11-11 10:07:36 -0700389 pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
Joerg Roedel086ac112011-09-27 15:57:16 +0200390
391 supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT;
392
393 return (1 << supported);
394}
395EXPORT_SYMBOL_GPL(pci_max_pasids);
396#endif /* CONFIG_PCI_PASID */