imrcpan/freebsd
1
0 Fork 0
mirror of https://github.com/freebsd/freebsd-src.git synced 2026-10-28 19:34:43 +01:00
Code Issues Packages Projects Releases Wiki Activity

OFED: Various changes from Linux 4.20

Browse Source 
This contains changes from the following Linux commits:

627212c9d49b RDMA/core: Replace open-coded variant of get_device
adee9f3f3bbb RDMA/core: Depend on device_add() to add device attributes
954a8e3aea87 RDMA/cma: Protect cma dev list with lock
722c7b2bfead RDMA/{cma, core}: Avoid callback on rdma_addr_cancel()
f9d08f1e1939 RDMA/core: Rate limit MAD error messages
e1f540c3ed0e RDMA/core: Define client_data_lock as rwlock instead of spinlock
2d65f49ff961 RDMA/core: Use simpler spin lock irq API from blocking context
4512acd0d34c RDMA/core: Remove context entries from list while unregistering device
f7b65d9bf2db RDMA/core: Use simplified list_for_each
93688ddbe1da RDMA/core: No need to protect kfree with spin lock and semaphore
50704e039ab1 RDMA/umem: Restore lockdep check while downgrading lock
eb93c82ed8c7 RDMA/core: Document QP @event_handler function
4269024639f6 RDMA/core: Document CM @event_handler function
77addc524473 RDMA/core: Rename rdma_copy_addr to rdma_copy_src_l2_addr
a362ea1d9e1a RDMA/core: Introduce and use rdma_set_src_addr() between IPv4 and IPv6
89c5691cdd95 RDMA/core: Let protocol specific function typecast sockaddr structure
caf1e3ae9fa6 RDMA/core Introduce and use rdma_find_ndev_for_src_ip_rcu
c31d4b2ddf07 RDMA/core: Protect against changing dst->dev during destination resolve
0e9d2c19bff1 RDMA/core: Consider net ns of gid attribute for RoCE
d6b1764a8c5a RDMA/core: Introduce rdma_read_gid_attr_ndev_rcu() to check GID attribute
6aaecd385685 RDMA/core: Simplify roce_resolve_route_from_path()
0965cc953a23 RDMA/core: Properly return the error code of rdma_set_src_addr_rcu
d4b4dd1b9706 RDMA/umem: Do not use current->tgid to track the mm_struct
c9990ab39b6e RDMA/umem: Move all the ODP related stuff out of ucontext and into per_mm
597ecc5a0954 RDMA/umem: Get rid of struct ib_umem.odp_data
41b4deeaa123 RDMA/umem: Make ib_umem_odp into a sub structure of ib_umem
b5231b019d76 RDMA/umem: Use ib_umem_odp in all function signatures connected to ODP
56ac9dd9177c RDMA/umem: Avoid synchronize_srcu in the ODP MR destruction path
be7a57b41ad8 RDMA/umem: Handle a half-complete start/end sequence
ca748c39ea3f RDMA/umem: Get rid of per_mm->notifier_count
f27a0d50a4bc RDMA/umem: Use umem->owning_mm inside ODP

Tested by:	Wafa Hamzah <[email protected]> (mlx5_ib)
Tested by:	John Baldwin <[email protected]> (iw_cxgbe)
Sponsored by:	Chelsio Communications
This commit is contained in:
John Baldwin committed 2026-06-24 15:30:56 -04:00
1 parent 6a75471dbc
commit 4726b80d93
17 files changed
+779 -671
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

sys/ofed/drivers/infiniband/core/core_priv.h
+9 -1
View File
@@ -190,8 +190,16 @@ int rdma_resolve_ip_route(struct sockaddr *src_addr,
int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
const union ib_gid *dgid,
u8 *dmac, if_t ndev,
u8 *dmac, const struct ib_gid_attr *sgid_attr,
int *hoplimit);
void rdma_copy_src_l2_addr(struct rdma_dev_addr *dev_addr,
const if_t dev);
struct sa_path_rec;
int roce_resolve_route_from_path(struct sa_path_rec *rec,
const struct ib_gid_attr *attr);
if_t rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr);
struct rdma_umap_priv {
struct vm_area_struct *vma;
sys/ofed/drivers/infiniband/core/ib_addr.c
+222 -95
View File
@@ -45,6 +45,7 @@
#include <net/route/nhop.h>
#include <net/netevent.h>
#include <rdma/ib_addr.h>
#include <rdma/ib_sa.h>
#include <rdma/ib.h>
#include <netinet/in_fib.h>
@@ -66,6 +67,7 @@ struct addr_req {
struct rdma_dev_addr *addr, void *context);
unsigned long timeout;
struct delayed_work work;
bool resolve_by_gid_attr; /* Consider gid attr in resolve phase */
int status;
};
@@ -113,9 +115,17 @@ rdma_copy_addr_sub(u8 *dst, const u8 *src, unsigned min, unsigned max)
memset(dst + min, 0, max - min);
}
void rdma_copy_addr(struct rdma_dev_addr *dev_addr,
const if_t dev,
const unsigned char *dst_dev_addr)
/**
* rdma_copy_src_l2_addr - Copy netdevice source addresses
* @dev_addr: Destination address pointer where to copy the addresses
* @dev: Netdevice whose source addresses to copy
*
* rdma_copy_src_l2_addr() copies source addresses from the specified netdevice.
* This includes unicast address, broadcast address, device type and
* interface index.
*/
void rdma_copy_src_l2_addr(struct rdma_dev_addr *dev_addr,
const if_t dev)
{
int dev_type = if_gettype(dev);
@@ -137,13 +147,36 @@ void rdma_copy_addr(struct rdma_dev_addr *dev_addr,
if_getaddrlen(dev), MAX_ADDR_LEN);
rdma_copy_addr_sub(dev_addr->broadcast, if_getbroadcastaddr(dev),
if_getaddrlen(dev), MAX_ADDR_LEN);
if (dst_dev_addr != NULL) {
rdma_copy_addr_sub(dev_addr->dst_dev_addr, dst_dev_addr,
if_getaddrlen(dev), MAX_ADDR_LEN);
}
dev_addr->bound_dev_if = if_getindex(dev);
}
EXPORT_SYMBOL(rdma_copy_addr);
EXPORT_SYMBOL(rdma_copy_src_l2_addr);
static if_t
rdma_find_ndev_for_src_ip_rcu(struct vnet *net, const struct sockaddr *src_in)
{
if_t dev = NULL;
int ret = -EADDRNOTAVAIL;
switch (src_in->sa_family) {
#ifdef INET
case AF_INET:
dev = ip_ifp_find(net,
((const struct sockaddr_in *)src_in)->sin_addr.s_addr);
if (dev)
ret = 0;
break;
#endif
#ifdef INET6
case AF_INET6:
dev = ip6_ifp_find(net,
((const struct sockaddr_in6 *)src_in)->sin6_addr, 0);
if (dev)
ret = 0;
break;
#endif
}
return ret ? ERR_PTR(ret) : dev;
}
int rdma_translate_ip(const struct sockaddr *addr,
struct rdma_dev_addr *dev_addr)
@@ -153,34 +186,22 @@ int rdma_translate_ip(const struct sockaddr *addr,
if (dev_addr->bound_dev_if) {
dev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
} else switch (addr->sa_family) {
#ifdef INET
case AF_INET:
dev = ip_ifp_find(dev_addr->net,
((const struct sockaddr_in *)addr)->sin_addr.s_addr);
break;
#endif
#ifdef INET6
case AF_INET6:
dev = ip6_ifp_find(dev_addr->net,
((const struct sockaddr_in6 *)addr)->sin6_addr, 0);
break;
#endif
default:
dev = NULL;
break;
if (!dev)
return -ENODEV;
} else {
rcu_read_lock();
dev = rdma_find_ndev_for_src_ip_rcu(dev_addr->net, addr);
rcu_read_unlock();
if (IS_ERR(dev))
return PTR_ERR(dev);
}
if (dev != NULL) {
/* disallow connections through 127.0.0.1 itself */
if (if_getflags(dev) & IFF_LOOPBACK)
ret = -EINVAL;
else
rdma_copy_addr(dev_addr, dev, NULL);
dev_put(dev);
} else {
ret = -ENODEV;
}
/* disallow connections through 127.0.0.1 itself */
if (if_getflags(dev) & IFF_LOOPBACK)
ret = -EINVAL;
else
rdma_copy_src_l2_addr(dev_addr, dev);
dev_put(dev);
return ret;
}
EXPORT_SYMBOL(rdma_translate_ip);
@@ -226,12 +247,15 @@ static int addr_resolve_multi(u8 *edst, if_t ifp, struct sockaddr *dst_in)
#endif
#ifdef INET
static int addr4_resolve(struct sockaddr_in *src_in,
const struct sockaddr_in *dst_in,
static int addr4_resolve(struct sockaddr *src_sock,
const struct sockaddr *dst_sock,
struct rdma_dev_addr *addr,
u8 *edst,
if_t *ifpp)
{
struct sockaddr_in *src_in = (struct sockaddr_in *)src_sock;
const struct sockaddr_in *dst_in =
(const struct sockaddr_in *)dst_sock;
enum {
ADDR_VALID = 0,
ADDR_SRC_ANY = 1,
@@ -398,8 +422,8 @@ done:
return (-error);
}
#else
static int addr4_resolve(struct sockaddr_in *src_in,
const struct sockaddr_in *dst_in,
static int addr4_resolve(struct sockaddr *src_sock,
const struct sockaddr *dst_sock,
struct rdma_dev_addr *addr,
u8 *edst,
if_t *ifpp)
@@ -409,12 +433,15 @@ static int addr4_resolve(struct sockaddr_in *src_in,
#endif
#ifdef INET6
static int addr6_resolve(struct sockaddr_in6 *src_in,
const struct sockaddr_in6 *dst_in,
static int addr6_resolve(struct sockaddr *src_sock,
const struct sockaddr *dst_sock,
struct rdma_dev_addr *addr,
u8 *edst,
if_t *ifpp)
{
struct sockaddr_in6 *src_in = (struct sockaddr_in6 *)src_sock;
const struct sockaddr_in6 *dst_in =
(const struct sockaddr_in6 *)dst_sock;
enum {
ADDR_VALID = 0,
ADDR_SRC_ANY = 1,
@@ -586,8 +613,8 @@ done:
return (-error);
}
#else
static int addr6_resolve(struct sockaddr_in6 *src_in,
const struct sockaddr_in6 *dst_in,
static int addr6_resolve(struct sockaddr *src_sock,
const struct sockaddr *dst_sock,
struct rdma_dev_addr *addr,
u8 *edst,
if_t *ifpp)
@@ -599,11 +626,12 @@ static int addr6_resolve(struct sockaddr_in6 *src_in,
static int addr_resolve_neigh(if_t dev,
const struct sockaddr *dst_in,
u8 *edst,
struct rdma_dev_addr *addr)
struct rdma_dev_addr *addr,
unsigned int ndev_flags)
{
if (if_getflags(dev) & IFF_LOOPBACK) {
int ret;
int ret = 0;
if (ndev_flags & IFF_LOOPBACK) {
/*
* Binding to a loopback device is not allowed. Make
* sure the destination device address is global by
@@ -612,66 +640,143 @@ static int addr_resolve_neigh(if_t dev,
if (addr->bound_dev_if == if_getindex(dev))
addr->bound_dev_if = 0;
ret = rdma_translate_ip(dst_in, addr);
if (ret == 0) {
memcpy(addr->dst_dev_addr, addr->src_dev_addr,
MAX_ADDR_LEN);
memcpy(addr->dst_dev_addr, addr->src_dev_addr, MAX_ADDR_LEN);
} else {
if (!(ndev_flags & IFF_NOARP)) {
/* If the device doesn't do ARP internally */
memcpy(addr->dst_dev_addr, edst, MAX_ADDR_LEN);
}
return ret;
}
/* If the device doesn't do ARP internally */
if (!(if_getflags(dev) & IFF_NOARP)) {
rdma_copy_addr(addr, dev, edst);
return 0;
}
rdma_copy_addr(addr, dev, NULL);
return ret;
}
static int copy_src_l2_addr(struct rdma_dev_addr *dev_addr,
const struct sockaddr *dst_in,
if_t ndev)
{
int ret = 0;
if (if_getflags(ndev) & IFF_LOOPBACK)
ret = rdma_translate_ip(dst_in, dev_addr);
else
rdma_copy_src_l2_addr(dev_addr, ndev);
return ret;
}
static int rdma_set_src_addr_rcu(struct rdma_dev_addr *dev_addr,
unsigned int *ndev_flags,
const struct sockaddr *dst_in,
if_t ndev)
{
*ndev_flags = if_getflags(ndev);
/* A physical device must be the RDMA device to use */
if (if_getflags(ndev) & IFF_LOOPBACK) {
/*
* RDMA (IB/RoCE, iWarp) doesn't run on lo interface or
* loopback IP address. So if route is resolved to loopback
* interface, translate that to a real ndev based on non
* loopback IP address.
*/
ndev = rdma_find_ndev_for_src_ip_rcu(dev_net(ndev), dst_in);
if (!ndev)
return -ENODEV;
}
return copy_src_l2_addr(dev_addr, dst_in, ndev);
}
static int set_addr_netns_by_gid_rcu(struct rdma_dev_addr *addr)
{
if_t ndev;
ndev = rdma_read_gid_attr_ndev_rcu(addr->sgid_attr);
if (IS_ERR(ndev))
return PTR_ERR(ndev);
/*
* Since we are holding the rcu, reading net and ifindex
* are safe without any additional reference; because
* change_net_namespace() in net/core/ib_dev.c does rcu sync
* after it changes the state to IFF_DOWN and before
* updating netdev fields {net, ifindex}.
*/
addr->net = dev_net(ndev);
addr->bound_dev_if = if_getindex(ndev);
return 0;
}
static void rdma_addr_set_net_defaults(struct rdma_dev_addr *addr)
{
addr->net = &init_net;
addr->bound_dev_if = 0;
}
static int addr_resolve(struct sockaddr *src_in,
const struct sockaddr *dst_in,
struct rdma_dev_addr *addr)
struct rdma_dev_addr *addr,
bool resolve_by_gid_attr)
{
struct epoch_tracker et;
if_t ndev = NULL;
u8 edst[MAX_ADDR_LEN];
unsigned int ndev_flags = 0;
int ret;
if (dst_in->sa_family != src_in->sa_family)
return -EINVAL;
NET_EPOCH_ENTER(et);
if (resolve_by_gid_attr) {
if (!addr->sgid_attr) {
NET_EPOCH_ENTER(et);
pr_warn_ratelimited("%s: missing gid_attr\n", __func__);
return -EINVAL;
}
/*
* If the request is for a specific gid attribute of the
* rdma_dev_addr, derive net from the netdevice of the
* GID attribute.
*/
ret = set_addr_netns_by_gid_rcu(addr);
if (ret) {
NET_EPOCH_ENTER(et);
return ret;
}
}
switch (src_in->sa_family) {
case AF_INET:
ret = addr4_resolve((struct sockaddr_in *)src_in,
(const struct sockaddr_in *)dst_in,
addr, edst, &ndev);
ret = addr4_resolve(src_in, dst_in, addr, edst, &ndev);
break;
case AF_INET6:
ret = addr6_resolve((struct sockaddr_in6 *)src_in,
(const struct sockaddr_in6 *)dst_in, addr,
edst, &ndev);
ret = addr6_resolve(src_in, dst_in, addr, edst, &ndev);
break;
default:
ret = -EADDRNOTAVAIL;
break;
}
if (ret) {
NET_EPOCH_EXIT(et);
goto done;
}
ret = rdma_set_src_addr_rcu(addr, &ndev_flags, dst_in, ndev);
NET_EPOCH_EXIT(et);
/* check for error */
if (ret != 0)
return ret;
/* store MAC addresses and check for loopback */
ret = addr_resolve_neigh(ndev, dst_in, edst, addr);
if (!ret)
ret = addr_resolve_neigh(ndev, dst_in, edst, addr, ndev_flags);
/* set belonging VNET, if any */
addr->net = dev_net(ndev);
dev_put(ndev);
done:
/*
* Clear the addr net to go back to its original state, only if it was
* derived from GID attribute in this context.
*/
if (resolve_by_gid_attr)
rdma_addr_set_net_defaults(addr);
return ret;
}
@@ -685,7 +790,8 @@ static void process_one_req(struct work_struct *_work)
if (req->status == -ENODATA) {
src_in = (struct sockaddr *)&req->src_addr;
dst_in = (struct sockaddr *)&req->dst_addr;
req->status = addr_resolve(src_in, dst_in, req->addr);
req->status = addr_resolve(src_in, dst_in, req->addr,
req->resolve_by_gid_attr);
if (req->status && time_after_eq(jiffies, req->timeout)) {
req->status = -ETIMEDOUT;
} else if (req->status == -ENODATA) {
@@ -720,6 +826,7 @@ int rdma_resolve_ip(struct sockaddr *src_addr, const struct sockaddr *dst_addr,
struct rdma_dev_addr *addr, int timeout_ms,
void (*callback)(int status, struct sockaddr *src_addr,
struct rdma_dev_addr *addr, void *context),
bool resolve_by_gid_attr,
void *context)
{
struct sockaddr *src_in, *dst_in;
@@ -748,9 +855,11 @@ int rdma_resolve_ip(struct sockaddr *src_addr, const struct sockaddr *dst_addr,
req->addr = addr;
req->callback = callback;
req->context = context;
req->resolve_by_gid_attr = resolve_by_gid_attr;
INIT_DELAYED_WORK(&req->work, process_one_req);
req->status = addr_resolve(src_in, dst_in, addr);
req->status = addr_resolve(src_in, dst_in, addr,
req->resolve_by_gid_attr);
switch (req->status) {
case 0:
req->timeout = jiffies;
@@ -771,25 +880,49 @@ err:
}
EXPORT_SYMBOL(rdma_resolve_ip);
int rdma_resolve_ip_route(struct sockaddr *src_addr,
const struct sockaddr *dst_addr,
struct rdma_dev_addr *addr)
int roce_resolve_route_from_path(struct sa_path_rec *rec,
const struct ib_gid_attr *attr)
{
struct sockaddr_storage ssrc_addr = {};
struct sockaddr *src_in = (struct sockaddr *)&ssrc_addr;
union rdma_sockaddr sgid, dgid;
struct rdma_dev_addr dev_addr = {};
int ret;
if (src_addr) {
if (src_addr->sa_family != dst_addr->sa_family)
return -EINVAL;
if (rec->roce.route_resolved)
return 0;
memcpy(src_in, src_addr, rdma_addr_size(src_addr));
} else {
src_in->sa_family = dst_addr->sa_family;
}
rdma_gid2ip(&sgid._sockaddr, &rec->sgid);
rdma_gid2ip(&dgid._sockaddr, &rec->dgid);
return addr_resolve(src_in, dst_addr, addr);
if (sgid._sockaddr.sa_family != dgid._sockaddr.sa_family)
return -EINVAL;
if (!attr || !attr->ndev)
return -EINVAL;
dev_addr.net = &init_net;
dev_addr.sgid_attr = attr;
ret = addr_resolve(&sgid._sockaddr, &dgid._sockaddr,
&dev_addr, true);
if (ret)
return ret;
if ((dev_addr.network == RDMA_NETWORK_IPV4 ||
dev_addr.network == RDMA_NETWORK_IPV6) &&
rec->rec_type != SA_PATH_REC_TYPE_ROCE_V2)
return -EINVAL;
rec->roce.route_resolved = true;
return 0;
}
/**
* rdma_addr_cancel - Cancel resolve ip request
* @addr: Pointer to address structure given previously
* during rdma_resolve_ip().
* rdma_addr_cancel() is synchronous function which cancels any pending
* request if there is any.
*/
void rdma_addr_cancel(struct rdma_dev_addr *addr)
{
struct addr_req *req, *temp_req;
@@ -817,11 +950,6 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr)
* guarentees no work is running and none will be started.
*/
cancel_delayed_work_sync(&found->work);
if (found->callback)
found->callback(-ECANCELED, (struct sockaddr *)&found->src_addr,
found->addr, found->context);
kfree(found);
}
EXPORT_SYMBOL(rdma_addr_cancel);
@@ -840,7 +968,7 @@ static void resolve_cb(int status, struct sockaddr *src_addr,
int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
const union ib_gid *dgid,
u8 *dmac, if_t ndev,
u8 *dmac, const struct ib_gid_attr *sgid_attr,
int *hoplimit)
{
struct rdma_dev_addr dev_addr;
@@ -852,13 +980,12 @@ int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
rdma_gid2ip(&dgid_addr._sockaddr, dgid);
memset(&dev_addr, 0, sizeof(dev_addr));
dev_addr.bound_dev_if = if_getindex(ndev);
dev_addr.net = dev_net(ndev);
dev_addr.net = &init_net;
dev_addr.sgid_attr = sgid_attr;
init_completion(&ctx.comp);
ret = rdma_resolve_ip(&sgid_addr._sockaddr, &dgid_addr._sockaddr,
&dev_addr, 1000, resolve_cb, &ctx);
&dev_addr, 1000, resolve_cb, true, &ctx);
if (ret)
return ret;
sys/ofed/drivers/infiniband/core/ib_cache.c
+33
View File
@@ -1265,6 +1265,39 @@ void rdma_hold_gid_attr(const struct ib_gid_attr *attr)
}
EXPORT_SYMBOL(rdma_hold_gid_attr);
/**
* rdma_read_gid_attr_ndev_rcu - Read GID attribute netdevice
* which must be in UP state.
*
* @attr:Pointer to the GID attribute
*
* Returns pointer to netdevice if the netdevice was attached to GID and
* netdevice is in UP state. Caller must hold RCU lock as this API
* reads the netdev flags which can change while netdevice migrates to
* different net namespace. Returns ERR_PTR with error code otherwise.
*
*/
if_t rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr)
{
struct ib_gid_table_entry *entry =
container_of(attr, struct ib_gid_table_entry, attr);
struct ib_device *device = entry->attr.device;
if_t ndev = ERR_PTR(-ENODEV);
u8 port_num = entry->attr.port_num;
struct ib_gid_table *table;
unsigned long flags;
bool valid;
table = rdma_gid_table(device, port_num);
read_lock_irqsave(&table->rwlock, flags);
valid = is_gid_entry_valid(table->data_vec[attr->index]);
if (valid && attr->ndev && (if_getflags(attr->ndev) & IFF_UP))
ndev = attr->ndev;
read_unlock_irqrestore(&table->rwlock, flags);
return ndev;
}
static int config_non_roce_gid_cache(struct ib_device *device,
u8 port, int gid_tbl_len)
{
sys/ofed/drivers/infiniband/core/ib_cma.c
+13 -14
View File
@@ -729,6 +729,7 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
dgid = (union ib_gid *) &addr->sib_addr;
pkey = ntohs(addr->sib_pkey);
mutex_lock(&lock);
list_for_each_entry(cur_dev, &dev_list, list) {
for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) {
if (!rdma_cap_af_ib(cur_dev->device, p))
@@ -755,18 +756,19 @@ static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
cma_dev = cur_dev;
sgid = gid;
id_priv->id.port_num = p;
goto found;
}
}
}
}
if (!cma_dev)
return -ENODEV;
mutex_unlock(&lock);
return -ENODEV;
found:
cma_attach_to_dev(id_priv, cma_dev);
addr = (struct sockaddr_ib *) cma_src_addr(id_priv);
memcpy(&addr->sib_addr, &sgid, sizeof sgid);
mutex_unlock(&lock);
addr = (struct sockaddr_ib *)cma_src_addr(id_priv);
memcpy(&addr->sib_addr, &sgid, sizeof(sgid));
cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr);
return 0;
}
@@ -2104,7 +2106,7 @@ cma_ib_new_conn_id(const struct rdma_cm_id *listen_id,
rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
if (net_dev) {
rdma_copy_addr(&rt->addr.dev_addr, net_dev, NULL);
rdma_copy_src_l2_addr(&rt->addr.dev_addr, net_dev);
} else {
if (!cma_protocol_roce(listen_id) &&
cma_any_addr(vnet, cma_src_addr(id_priv))) {
@@ -2151,7 +2153,7 @@ cma_ib_new_udp_id(const struct rdma_cm_id *listen_id,
goto err;
if (net_dev) {
rdma_copy_addr(&id->route.addr.dev_addr, net_dev, NULL);
rdma_copy_src_l2_addr(&id->route.addr.dev_addr, net_dev);
} else {
if (!cma_any_addr(vnet, cma_src_addr(id_priv))) {
ret = cma_translate_addr(cma_src_addr(id_priv),
@@ -3067,13 +3069,11 @@ static void addr_handler(int status, struct sockaddr *src_addr,
if (id_priv->id.event_handler(&id_priv->id, &event)) {
cma_exch(id_priv, RDMA_CM_DESTROYING);
mutex_unlock(&id_priv->handler_mutex);
cma_deref_id(id_priv);
rdma_destroy_id(&id_priv->id);
return;
}
out:
mutex_unlock(&id_priv->handler_mutex);
cma_deref_id(id_priv);
}
static int cma_resolve_loopback(struct rdma_id_private *id_priv)
@@ -3171,7 +3171,6 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
return -EINVAL;
memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
atomic_inc(&id_priv->refcount);
if (cma_any_addr(vnet, dst_addr)) {
ret = cma_resolve_loopback(id_priv);
} else {
@@ -3182,9 +3181,10 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
if (ret)
goto err;
ret = rdma_resolve_ip(cma_src_addr(id_priv),
dst_addr, &id->route.addr.dev_addr,
timeout_ms, addr_handler, id_priv);
ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
&id->route.addr.dev_addr,
timeout_ms, addr_handler,
false, id_priv);
}
}
if (ret)
@@ -3193,7 +3193,6 @@ int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
return 0;
err:
cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
cma_deref_id(id_priv);
return ret;
}
EXPORT_SYMBOL(rdma_resolve_addr);
sys/ofed/drivers/infiniband/core/ib_device.c
+26 -25
View File
@@ -217,7 +217,7 @@ struct ib_device *ib_alloc_device(size_t size)
INIT_LIST_HEAD(&device->event_handler_list);
spin_lock_init(&device->event_handler_lock);
spin_lock_init(&device->client_data_lock);
rwlock_init(&device->client_data_lock);
INIT_LIST_HEAD(&device->client_data_list);
INIT_LIST_HEAD(&device->port_list);
@@ -233,6 +233,7 @@ EXPORT_SYMBOL(ib_alloc_device);
*/
void ib_dealloc_device(struct ib_device *device)
{
WARN_ON(!list_empty(&device->client_data_list));
WARN_ON(device->reg_state != IB_DEV_UNREGISTERED &&
device->reg_state != IB_DEV_UNINITIALIZED);
kobject_put(&device->dev.kobj);
@@ -242,9 +243,8 @@ EXPORT_SYMBOL(ib_dealloc_device);
static int add_client_context(struct ib_device *device, struct ib_client *client)
{
struct ib_client_data *context;
unsigned long flags;
context = kmalloc(sizeof *context, GFP_KERNEL);
context = kmalloc(sizeof(*context), GFP_KERNEL);
if (!context)
return -ENOMEM;
@@ -253,9 +253,9 @@ static int add_client_context(struct ib_device *device, struct ib_client *client
context->going_down = false;
down_write(&lists_rwsem);
spin_lock_irqsave(&device->client_data_lock, flags);
write_lock_irq(&device->client_data_lock);
list_add(&context->list, &device->client_data_list);
spin_unlock_irqrestore(&device->client_data_lock, flags);
write_unlock_irq(&device->client_data_lock);
up_write(&lists_rwsem);
return 0;
@@ -403,14 +403,13 @@ void ib_unregister_device(struct ib_device *device)
down_write(&lists_rwsem);
list_del(&device->core_list);
spin_lock_irqsave(&device->client_data_lock, flags);
list_for_each_entry_safe(context, tmp, &device->client_data_list, list)
write_lock_irq(&device->client_data_lock);
list_for_each_entry(context, &device->client_data_list, list)
context->going_down = true;
spin_unlock_irqrestore(&device->client_data_lock, flags);
write_unlock_irq(&device->client_data_lock);
downgrade_write(&lists_rwsem);
list_for_each_entry_safe(context, tmp, &device->client_data_list,
list) {
list_for_each_entry(context, &device->client_data_list, list) {
if (context->client->remove)
context->client->remove(device, context->data);
}
@@ -423,10 +422,13 @@ void ib_unregister_device(struct ib_device *device)
ib_cache_cleanup_one(device);
down_write(&lists_rwsem);
spin_lock_irqsave(&device->client_data_lock, flags);
list_for_each_entry_safe(context, tmp, &device->client_data_list, list)
write_lock_irqsave(&device->client_data_lock, flags);
list_for_each_entry_safe(context, tmp, &device->client_data_list,
list) {
list_del(&context->list);
kfree(context);
spin_unlock_irqrestore(&device->client_data_lock, flags);
}
write_unlock_irqrestore(&device->client_data_lock, flags);
up_write(&lists_rwsem);
device->reg_state = IB_DEV_UNREGISTERED;
@@ -476,9 +478,8 @@ EXPORT_SYMBOL(ib_register_client);
*/
void ib_unregister_client(struct ib_client *client)
{
struct ib_client_data *context, *tmp;
struct ib_client_data *context;
struct ib_device *device;
unsigned long flags;
mutex_lock(&device_mutex);
@@ -490,14 +491,14 @@ void ib_unregister_client(struct ib_client *client)
struct ib_client_data *found_context = NULL;
down_write(&lists_rwsem);
spin_lock_irqsave(&device->client_data_lock, flags);
list_for_each_entry_safe(context, tmp, &device->client_data_list, list)
write_lock_irq(&device->client_data_lock);
list_for_each_entry(context, &device->client_data_list, list)
if (context->client == client) {
context->going_down = true;
found_context = context;
break;
}
spin_unlock_irqrestore(&device->client_data_lock, flags);
write_unlock_irq(&device->client_data_lock);
up_write(&lists_rwsem);
if (client->remove)
@@ -511,11 +512,11 @@ void ib_unregister_client(struct ib_client *client)
}
down_write(&lists_rwsem);
spin_lock_irqsave(&device->client_data_lock, flags);
write_lock_irq(&device->client_data_lock);
list_del(&found_context->list);
kfree(found_context);
spin_unlock_irqrestore(&device->client_data_lock, flags);
write_unlock_irq(&device->client_data_lock);
up_write(&lists_rwsem);
kfree(found_context);
}
mutex_unlock(&device_mutex);
@@ -536,13 +537,13 @@ void *ib_get_client_data(struct ib_device *device, struct ib_client *client)
void *ret = NULL;
unsigned long flags;
spin_lock_irqsave(&device->client_data_lock, flags);
read_lock_irqsave(&device->client_data_lock, flags);
list_for_each_entry(context, &device->client_data_list, list)
if (context->client == client) {
ret = context->data;
break;
}
spin_unlock_irqrestore(&device->client_data_lock, flags);
read_unlock_irqrestore(&device->client_data_lock, flags);
return ret;
}
@@ -563,7 +564,7 @@ void ib_set_client_data(struct ib_device *device, struct ib_client *client,
struct ib_client_data *context;
unsigned long flags;
spin_lock_irqsave(&device->client_data_lock, flags);
write_lock_irqsave(&device->client_data_lock, flags);
list_for_each_entry(context, &device->client_data_list, list)
if (context->client == client) {
context->data = data;
@@ -574,7 +575,7 @@ void ib_set_client_data(struct ib_device *device, struct ib_client *client,
device->name, client->name);
out:
spin_unlock_irqrestore(&device->client_data_lock, flags);
write_unlock_irqrestore(&device->client_data_lock, flags);
}
EXPORT_SYMBOL(ib_set_client_data);
sys/ofed/drivers/infiniband/core/ib_mad.c
+37 -35
View File
@@ -224,30 +224,30 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
/* Validate parameters */
qpn = get_spl_qp_index(qp_type);
if (qpn == -1) {
dev_notice(&device->dev,
"ib_register_mad_agent: invalid QP Type %d\n",
qp_type);
dev_dbg_ratelimited(&device->dev, "%s: invalid QP Type %d\n",
__func__, qp_type);
goto error1;
}
if (rmpp_version && rmpp_version != IB_MGMT_RMPP_VERSION) {
dev_notice(&device->dev,
"ib_register_mad_agent: invalid RMPP Version %u\n",
rmpp_version);
dev_dbg_ratelimited(&device->dev,
"%s: invalid RMPP Version %u\n",
__func__, rmpp_version);
goto error1;
}
/* Validate MAD registration request if supplied */
if (mad_reg_req) {
if (mad_reg_req->mgmt_class_version >= MAX_MGMT_VERSION) {
dev_notice(&device->dev,
"ib_register_mad_agent: invalid Class Version %u\n",
mad_reg_req->mgmt_class_version);
dev_dbg_ratelimited(&device->dev,
"%s: invalid Class Version %u\n",
__func__,
mad_reg_req->mgmt_class_version);
goto error1;
}
if (!recv_handler) {
dev_notice(&device->dev,
"ib_register_mad_agent: no recv_handler\n");
dev_dbg_ratelimited(&device->dev,
"%s: no recv_handler\n", __func__);
goto error1;
}
if (mad_reg_req->mgmt_class >= MAX_MGMT_CLASS) {
@@ -257,9 +257,9 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
*/
if (mad_reg_req->mgmt_class !=
IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
dev_notice(&device->dev,
"ib_register_mad_agent: Invalid Mgmt Class 0x%x\n",
mad_reg_req->mgmt_class);
dev_dbg_ratelimited(&device->dev,
"%s: Invalid Mgmt Class 0x%x\n",
__func__, mad_reg_req->mgmt_class);
goto error1;
}
} else if (mad_reg_req->mgmt_class == 0) {
@@ -267,8 +267,9 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
* Class 0 is reserved in IBA and is used for
* aliasing of IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
*/
dev_notice(&device->dev,
"ib_register_mad_agent: Invalid Mgmt Class 0\n");
dev_dbg_ratelimited(&device->dev,
"%s: Invalid Mgmt Class 0\n",
__func__);
goto error1;
} else if (is_vendor_class(mad_reg_req->mgmt_class)) {
/*
@@ -276,18 +277,19 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
* ensure supplied OUI is not zero
*/
if (!is_vendor_oui(mad_reg_req->oui)) {
dev_notice(&device->dev,
"ib_register_mad_agent: No OUI specified for class 0x%x\n",
mad_reg_req->mgmt_class);
dev_dbg_ratelimited(&device->dev,
"%s: No OUI specified for class 0x%x\n",
__func__,
mad_reg_req->mgmt_class);
goto error1;
}
}
/* Make sure class supplied is consistent with RMPP */
if (!ib_is_mad_class_rmpp(mad_reg_req->mgmt_class)) {
if (rmpp_version) {
dev_notice(&device->dev,
"ib_register_mad_agent: RMPP version for non-RMPP class 0x%x\n",
mad_reg_req->mgmt_class);
dev_dbg_ratelimited(&device->dev,
"%s: RMPP version for non-RMPP class 0x%x\n",
__func__, mad_reg_req->mgmt_class);
goto error1;
}
}
@@ -298,9 +300,9 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
IB_MGMT_CLASS_SUBN_LID_ROUTED) &&
(mad_reg_req->mgmt_class !=
IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
dev_notice(&device->dev,
"ib_register_mad_agent: Invalid SM QP type: class 0x%x\n",
mad_reg_req->mgmt_class);
dev_dbg_ratelimited(&device->dev,
"%s: Invalid SM QP type: class 0x%x\n",
__func__, mad_reg_req->mgmt_class);
goto error1;
}
} else {
@@ -308,9 +310,9 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
IB_MGMT_CLASS_SUBN_LID_ROUTED) ||
(mad_reg_req->mgmt_class ==
IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
dev_notice(&device->dev,
"ib_register_mad_agent: Invalid GS QP type: class 0x%x\n",
mad_reg_req->mgmt_class);
dev_dbg_ratelimited(&device->dev,
"%s: Invalid GS QP type: class 0x%x\n",
__func__, mad_reg_req->mgmt_class);
goto error1;
}
}
@@ -325,18 +327,18 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
/* Validate device and port */
port_priv = ib_get_mad_port(device, port_num);
if (!port_priv) {
dev_notice(&device->dev,
"ib_register_mad_agent: Invalid port %d\n",
port_num);
dev_dbg_ratelimited(&device->dev, "%s: Invalid port %d\n",
__func__, port_num);
ret = ERR_PTR(-ENODEV);
goto error1;
}
/* Verify the QP requested is supported. For example, Ethernet devices
* will not have QP0 */
/* Verify the QP requested is supported. For example, Ethernet devices
* will not have QP0.
*/
if (!port_priv->qp_info[qpn].qp) {
dev_notice(&device->dev,
"ib_register_mad_agent: QP %d not supported\n", qpn);
dev_dbg_ratelimited(&device->dev, "%s: QP %d not supported\n",
__func__, qpn);
ret = ERR_PTR(-EPROTONOSUPPORT);
goto error1;
}
sys/ofed/drivers/infiniband/core/ib_sa_query.c
-36
View File
@@ -790,42 +790,6 @@ static u8 get_src_path_mask(struct ib_device *device, u8 port_num)
return src_path_mask;
}
static int roce_resolve_route_from_path(struct sa_path_rec *rec,
const struct ib_gid_attr *attr)
{
struct rdma_dev_addr dev_addr = {};
union rdma_sockaddr sgid_addr, dgid_addr;
int ret;
if (rec->roce.route_resolved)
return 0;
if (!attr || !attr->ndev)
return -EINVAL;
dev_addr.bound_dev_if = if_getindex(attr->ndev);
/* TODO: Use net from the ib_gid_attr once it is added to it,
* until than, limit itself to init_net.
*/
dev_addr.net = &init_net;
rdma_gid2ip(&sgid_addr._sockaddr, &rec->sgid);
rdma_gid2ip(&dgid_addr._sockaddr, &rec->dgid);
/* validate the route */
ret = rdma_resolve_ip_route(&sgid_addr._sockaddr,
&dgid_addr._sockaddr, &dev_addr);
if (ret)
return ret;
if ((dev_addr.network == RDMA_NETWORK_IPV4 ||
dev_addr.network == RDMA_NETWORK_IPV6) &&
rec->rec_type != SA_PATH_REC_TYPE_ROCE_V2)
return -EINVAL;
rec->roce.route_resolved = true;
return 0;
}
static int init_ah_attr_grh_fields(struct ib_device *device, u8 port_num,
struct sa_path_rec *rec,
struct rdma_ah_attr *ah_attr,
sys/ofed/drivers/infiniband/core/ib_sysfs.c
+28 -35
View File
@@ -1207,7 +1207,7 @@ err_put:
return ret;
}
static ssize_t show_node_type(struct device *device,
static ssize_t node_type_show(struct device *device,
struct device_attribute *attr, char *buf)
{
struct ib_device *dev = container_of(device, struct ib_device, dev);
@@ -1222,8 +1222,9 @@ static ssize_t show_node_type(struct device *device,
default: return sprintf(buf, "%d: <unknown>\n", dev->node_type);
}
}
static DEVICE_ATTR_RO(node_type);
static ssize_t show_sys_image_guid(struct device *device,
static ssize_t sys_image_guid_show(struct device *device,
struct device_attribute *dev_attr, char *buf)
{
struct ib_device *dev = container_of(device, struct ib_device, dev);
@@ -1234,8 +1235,9 @@ static ssize_t show_sys_image_guid(struct device *device,
be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[2]),
be16_to_cpu(((__be16 *) &dev->attrs.sys_image_guid)[3]));
}
static DEVICE_ATTR_RO(sys_image_guid);
static ssize_t show_node_guid(struct device *device,
static ssize_t node_guid_show(struct device *device,
struct device_attribute *attr, char *buf)
{
struct ib_device *dev = container_of(device, struct ib_device, dev);
@@ -1246,8 +1248,9 @@ static ssize_t show_node_guid(struct device *device,
be16_to_cpu(((__be16 *) &dev->node_guid)[2]),
be16_to_cpu(((__be16 *) &dev->node_guid)[3]));
}
static DEVICE_ATTR_RO(node_guid);
static ssize_t show_node_desc(struct device *device,
static ssize_t node_desc_show(struct device *device,
struct device_attribute *attr, char *buf)
{
struct ib_device *dev = container_of(device, struct ib_device, dev);
@@ -1255,9 +1258,9 @@ static ssize_t show_node_desc(struct device *device,
return sprintf(buf, "%.64s\n", dev->node_desc);
}
static ssize_t set_node_desc(struct device *device,
struct device_attribute *attr,
const char *buf, size_t count)
static ssize_t node_desc_store(struct device *device,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct ib_device *dev = container_of(device, struct ib_device, dev);
struct ib_device_modify desc = {};
@@ -1273,8 +1276,9 @@ static ssize_t set_node_desc(struct device *device,
return count;
}
static DEVICE_ATTR_RW(node_desc);
static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
static ssize_t fw_ver_show(struct device *device, struct device_attribute *attr,
char *buf)
{
struct ib_device *dev = container_of(device, struct ib_device, dev);
@@ -1283,19 +1287,19 @@ static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
strlcat(buf, "\n", PAGE_SIZE);
return strlen(buf);
}
static DEVICE_ATTR_RO(fw_ver);
static DEVICE_ATTR(node_type, S_IRUGO, show_node_type, NULL);
static DEVICE_ATTR(sys_image_guid, S_IRUGO, show_sys_image_guid, NULL);
static DEVICE_ATTR(node_guid, S_IRUGO, show_node_guid, NULL);
static DEVICE_ATTR(node_desc, S_IRUGO | S_IWUSR, show_node_desc, set_node_desc);
static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
static struct attribute *ib_dev_attrs[] = {
&dev_attr_node_type.attr,
&dev_attr_node_guid.attr,
&dev_attr_sys_image_guid.attr,
&dev_attr_fw_ver.attr,
&dev_attr_node_desc.attr,
NULL,
};
static struct device_attribute *ib_class_attributes[] = {
&dev_attr_node_type,
&dev_attr_sys_image_guid,
&dev_attr_node_guid,
&dev_attr_node_desc,
&dev_attr_fw_ver,
static const struct attribute_group dev_attr_group = {
.attrs = ib_dev_attrs,
};
static void free_port_list_attributes(struct ib_device *device)
@@ -1338,16 +1342,13 @@ int ib_device_register_sysfs(struct ib_device *device,
if (ret)
return ret;
device->groups[0] = &dev_attr_group;
class_dev->groups = device->groups;
ret = device_add(class_dev);
if (ret)
goto err;
for (i = 0; i < ARRAY_SIZE(ib_class_attributes); ++i) {
ret = device_create_file(class_dev, ib_class_attributes[i]);
if (ret)
goto err_unregister;
}
device->ports_parent = kobject_create_and_add("ports",
&class_dev->kobj);
if (!device->ports_parent) {
@@ -1374,20 +1375,15 @@ int ib_device_register_sysfs(struct ib_device *device,
err_put:
free_port_list_attributes(device);
err_unregister:
device_del(class_dev);
err:
return ret;
}
void ib_device_unregister_sysfs(struct ib_device *device)
{
int i;
/* Hold kobject until ib_dealloc_device() */
kobject_get(&device->dev.kobj);
/* Hold device until ib_dealloc_device() */
get_device(&device->dev);
free_port_list_attributes(device);
@@ -1396,9 +1392,6 @@ void ib_device_unregister_sysfs(struct ib_device *device)
free_hsag(&device->dev.kobj, device->hw_stats_ag);
}
for (i = 0; i < ARRAY_SIZE(ib_class_attributes); ++i)
device_remove_file(&device->dev, ib_class_attributes[i]);
device_unregister(&device->dev);
}
sys/ofed/drivers/infiniband/core/ib_umem.c
+49 -51
View File
@@ -87,6 +87,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
struct page **page_list;
struct vm_area_struct **vma_list;
unsigned long cur_base;
struct mm_struct *mm;
unsigned long npages;
int ret;
int i;
@@ -108,26 +109,32 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
if (priv_check(curthread, PRIV_VM_MLOCK) != 0)
return ERR_PTR(-EPERM);
umem = kzalloc(sizeof *umem, GFP_KERNEL);
if (!umem)
return ERR_PTR(-ENOMEM);
if (access & IB_ACCESS_ON_DEMAND) {
umem = kzalloc(sizeof(struct ib_umem_odp), GFP_KERNEL);
if (!umem)
return ERR_PTR(-ENOMEM);
umem->is_odp = 1;
} else {
umem = kzalloc(sizeof(*umem), GFP_KERNEL);
if (!umem)
return ERR_PTR(-ENOMEM);
}
umem->context = context;
umem->length = size;
umem->address = addr;
umem->page_shift = PAGE_SHIFT;
umem->pid = get_pid(task_pid(current));
umem->writable = ib_access_writable(access);
umem->owning_mm = mm = current->mm;
mmgrab(mm);
if (access & IB_ACCESS_ON_DEMAND) {
ret = ib_umem_odp_get(context, umem, access);
ret = ib_umem_odp_get(to_ib_umem_odp(umem), access);
if (ret)
goto umem_kfree;
return umem;
}
umem->odp_data = NULL;
page_list = (struct page **) __get_free_page(GFP_KERNEL);
if (!page_list) {
ret = -ENOMEM;
@@ -138,9 +145,9 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
npages = ib_umem_num_pages(umem);
down_write(&current->mm->mmap_sem);
current->mm->pinned_vm += npages;
up_write(&current->mm->mmap_sem);
down_write(&mm->mmap_sem);
mm->pinned_vm += npages;
up_write(&mm->mmap_sem);
cur_base = addr & PAGE_MASK;
@@ -158,14 +165,14 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
sg_list_start = umem->sg_head.sgl;
down_read(&current->mm->mmap_sem);
down_read(&mm->mmap_sem);
while (npages) {
ret = get_user_pages(cur_base,
min_t(unsigned long, npages,
PAGE_SIZE / sizeof (struct page *)),
gup_flags, page_list, vma_list);
if (ret < 0) {
up_read(&current->mm->mmap_sem);
up_read(&mm->mmap_sem);
goto umem_release;
}
@@ -180,7 +187,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
/* preparing for next loop */
sg_list_start = sg;
}
up_read(&current->mm->mmap_sem);
up_read(&mm->mmap_sem);
umem->nmap = ib_dma_map_sg_attrs(context->device,
umem->sg_head.sgl,
@@ -208,22 +215,31 @@ out:
free_page((unsigned long) page_list);
umem_kfree:
if (ret) {
put_pid(umem->pid);
mmdrop(umem->owning_mm);
kfree(umem);
}
return ret ? ERR_PTR(ret) : umem;
}
EXPORT_SYMBOL(ib_umem_get);
static void ib_umem_account(struct work_struct *work)
static void __ib_umem_release_tail(struct ib_umem *umem)
{
mmdrop(umem->owning_mm);
if (umem->is_odp)
kfree(to_ib_umem_odp(umem));
else
kfree(umem);
}
static void ib_umem_release_defer(struct work_struct *work)
{
struct ib_umem *umem = container_of(work, struct ib_umem, work);
down_write(&umem->mm->mmap_sem);
umem->mm->pinned_vm -= umem->diff;
up_write(&umem->mm->mmap_sem);
mmput(umem->mm);
kfree(umem);
down_write(&umem->owning_mm->mmap_sem);
umem->owning_mm->pinned_vm -= ib_umem_num_pages(umem);
up_write(&umem->owning_mm->mmap_sem);
__ib_umem_release_tail(umem);
}
/**
@@ -232,56 +248,38 @@ static void ib_umem_account(struct work_struct *work)
*/
void ib_umem_release(struct ib_umem *umem)
{
struct mm_struct *mm;
struct task_struct *task;
unsigned long diff;
if (!umem)
return;
if (umem->odp_data) {
ib_umem_odp_release(umem);
if (umem->is_odp) {
ib_umem_odp_release(to_ib_umem_odp(umem));
__ib_umem_release_tail(umem);
return;
}
__ib_umem_release(umem->context->device, umem, 1);
task = get_pid_task(umem->pid, PIDTYPE_PID);
put_pid(umem->pid);
if (!task)
goto out;
mm = get_task_mm(task);
put_task_struct(task);
if (!mm)
goto out;
diff = ib_umem_num_pages(umem);
/*
* We may be called with the mm's mmap_sem already held. This
* can happen when a userspace munmap() is the call that drops
* the last reference to our file and calls our release
* method. If there are memory regions to destroy, we'll end
* up here and not be able to take the mmap_sem. In that case
* we defer the vm_locked accounting to the system workqueue.
* we defer the vm_locked accounting a workqueue.
*/
if (umem->context->closing) {
if (!down_write_trylock(&mm->mmap_sem)) {
INIT_WORK(&umem->work, ib_umem_account);
umem->mm = mm;
umem->diff = diff;
if (!down_write_trylock(&umem->owning_mm->mmap_sem)) {
INIT_WORK(&umem->work, ib_umem_release_defer);
queue_work(ib_wq, &umem->work);
return;
}
} else
down_write(&mm->mmap_sem);
} else {
down_write(&umem->owning_mm->mmap_sem);
}
umem->owning_mm->pinned_vm -= ib_umem_num_pages(umem);
up_write(&umem->owning_mm->mmap_sem);
mm->pinned_vm -= diff;
up_write(&mm->mmap_sem);
mmput(mm);
out:
kfree(umem);
__ib_umem_release_tail(umem);
}
EXPORT_SYMBOL(ib_umem_release);
@@ -291,7 +289,7 @@ int ib_umem_page_count(struct ib_umem *umem)
int n;
struct scatterlist *sg;
if (umem->odp_data)
if (umem->is_odp)
return ib_umem_num_pages(umem);
n = 0;
sys/ofed/drivers/infiniband/core/ib_umem_odp.c
+285 -332
View File
@@ -57,7 +57,7 @@ static u64 node_start(struct umem_odp_node *n)
struct ib_umem_odp *umem_odp =
container_of(n, struct ib_umem_odp, interval_tree);
return ib_umem_start(umem_odp->umem);
return ib_umem_start(&umem_odp->umem);
}
/* Note that the representation of the intervals in the interval tree
@@ -70,140 +70,86 @@ static u64 node_last(struct umem_odp_node *n)
struct ib_umem_odp *umem_odp =
container_of(n, struct ib_umem_odp, interval_tree);
return ib_umem_end(umem_odp->umem) - 1;
return ib_umem_end(&umem_odp->umem) - 1;
}
INTERVAL_TREE_DEFINE(struct umem_odp_node, rb, u64, __subtree_last,
node_start, node_last, static, rbt_ib_umem)
static void ib_umem_notifier_start_account(struct ib_umem *item)
static void ib_umem_notifier_start_account(struct ib_umem_odp *umem_odp)
{
mutex_lock(&item->odp_data->umem_mutex);
/* Only update private counters for this umem if it has them.
* Otherwise skip it. All page faults will be delayed for this umem. */
if (item->odp_data->mn_counters_active) {
int notifiers_count = item->odp_data->notifiers_count++;
if (notifiers_count == 0)
/* Initialize the completion object for waiting on
* notifiers. Since notifier_count is zero, no one
* should be waiting right now. */
reinit_completion(&item->odp_data->notifier_completion);
}
mutex_unlock(&item->odp_data->umem_mutex);
}
static void ib_umem_notifier_end_account(struct ib_umem *item)
{
mutex_lock(&item->odp_data->umem_mutex);
/* Only update private counters for this umem if it has them.
* Otherwise skip it. All page faults will be delayed for this umem. */
if (item->odp_data->mn_counters_active) {
mutex_lock(&umem_odp->umem_mutex);
if (umem_odp->notifiers_count++ == 0)
/*
* This sequence increase will notify the QP page fault that
* the page that is going to be mapped in the spte could have
* been freed.
* Initialize the completion object for waiting on
* notifiers. Since notifier_count is zero, no one should be
* waiting right now.
*/
++item->odp_data->notifiers_seq;
if (--item->odp_data->notifiers_count == 0)
complete_all(&item->odp_data->notifier_completion);
}
mutex_unlock(&item->odp_data->umem_mutex);
reinit_completion(&umem_odp->notifier_completion);
mutex_unlock(&umem_odp->umem_mutex);
}
/* Account for a new mmu notifier in an ib_ucontext. */
static void ib_ucontext_notifier_start_account(struct ib_ucontext *context)
static void ib_umem_notifier_end_account(struct ib_umem_odp *umem_odp)
{
atomic_inc(&context->notifier_count);
mutex_lock(&umem_odp->umem_mutex);
/*
* This sequence increase will notify the QP page fault that the page
* that is going to be mapped in the spte could have been freed.
*/
++umem_odp->notifiers_seq;
if (--umem_odp->notifiers_count == 0)
complete_all(&umem_odp->notifier_completion);
mutex_unlock(&umem_odp->umem_mutex);
}
/* Account for a terminating mmu notifier in an ib_ucontext.
*
* Must be called with the ib_ucontext->umem_rwsem semaphore unlocked, since
* the function takes the semaphore itself. */
static void ib_ucontext_notifier_end_account(struct ib_ucontext *context)
static int ib_umem_notifier_release_trampoline(struct ib_umem_odp *umem_odp,
u64 start, u64 end, void *cookie)
{
int zero_notifiers = atomic_dec_and_test(&context->notifier_count);
struct ib_umem *umem = &umem_odp->umem;
if (zero_notifiers &&
!list_empty(&context->no_private_counters)) {
/* No currently running mmu notifiers. Now is the chance to
* add private accounting to all previously added umems. */
struct ib_umem_odp *odp_data, *next;
/* Prevent concurrent mmu notifiers from working on the
* no_private_counters list. */
down_write(&context->umem_rwsem);
/* Read the notifier_count again, with the umem_rwsem
* semaphore taken for write. */
if (!atomic_read(&context->notifier_count)) {
list_for_each_entry_safe(odp_data, next,
&context->no_private_counters,
no_private_counters) {
mutex_lock(&odp_data->umem_mutex);
odp_data->mn_counters_active = true;
list_del(&odp_data->no_private_counters);
complete_all(&odp_data->notifier_completion);
mutex_unlock(&odp_data->umem_mutex);
}
}
up_write(&context->umem_rwsem);
}
}
static int ib_umem_notifier_release_trampoline(struct ib_umem *item, u64 start,
u64 end, void *cookie) {
/*
* Increase the number of notifiers running, to
* prevent any further fault handling on this MR.
*/
ib_umem_notifier_start_account(item);
item->odp_data->dying = 1;
ib_umem_notifier_start_account(umem_odp);
umem_odp->dying = 1;
/* Make sure that the fact the umem is dying is out before we release
* all pending page faults. */
smp_wmb();
complete_all(&item->odp_data->notifier_completion);
item->context->invalidate_range(item, ib_umem_start(item),
ib_umem_end(item));
complete_all(&umem_odp->notifier_completion);
umem->context->invalidate_range(umem_odp, ib_umem_start(umem),
ib_umem_end(umem));
return 0;
}
static void ib_umem_notifier_release(struct mmu_notifier *mn,
struct mm_struct *mm)
{
struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
struct ib_ucontext_per_mm *per_mm =
container_of(mn, struct ib_ucontext_per_mm, mn);
if (!context->invalidate_range)
return;
ib_ucontext_notifier_start_account(context);
down_read(&context->umem_rwsem);
rbt_ib_umem_for_each_in_range(&context->umem_tree, 0,
ULLONG_MAX,
ib_umem_notifier_release_trampoline,
true,
NULL);
up_read(&context->umem_rwsem);
down_read(&per_mm->umem_rwsem);
if (per_mm->active)
rbt_ib_umem_for_each_in_range(
&per_mm->umem_tree, 0, ULLONG_MAX,
ib_umem_notifier_release_trampoline, true, NULL);
up_read(&per_mm->umem_rwsem);
}
static int invalidate_page_trampoline(struct ib_umem *item, u64 start,
static int invalidate_page_trampoline(struct ib_umem_odp *item, u64 start,
u64 end, void *cookie)
{
ib_umem_notifier_start_account(item);
item->context->invalidate_range(item, start, start + PAGE_SIZE);
item->umem.context->invalidate_range(item, start, start + PAGE_SIZE);
ib_umem_notifier_end_account(item);
return 0;
}
static int invalidate_range_start_trampoline(struct ib_umem *item, u64 start,
u64 end, void *cookie)
static int invalidate_range_start_trampoline(struct ib_umem_odp *item,
u64 start, u64 end, void *cookie)
{
ib_umem_notifier_start_account(item);
item->context->invalidate_range(item, start, end);
item->umem.context->invalidate_range(item, start, end);
return 0;
}
@@ -213,28 +159,30 @@ static int ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,
unsigned long end,
bool blockable)
{
struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
int ret;
if (!context->invalidate_range)
return 0;
struct ib_ucontext_per_mm *per_mm =
container_of(mn, struct ib_ucontext_per_mm, mn);
if (blockable)
down_read(&context->umem_rwsem);
else if (!down_read_trylock(&context->umem_rwsem))
down_read(&per_mm->umem_rwsem);
else if (!down_read_trylock(&per_mm->umem_rwsem))
return -EAGAIN;
ib_ucontext_notifier_start_account(context);
ret = rbt_ib_umem_for_each_in_range(&context->umem_tree, start,
end,
invalidate_range_start_trampoline,
blockable, NULL);
up_read(&context->umem_rwsem);
if (!per_mm->active) {
up_read(&per_mm->umem_rwsem);
/*
* At this point active is permanently set and visible to this
* CPU without a lock, that fact is relied on to skip the unlock
* in range_end.
*/
return 0;
}
return ret;
return rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, start, end,
invalidate_range_start_trampoline,
blockable, NULL);
}
static int invalidate_range_end_trampoline(struct ib_umem *item, u64 start,
static int invalidate_range_end_trampoline(struct ib_umem_odp *item, u64 start,
u64 end, void *cookie)
{
ib_umem_notifier_end_account(item);
@@ -246,22 +194,16 @@ static void ib_umem_notifier_invalidate_range_end(struct mmu_notifier *mn,
unsigned long start,
unsigned long end)
{
struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
struct ib_ucontext_per_mm *per_mm =
container_of(mn, struct ib_ucontext_per_mm, mn);
if (!context->invalidate_range)
if (unlikely(!per_mm->active))
return;
/*
* TODO: we currently bail out if there is any sleepable work to be done
* in ib_umem_notifier_invalidate_range_start so we shouldn't really block
* here. But this is ugly and fragile.
*/
down_read(&context->umem_rwsem);
rbt_ib_umem_for_each_in_range(&context->umem_tree, start,
rbt_ib_umem_for_each_in_range(&per_mm->umem_tree, start,
end,
invalidate_range_end_trampoline, true, NULL);
up_read(&context->umem_rwsem);
ib_ucontext_notifier_end_account(context);
up_read(&per_mm->umem_rwsem);
}
static const struct mmu_notifier_ops ib_umem_notifiers = {
@@ -270,31 +212,158 @@ static const struct mmu_notifier_ops ib_umem_notifiers = {
.invalidate_range_end = ib_umem_notifier_invalidate_range_end,
};
struct ib_umem *ib_alloc_odp_umem(struct ib_ucontext *context,
unsigned long addr,
size_t size)
static void add_umem_to_per_mm(struct ib_umem_odp *umem_odp)
{
struct ib_umem *umem;
struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
struct ib_umem *umem = &umem_odp->umem;
down_write(&per_mm->umem_rwsem);
if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
rbt_ib_umem_insert(&umem_odp->interval_tree,
&per_mm->umem_tree);
up_write(&per_mm->umem_rwsem);
}
static void remove_umem_from_per_mm(struct ib_umem_odp *umem_odp)
{
struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
struct ib_umem *umem = &umem_odp->umem;
down_write(&per_mm->umem_rwsem);
if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
rbt_ib_umem_remove(&umem_odp->interval_tree,
&per_mm->umem_tree);
complete_all(&umem_odp->notifier_completion);
up_write(&per_mm->umem_rwsem);
}
static struct ib_ucontext_per_mm *alloc_per_mm(struct ib_ucontext *ctx,
struct mm_struct *mm)
{
struct ib_ucontext_per_mm *per_mm;
int ret;
per_mm = kzalloc(sizeof(*per_mm), GFP_KERNEL);
if (!per_mm)
return ERR_PTR(-ENOMEM);
per_mm->context = ctx;
per_mm->mm = mm;
per_mm->umem_tree = RB_ROOT_CACHED;
init_rwsem(&per_mm->umem_rwsem);
per_mm->active = ctx->invalidate_range;
rcu_read_lock();
per_mm->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
rcu_read_unlock();
WARN_ON(mm != current->mm);
per_mm->mn.ops = &ib_umem_notifiers;
ret = mmu_notifier_register(&per_mm->mn, per_mm->mm);
if (ret) {
dev_err(&ctx->device->dev,
"Failed to register mmu_notifier %d\n", ret);
goto out_pid;
}
list_add(&per_mm->ucontext_list, &ctx->per_mm_list);
return per_mm;
out_pid:
put_pid(per_mm->tgid);
kfree(per_mm);
return ERR_PTR(ret);
}
static int get_per_mm(struct ib_umem_odp *umem_odp)
{
struct ib_ucontext *ctx = umem_odp->umem.context;
struct ib_ucontext_per_mm *per_mm;
/*
* Generally speaking we expect only one or two per_mm in this list,
* so no reason to optimize this search today.
*/
mutex_lock(&ctx->per_mm_list_lock);
list_for_each_entry(per_mm, &ctx->per_mm_list, ucontext_list) {
if (per_mm->mm == umem_odp->umem.owning_mm)
goto found;
}
per_mm = alloc_per_mm(ctx, umem_odp->umem.owning_mm);
if (IS_ERR(per_mm)) {
mutex_unlock(&ctx->per_mm_list_lock);
return PTR_ERR(per_mm);
}
found:
umem_odp->per_mm = per_mm;
per_mm->odp_mrs_count++;
mutex_unlock(&ctx->per_mm_list_lock);
return 0;
}
static void free_per_mm(struct rcu_head *rcu)
{
kfree(container_of(rcu, struct ib_ucontext_per_mm, rcu));
}
void put_per_mm(struct ib_umem_odp *umem_odp)
{
struct ib_ucontext_per_mm *per_mm = umem_odp->per_mm;
struct ib_ucontext *ctx = umem_odp->umem.context;
bool need_free;
mutex_lock(&ctx->per_mm_list_lock);
umem_odp->per_mm = NULL;
per_mm->odp_mrs_count--;
need_free = per_mm->odp_mrs_count == 0;
if (need_free)
list_del(&per_mm->ucontext_list);
mutex_unlock(&ctx->per_mm_list_lock);
if (!need_free)
return;
/*
* NOTE! mmu_notifier_unregister() can happen between a start/end
* callback, resulting in an start/end, and thus an unbalanced
* lock. This doesn't really matter to us since we are about to kfree
* the memory that holds the lock, however LOCKDEP doesn't like this.
*/
down_write(&per_mm->umem_rwsem);
per_mm->active = false;
up_write(&per_mm->umem_rwsem);
WARN_ON(!RB_EMPTY_ROOT(&per_mm->umem_tree.rb_root));
mmu_notifier_unregister_no_release(&per_mm->mn, per_mm->mm);
put_pid(per_mm->tgid);
mmu_notifier_call_srcu(&per_mm->rcu, free_per_mm);
}
struct ib_umem_odp *ib_alloc_odp_umem(struct ib_ucontext_per_mm *per_mm,
unsigned long addr, size_t size)
{
struct ib_ucontext *ctx = per_mm->context;
struct ib_umem_odp *odp_data;
struct ib_umem *umem;
int pages = size >> PAGE_SHIFT;
int ret;
umem = kzalloc(sizeof(*umem), GFP_KERNEL);
if (!umem)
odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL);
if (!odp_data)
return ERR_PTR(-ENOMEM);
umem->context = context;
umem = &odp_data->umem;
umem->context = ctx;
umem->length = size;
umem->address = addr;
umem->page_shift = PAGE_SHIFT;
umem->writable = 1;
odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL);
if (!odp_data) {
ret = -ENOMEM;
goto out_umem;
}
odp_data->umem = umem;
umem->is_odp = 1;
odp_data->per_mm = per_mm;
mutex_init(&odp_data->umem_mutex);
init_completion(&odp_data->notifier_completion);
@@ -313,39 +382,34 @@ struct ib_umem *ib_alloc_odp_umem(struct ib_ucontext *context,
goto out_page_list;
}
down_write(&context->umem_rwsem);
context->odp_mrs_count++;
rbt_ib_umem_insert(&odp_data->interval_tree, &context->umem_tree);
if (likely(!atomic_read(&context->notifier_count)))
odp_data->mn_counters_active = true;
else
list_add(&odp_data->no_private_counters,
&context->no_private_counters);
up_write(&context->umem_rwsem);
/*
* Caller must ensure that the umem_odp that the per_mm came from
* cannot be freed during the call to ib_alloc_odp_umem.
*/
mutex_lock(&ctx->per_mm_list_lock);
per_mm->odp_mrs_count++;
mutex_unlock(&ctx->per_mm_list_lock);
add_umem_to_per_mm(odp_data);
umem->odp_data = odp_data;
return umem;
return odp_data;
out_page_list:
vfree(odp_data->page_list);
out_odp_data:
kfree(odp_data);
out_umem:
kfree(umem);
return ERR_PTR(ret);
}
EXPORT_SYMBOL(ib_alloc_odp_umem);
int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
int access)
int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access)
{
struct ib_umem *umem = &umem_odp->umem;
/*
* NOTE: This must called in a process context where umem->owning_mm
* == current->mm
*/
struct mm_struct *mm = umem->owning_mm;
int ret_val;
pid_t our_pid;
struct mm_struct *mm = get_task_mm(current);
if (!mm)
return -EINVAL;
if (access & IB_ACCESS_HUGETLB) {
struct vm_area_struct *vma;
@@ -361,109 +425,43 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
umem->hugetlb = 0;
}
/* Prevent creating ODP MRs in child processes */
rcu_read_lock();
our_pid = get_pid(task_pid_group_leader(current));
rcu_read_unlock();
put_pid(our_pid);
if (context->tgid != our_pid) {
ret_val = -EINVAL;
goto out_mm;
}
mutex_init(&umem_odp->umem_mutex);
umem->odp_data = kzalloc(sizeof(*umem->odp_data), GFP_KERNEL);
if (!umem->odp_data) {
ret_val = -ENOMEM;
goto out_mm;
}
umem->odp_data->umem = umem;
mutex_init(&umem->odp_data->umem_mutex);
init_completion(&umem->odp_data->notifier_completion);
init_completion(&umem_odp->notifier_completion);
if (ib_umem_num_pages(umem)) {
umem->odp_data->page_list =
vzalloc(array_size(sizeof(*umem->odp_data->page_list),
umem_odp->page_list =
vzalloc(array_size(sizeof(*umem_odp->page_list),
ib_umem_num_pages(umem)));
if (!umem->odp_data->page_list) {
ret_val = -ENOMEM;
goto out_odp_data;
}
if (!umem_odp->page_list)
return -ENOMEM;
umem->odp_data->dma_list =
vzalloc(array_size(sizeof(*umem->odp_data->dma_list),
umem_odp->dma_list =
vzalloc(array_size(sizeof(*umem_odp->dma_list),
ib_umem_num_pages(umem)));
if (!umem->odp_data->dma_list) {
if (!umem_odp->dma_list) {
ret_val = -ENOMEM;
goto out_page_list;
}
}
/*
* When using MMU notifiers, we will get a
* notification before the "current" task (and MM) is
* destroyed. We use the umem_rwsem semaphore to synchronize.
*/
down_write(&context->umem_rwsem);
context->odp_mrs_count++;
if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
rbt_ib_umem_insert(&umem->odp_data->interval_tree,
&context->umem_tree);
if (likely(!atomic_read(&context->notifier_count)) ||
context->odp_mrs_count == 1)
umem->odp_data->mn_counters_active = true;
else
list_add(&umem->odp_data->no_private_counters,
&context->no_private_counters);
downgrade_write(&context->umem_rwsem);
ret_val = get_per_mm(umem_odp);
if (ret_val)
goto out_dma_list;
add_umem_to_per_mm(umem_odp);
if (context->odp_mrs_count == 1) {
/*
* Note that at this point, no MMU notifier is running
* for this context!
*/
atomic_set(&context->notifier_count, 0);
INIT_HLIST_NODE(&context->mn.hlist);
context->mn.ops = &ib_umem_notifiers;
/*
* Lock-dep detects a false positive for mmap_sem vs.
* umem_rwsem, due to not grasping downgrade_write correctly.
*/
ret_val = mmu_notifier_register(&context->mn, mm);
if (ret_val) {
pr_err("Failed to register mmu_notifier %d\n", ret_val);
ret_val = -EBUSY;
goto out_mutex;
}
}
up_read(&context->umem_rwsem);
/*
* Note that doing an mmput can cause a notifier for the relevant mm.
* If the notifier is called while we hold the umem_rwsem, this will
* cause a deadlock. Therefore, we release the reference only after we
* released the semaphore.
*/
mmput(mm);
return 0;
out_mutex:
up_read(&context->umem_rwsem);
vfree(umem->odp_data->dma_list);
out_dma_list:
vfree(umem_odp->dma_list);
out_page_list:
vfree(umem->odp_data->page_list);
out_odp_data:
kfree(umem->odp_data);
out_mm:
mmput(mm);
vfree(umem_odp->page_list);
return ret_val;
}
void ib_umem_odp_release(struct ib_umem *umem)
void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
{
struct ib_ucontext *context = umem->context;
struct ib_umem *umem = &umem_odp->umem;
/*
* Ensure that no more pages are mapped in the umem.
@@ -471,61 +469,13 @@ void ib_umem_odp_release(struct ib_umem *umem)
* It is the driver's responsibility to ensure, before calling us,
* that the hardware will not attempt to access the MR any more.
*/
ib_umem_odp_unmap_dma_pages(umem, ib_umem_start(umem),
ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem),
ib_umem_end(umem));
down_write(&context->umem_rwsem);
if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
rbt_ib_umem_remove(&umem->odp_data->interval_tree,
&context->umem_tree);
context->odp_mrs_count--;
if (!umem->odp_data->mn_counters_active) {
list_del(&umem->odp_data->no_private_counters);
complete_all(&umem->odp_data->notifier_completion);
}
/*
* Downgrade the lock to a read lock. This ensures that the notifiers
* (who lock the mutex for reading) will be able to finish, and we
* will be able to enventually obtain the mmu notifiers SRCU. Note
* that since we are doing it atomically, no other user could register
* and unregister while we do the check.
*/
downgrade_write(&context->umem_rwsem);
if (!context->odp_mrs_count) {
struct task_struct *owning_process = NULL;
struct mm_struct *owning_mm = NULL;
owning_process = get_pid_task(context->tgid,
PIDTYPE_PID);
if (owning_process == NULL)
/*
* The process is already dead, notifier were removed
* already.
*/
goto out;
owning_mm = get_task_mm(owning_process);
if (owning_mm == NULL)
/*
* The process' mm is already dead, notifier were
* removed already.
*/
goto out_put_task;
mmu_notifier_unregister(&context->mn, owning_mm);
mmput(owning_mm);
out_put_task:
put_task_struct(owning_process);
}
out:
up_read(&context->umem_rwsem);
vfree(umem->odp_data->dma_list);
vfree(umem->odp_data->page_list);
kfree(umem->odp_data);
kfree(umem);
remove_umem_from_per_mm(umem_odp);
put_per_mm(umem_odp);
vfree(umem_odp->dma_list);
vfree(umem_odp->page_list);
}
/*
@@ -537,7 +487,7 @@ out:
* @access_mask: access permissions needed for this page.
* @current_seq: sequence number for synchronization with invalidations.
* the sequence number is taken from
* umem->odp_data->notifiers_seq.
* umem_odp->notifiers_seq.
*
* The function returns -EFAULT if the DMA mapping operation fails. It returns
* -EAGAIN if a concurrent invalidation prevents us from updating the page.
@@ -547,12 +497,13 @@ out:
* umem.
*/
static int ib_umem_odp_map_dma_single_page(
struct ib_umem *umem,
struct ib_umem_odp *umem_odp,
int page_index,
struct page *page,
u64 access_mask,
unsigned long current_seq)
{
struct ib_umem *umem = &umem_odp->umem;
struct ib_device *dev = umem->context->device;
dma_addr_t dma_addr;
int stored_page = 0;
@@ -564,11 +515,11 @@ static int ib_umem_odp_map_dma_single_page(
* handle case of a racing notifier. This check also allows us to bail
* early if we have a notifier running in parallel with us.
*/
if (ib_umem_mmu_notifier_retry(umem, current_seq)) {
if (ib_umem_mmu_notifier_retry(umem_odp, current_seq)) {
ret = -EAGAIN;
goto out;
}
if (!(umem->odp_data->dma_list[page_index])) {
if (!(umem_odp->dma_list[page_index])) {
dma_addr = ib_dma_map_page(dev,
page,
0, BIT(umem->page_shift),
@@ -577,15 +528,15 @@ static int ib_umem_odp_map_dma_single_page(
ret = -EFAULT;
goto out;
}
umem->odp_data->dma_list[page_index] = dma_addr | access_mask;
umem->odp_data->page_list[page_index] = page;
umem_odp->dma_list[page_index] = dma_addr | access_mask;
umem_odp->page_list[page_index] = page;
umem->npages++;
stored_page = 1;
} else if (umem->odp_data->page_list[page_index] == page) {
umem->odp_data->dma_list[page_index] |= access_mask;
} else if (umem_odp->page_list[page_index] == page) {
umem_odp->dma_list[page_index] |= access_mask;
} else {
pr_err("error: got different pages in IB device and from get_user_pages. IB device page: %p, gup page: %p\n",
umem->odp_data->page_list[page_index], page);
umem_odp->page_list[page_index], page);
/* Better remove the mapping now, to prevent any further
* damage. */
remove_existing_mapping = 1;
@@ -598,7 +549,7 @@ out:
if (remove_existing_mapping && umem->context->invalidate_range) {
invalidate_page_trampoline(
umem,
umem_odp,
ib_umem_start(umem) + (page_index >> umem->page_shift),
ib_umem_start(umem) + ((page_index + 1) >>
umem->page_shift),
@@ -614,7 +565,7 @@ out:
*
* Pins the range of pages passed in the argument, and maps them to
* DMA addresses. The DMA addresses of the mapped pages is updated in
* umem->odp_data->dma_list.
* umem_odp->dma_list.
*
* Returns the number of pages mapped in success, negative error code
* for failure.
@@ -622,7 +573,7 @@ out:
* the function from completing its task.
* An -ENOENT error code indicates that userspace process is being terminated
* and mm was already destroyed.
* @umem: the umem to map and pin
* @umem_odp: the umem to map and pin
* @user_virt: the address from which we need to map.
* @bcnt: the minimal number of bytes to pin and map. The mapping might be
* bigger due to alignment, and may also be smaller in case of an error
@@ -632,13 +583,15 @@ out:
* range.
* @current_seq: the MMU notifiers sequance value for synchronization with
* invalidations. the sequance number is read from
* umem->odp_data->notifiers_seq before calling this function
* umem_odp->notifiers_seq before calling this function
*/
int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
u64 access_mask, unsigned long current_seq)
int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
u64 bcnt, u64 access_mask,
unsigned long current_seq)
{
struct ib_umem *umem = &umem_odp->umem;
struct task_struct *owning_process = NULL;
struct mm_struct *owning_mm = NULL;
struct mm_struct *owning_mm = umem_odp->umem.owning_mm;
struct page **local_page_list = NULL;
u64 page_mask, off;
int j, k, ret = 0, start_idx, npages = 0, page_shift;
@@ -662,15 +615,14 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
user_virt = user_virt & page_mask;
bcnt += off; /* Charge for the first page offset as well. */
owning_process = get_pid_task(umem->context->tgid, PIDTYPE_PID);
if (owning_process == NULL) {
/*
* owning_process is allowed to be NULL, this means somehow the mm is
* existing beyond the lifetime of the originating process.. Presumably
* mmget_not_zero will fail in this case.
*/
owning_process = get_pid_task(umem_odp->per_mm->tgid, PIDTYPE_PID);
if (WARN_ON(!mmget_not_zero(umem_odp->umem.owning_mm))) {
ret = -EINVAL;
goto out_no_task;
}
owning_mm = get_task_mm(owning_process);
if (owning_mm == NULL) {
ret = -ENOENT;
goto out_put_task;
}
@@ -702,7 +654,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
break;
bcnt -= min_t(size_t, npages << PAGE_SHIFT, bcnt);
mutex_lock(&umem->odp_data->umem_mutex);
mutex_lock(&umem_odp->umem_mutex);
for (j = 0; j < npages; j++, user_virt += PAGE_SIZE) {
if (user_virt & ~page_mask) {
p += PAGE_SIZE;
@@ -715,7 +667,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
}
ret = ib_umem_odp_map_dma_single_page(
umem, k, local_page_list[j],
umem_odp, k, local_page_list[j],
access_mask, current_seq);
if (ret < 0)
break;
@@ -723,7 +675,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
p = page_to_phys(local_page_list[j]);
k++;
}
mutex_unlock(&umem->odp_data->umem_mutex);
mutex_unlock(&umem_odp->umem_mutex);
if (ret < 0) {
/* Release left over pages when handling errors. */
@@ -742,16 +694,17 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
mmput(owning_mm);
out_put_task:
put_task_struct(owning_process);
out_no_task:
if (owning_process)
put_task_struct(owning_process);
free_page((unsigned long)local_page_list);
return ret;
}
EXPORT_SYMBOL(ib_umem_odp_map_dma_pages);
void ib_umem_odp_unmap_dma_pages(struct ib_umem *umem, u64 virt,
void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,
u64 bound)
{
struct ib_umem *umem = &umem_odp->umem;
int idx;
u64 addr;
struct ib_device *dev = umem->context->device;
@@ -763,12 +716,12 @@ void ib_umem_odp_unmap_dma_pages(struct ib_umem *umem, u64 virt,
* faults from completion. We might be racing with other
* invalidations, so we must make sure we free each page only
* once. */
mutex_lock(&umem->odp_data->umem_mutex);
mutex_lock(&umem_odp->umem_mutex);
for (addr = virt; addr < bound; addr += BIT(umem->page_shift)) {
idx = (addr - ib_umem_start(umem)) >> umem->page_shift;
if (umem->odp_data->page_list[idx]) {
struct page *page = umem->odp_data->page_list[idx];
dma_addr_t dma = umem->odp_data->dma_list[idx];
if (umem_odp->page_list[idx]) {
struct page *page = umem_odp->page_list[idx];
dma_addr_t dma = umem_odp->dma_list[idx];
dma_addr_t dma_addr = dma & ODP_DMA_ADDR_MASK;
WARN_ON(!dma_addr);
@@ -791,12 +744,12 @@ void ib_umem_odp_unmap_dma_pages(struct ib_umem *umem, u64 virt,
/* on demand pinning support */
if (!umem->context->invalidate_range)
put_page(page);
umem->odp_data->page_list[idx] = NULL;
umem->odp_data->dma_list[idx] = 0;
umem_odp->page_list[idx] = NULL;
umem_odp->dma_list[idx] = 0;
umem->npages--;
}
}
mutex_unlock(&umem->odp_data->umem_mutex);
mutex_unlock(&umem_odp->umem_mutex);
}
EXPORT_SYMBOL(ib_umem_odp_unmap_dma_pages);
@@ -823,7 +776,7 @@ int rbt_ib_umem_for_each_in_range(struct rb_root_cached *root,
return -EAGAIN;
next = rbt_ib_umem_iter_next(node, start, last - 1);
umem = container_of(node, struct ib_umem_odp, interval_tree);
ret_val = cb(umem->umem, start, last, cookie) || ret_val;
ret_val = cb(umem, start, last, cookie) || ret_val;
}
return ret_val;
sys/ofed/drivers/infiniband/core/ib_uverbs_cmd.c
+8
View File
@@ -253,6 +253,14 @@ int ib_init_ucontext(struct uverbs_attr_bundle *attrs)
if (ret)
goto err_uncharge;
#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
mutex_init(&ucontext->per_mm_list_lock);
INIT_LIST_HEAD(&ucontext->per_mm_list);
if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_ON_DEMAND_PAGING))
ucontext->invalidate_range = NULL;
#endif
/*
* Make sure that ib_uverbs_get_ucontext() sees the pointer update
* only after all writes to setup the ucontext have completed
sys/ofed/drivers/infiniband/core/ib_verbs.c
+1 -1
View File
@@ -764,7 +764,7 @@ static int ib_resolve_unicast_gid_dmac(struct ib_device *device,
ret = rdma_addr_find_l2_eth_by_grh(&sgid_attr->gid, &grh->dgid,
ah_attr->roce.dmac,
sgid_attr->ndev, &hop_limit);
sgid_attr, &hop_limit);
grh->hop_limit = hop_limit;
return ret;
sys/ofed/include/rdma/ib_addr.h
+3 -4
View File
@@ -110,20 +110,19 @@ int rdma_translate_ip(const struct sockaddr *addr,
* @timeout_ms: Amount of time to wait for the address resolution to complete.
* @callback: Call invoked once address resolution has completed, timed out,
* or been canceled. A status of 0 indicates success.
* @resolve_by_gid_attr: Resolve the ip based on the GID attribute from
* rdma_dev_addr.
* @context: User-specified context associated with the call.
*/
int rdma_resolve_ip(struct sockaddr *src_addr, const struct sockaddr *dst_addr,
struct rdma_dev_addr *addr, int timeout_ms,
void (*callback)(int status, struct sockaddr *src_addr,
struct rdma_dev_addr *addr, void *context),
bool resolve_by_gid_attr,
void *context);
void rdma_addr_cancel(struct rdma_dev_addr *addr);
void rdma_copy_addr(struct rdma_dev_addr *dev_addr,
const if_t dev,
const unsigned char *dst_dev_addr);
int rdma_addr_size(const struct sockaddr *addr);
int rdma_addr_size_in6(struct sockaddr_in6 *addr);
int rdma_addr_size_kss(struct sockaddr_storage *addr);
sys/ofed/include/rdma/ib_umem.h
+4 -5
View File
@@ -44,15 +44,14 @@ struct ib_umem_odp;
struct ib_umem {
struct ib_ucontext *context;
struct mm_struct *owning_mm;
size_t length;
unsigned long address;
int page_shift;
int writable;
u32 writable : 1;
u32 hugetlb : 1;
u32 is_odp : 1;
struct work_struct work;
pid_t pid;
struct mm_struct *mm;
unsigned long diff;
struct ib_umem_odp *odp_data;
struct sg_table sg_head;
int nmap;
int npages;
sys/ofed/include/rdma/ib_umem_odp.h
+41 -34
View File
@@ -46,6 +46,9 @@ struct umem_odp_node {
};
struct ib_umem_odp {
struct ib_umem umem;
struct ib_ucontext_per_mm *per_mm;
/*
* An array of the pages included in the on-demand paging umem.
* Indices of pages that are currently not mapped into the device will
@@ -67,16 +70,9 @@ struct ib_umem_odp {
struct mutex umem_mutex;
void *private; /* for the HW driver to use. */
/* When false, use the notifier counter in the ucontext struct. */
bool mn_counters_active;
int notifiers_seq;
int notifiers_count;
/* A linked list of umems that don't have private mmu notifier
* counters yet. */
struct list_head no_private_counters;
struct ib_umem *umem;
/* Tree tracking */
struct umem_odp_node interval_tree;
@@ -85,15 +81,34 @@ struct ib_umem_odp {
struct work_struct work;
};
static inline struct ib_umem_odp *to_ib_umem_odp(struct ib_umem *umem)
{
return container_of(umem, struct ib_umem_odp, umem);
}
#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
int access);
struct ib_umem *ib_alloc_odp_umem(struct ib_ucontext *context,
unsigned long addr,
size_t size);
struct ib_ucontext_per_mm {
struct ib_ucontext *context;
struct mm_struct *mm;
struct pid *tgid;
bool active;
void ib_umem_odp_release(struct ib_umem *umem);
struct rb_root_cached umem_tree;
/* Protects umem_tree */
struct rw_semaphore umem_rwsem;
struct mmu_notifier mn;
unsigned int odp_mrs_count;
struct list_head ucontext_list;
struct rcu_head rcu;
};
int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access);
struct ib_umem_odp *ib_alloc_odp_umem(struct ib_ucontext_per_mm *per_mm,
unsigned long addr, size_t size);
void ib_umem_odp_release(struct ib_umem_odp *umem_odp);
/*
* The lower 2 bits of the DMA address signal the R/W permissions for
@@ -108,13 +123,14 @@ void ib_umem_odp_release(struct ib_umem *umem);
#define ODP_DMA_ADDR_MASK (~(ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT))
int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 start_offset, u64 bcnt,
u64 access_mask, unsigned long current_seq);
int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 start_offset,
u64 bcnt, u64 access_mask,
unsigned long current_seq);
void ib_umem_odp_unmap_dma_pages(struct ib_umem *umem, u64 start_offset,
void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 start_offset,
u64 bound);
typedef int (*umem_call_back)(struct ib_umem *item, u64 start, u64 end,
typedef int (*umem_call_back)(struct ib_umem_odp *item, u64 start, u64 end,
void *cookie);
/*
* Call the callback on each ib_umem in the range. Returns the logical or of
@@ -132,46 +148,37 @@ int rbt_ib_umem_for_each_in_range(struct rb_root_cached *root,
struct ib_umem_odp *rbt_ib_umem_lookup(struct rb_root_cached *root,
u64 addr, u64 length);
static inline int ib_umem_mmu_notifier_retry(struct ib_umem *item,
static inline int ib_umem_mmu_notifier_retry(struct ib_umem_odp *umem_odp,
unsigned long mmu_seq)
{
/*
* This code is strongly based on the KVM code from
* mmu_notifier_retry. Should be called with
* the relevant locks taken (item->odp_data->umem_mutex
* the relevant locks taken (umem_odp->umem_mutex
* and the ucontext umem_mutex semaphore locked for read).
*/
/* Do not allow page faults while the new ib_umem hasn't seen a state
* with zero notifiers yet, and doesn't have its own valid set of
* private counters. */
if (!item->odp_data->mn_counters_active)
if (unlikely(umem_odp->notifiers_count))
return 1;
if (unlikely(item->odp_data->notifiers_count))
return 1;
if (item->odp_data->notifiers_seq != mmu_seq)
if (umem_odp->notifiers_seq != mmu_seq)
return 1;
return 0;
}
#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
static inline int ib_umem_odp_get(struct ib_ucontext *context,
struct ib_umem *umem,
int access)
static inline int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access)
{
return -EINVAL;
}
static inline struct ib_umem *ib_alloc_odp_umem(struct ib_ucontext *context,
unsigned long addr,
size_t size)
static inline struct ib_umem_odp *
ib_alloc_odp_umem(struct ib_ucontext *context, unsigned long addr, size_t size)
{
return ERR_PTR(-EINVAL);
}
static inline void ib_umem_odp_release(struct ib_umem *umem) {}
static inline void ib_umem_odp_release(struct ib_umem_odp *umem_odp) {}
#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
sys/ofed/include/rdma/ib_verbs.h
+15 -2
View File
@@ -1088,7 +1088,9 @@ enum ib_qp_create_flags {
*/
struct ib_qp_init_attr {
/* Consumer's event_handler callback must not block */
void (*event_handler)(struct ib_event *, void *);
void *qp_context;
struct ib_cq *send_cq;
struct ib_cq *recv_cq;
@@ -1441,6 +1443,13 @@ struct ib_ucontext {
bool cleanup_retryable;
#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
void (*invalidate_range)(struct ib_umem_odp *umem_odp,
unsigned long start, unsigned long end);
struct mutex per_mm_list_lock;
struct list_head per_mm_list;
#endif
struct ib_rdmacg_object cg_obj;
/*
* Implementation details of the RDMA core, don't use in drivers:
@@ -2135,10 +2144,11 @@ struct ib_device {
struct list_head event_handler_list;
spinlock_t event_handler_lock;
spinlock_t client_data_lock;
rwlock_t client_data_lock;
struct list_head core_list;
/* Access to the client_data_list is protected by the client_data_lock
* spinlock and the lists_rwsem read-write semaphore */
* rwlock and the lists_rwsem read-write semaphore
*/
struct list_head client_data_list;
struct ib_cache cache;
@@ -2409,6 +2419,9 @@ struct ib_device {
struct module *owner;
struct device dev;
/* First group for device attributes, NULL terminated array */
const struct attribute_group *groups[2];
struct kobject *ports_parent;
struct list_head port_list;
sys/ofed/include/rdma/rdma_cm.h
+5 -1
View File
@@ -155,7 +155,11 @@ struct rdma_cm_id {
* @ps: RDMA port space.
* @qp_type: type of queue pair associated with the id.
*
* The id holds a reference on the network namespace until it is destroyed.
* Returns a new rdma_cm_id. The id holds a reference on the network
* namespace until it is destroyed.
*
* The event handler callback serializes on the id's mutex and is
* allowed to sleep.
*/
struct rdma_cm_id *rdma_create_id(struct vnet *net,
rdma_cm_event_handler event_handler,