Initial commit: Linux kernel 4.19 for Unplugged OS

This commit is contained in:
ziv committed 2025-10-06 13:52:42 +00:00
commit d30235b04c
80519 files changed
+35800681

No files matched your search

+41
View File
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: GPL-2.0
infiniband-$(CONFIG_INFINIBAND_ADDR_TRANS) := rdma_cm.o
user_access-$(CONFIG_INFINIBAND_ADDR_TRANS) := rdma_ucm.o
obj-$(CONFIG_INFINIBAND) += ib_core.o ib_cm.o iw_cm.o \
$(infiniband-y)
obj-$(CONFIG_INFINIBAND_USER_MAD) += ib_umad.o
obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o $(user_access-y)
obj-$(CONFIG_INFINIBAND_USER_ACCESS_UCM) += ib_ucm.o $(user_access-y)
ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
device.o fmr_pool.o cache.o netlink.o \
roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \
multicast.o mad.o smi.o agent.o mad_rmpp.o \
nldev.o restrack.o
ib_core-$(CONFIG_SECURITY_INFINIBAND) += security.o
ib_core-$(CONFIG_INFINIBAND_USER_MEM) += umem.o
ib_core-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += umem_odp.o
ib_core-$(CONFIG_CGROUP_RDMA) += cgroup.o
ib_cm-y := cm.o
iw_cm-y := iwcm.o iwpm_util.o iwpm_msg.o
rdma_cm-y := cma.o
rdma_cm-$(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS) += cma_configfs.o
rdma_ucm-y := ucma.o
ib_umad-y := user_mad.o
ib_ucm-y := ucm.o
ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \
rdma_core.o uverbs_std_types.o uverbs_ioctl.o \
uverbs_std_types_cq.o \
uverbs_std_types_flow_action.o uverbs_std_types_dm.o \
uverbs_std_types_mr.o uverbs_std_types_counters.o \
uverbs_uapi.o
+785
View File
@@ -0,0 +1,785 @@
/*
* Copyright (c) 2005 Voltaire Inc. All rights reserved.
* Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
* Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
* Copyright (c) 2005 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/mutex.h>
#include <linux/inetdevice.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/module.h>
#include <net/arp.h>
#include <net/neighbour.h>
#include <net/route.h>
#include <net/netevent.h>
#include <net/addrconf.h>
#include <net/ip6_route.h>
#include <rdma/ib_addr.h>
#include <rdma/ib.h>
#include <rdma/rdma_netlink.h>
#include <net/netlink.h>
#include "core_priv.h"
struct addr_req {
struct list_head list;
struct sockaddr_storage src_addr;
struct sockaddr_storage dst_addr;
struct rdma_dev_addr *addr;
void *context;
void (*callback)(int status, struct sockaddr *src_addr,
struct rdma_dev_addr *addr, void *context);
unsigned long timeout;
struct delayed_work work;
int status;
u32 seq;
};
static atomic_t ib_nl_addr_request_seq = ATOMIC_INIT(0);
static DEFINE_SPINLOCK(lock);
static LIST_HEAD(req_list);
static struct workqueue_struct *addr_wq;
static const struct nla_policy ib_nl_addr_policy[LS_NLA_TYPE_MAX] = {
[LS_NLA_TYPE_DGID] = {.type = NLA_BINARY,
.len = sizeof(struct rdma_nla_ls_gid)},
};
static inline bool ib_nl_is_good_ip_resp(const struct nlmsghdr *nlh)
{
struct nlattr *tb[LS_NLA_TYPE_MAX] = {};
int ret;
if (nlh->nlmsg_flags & RDMA_NL_LS_F_ERR)
return false;
ret = nla_parse(tb, LS_NLA_TYPE_MAX - 1, nlmsg_data(nlh),
nlmsg_len(nlh), ib_nl_addr_policy, NULL);
if (ret)
return false;
return true;
}
static void ib_nl_process_good_ip_rsep(const struct nlmsghdr *nlh)
{
const struct nlattr *head, *curr;
union ib_gid gid;
struct addr_req *req;
int len, rem;
int found = 0;
head = (const struct nlattr *)nlmsg_data(nlh);
len = nlmsg_len(nlh);
nla_for_each_attr(curr, head, len, rem) {
if (curr->nla_type == LS_NLA_TYPE_DGID)
memcpy(&gid, nla_data(curr), nla_len(curr));
}
spin_lock_bh(&lock);
list_for_each_entry(req, &req_list, list) {
if (nlh->nlmsg_seq != req->seq)
continue;
/* We set the DGID part, the rest was set earlier */
rdma_addr_set_dgid(req->addr, &gid);
req->status = 0;
found = 1;
break;
}
spin_unlock_bh(&lock);
if (!found)
pr_info("Couldn't find request waiting for DGID: %pI6\n",
&gid);
}
int ib_nl_handle_ip_res_resp(struct sk_buff *skb,
struct nlmsghdr *nlh,
struct netlink_ext_ack *extack)
{
if ((nlh->nlmsg_flags & NLM_F_REQUEST) ||
!(NETLINK_CB(skb).sk))
return -EPERM;
if (ib_nl_is_good_ip_resp(nlh))
ib_nl_process_good_ip_rsep(nlh);
return 0;
}
static int ib_nl_ip_send_msg(struct rdma_dev_addr *dev_addr,
const void *daddr,
u32 seq, u16 family)
{
struct sk_buff *skb = NULL;
struct nlmsghdr *nlh;
struct rdma_ls_ip_resolve_header *header;
void *data;
size_t size;
int attrtype;
int len;
if (family == AF_INET) {
size = sizeof(struct in_addr);
attrtype = RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_IPV4;
} else {
size = sizeof(struct in6_addr);
attrtype = RDMA_NLA_F_MANDATORY | LS_NLA_TYPE_IPV6;
}
len = nla_total_size(sizeof(size));
len += NLMSG_ALIGN(sizeof(*header));
skb = nlmsg_new(len, GFP_KERNEL);
if (!skb)
return -ENOMEM;
data = ibnl_put_msg(skb, &nlh, seq, 0, RDMA_NL_LS,
RDMA_NL_LS_OP_IP_RESOLVE, NLM_F_REQUEST);
if (!data) {
nlmsg_free(skb);
return -ENODATA;
}
/* Construct the family header first */
header = skb_put(skb, NLMSG_ALIGN(sizeof(*header)));
header->ifindex = dev_addr->bound_dev_if;
nla_put(skb, attrtype, size, daddr);
/* Repair the nlmsg header length */
nlmsg_end(skb, nlh);
rdma_nl_multicast(skb, RDMA_NL_GROUP_LS, GFP_KERNEL);
/* Make the request retry, so when we get the response from userspace
* we will have something.
*/
return -ENODATA;
}
int rdma_addr_size(const struct sockaddr *addr)
{
switch (addr->sa_family) {
case AF_INET:
return sizeof(struct sockaddr_in);
case AF_INET6:
return sizeof(struct sockaddr_in6);
case AF_IB:
return sizeof(struct sockaddr_ib);
default:
return 0;
}
}
EXPORT_SYMBOL(rdma_addr_size);
int rdma_addr_size_in6(struct sockaddr_in6 *addr)
{
int ret = rdma_addr_size((struct sockaddr *) addr);
return ret <= sizeof(*addr) ? ret : 0;
}
EXPORT_SYMBOL(rdma_addr_size_in6);
int rdma_addr_size_kss(struct __kernel_sockaddr_storage *addr)
{
int ret = rdma_addr_size((struct sockaddr *) addr);
return ret <= sizeof(*addr) ? ret : 0;
}
EXPORT_SYMBOL(rdma_addr_size_kss);
void rdma_copy_addr(struct rdma_dev_addr *dev_addr,
const struct net_device *dev,
const unsigned char *dst_dev_addr)
{
dev_addr->dev_type = dev->type;
memcpy(dev_addr->src_dev_addr, dev->dev_addr, MAX_ADDR_LEN);
memcpy(dev_addr->broadcast, dev->broadcast, MAX_ADDR_LEN);
if (dst_dev_addr)
memcpy(dev_addr->dst_dev_addr, dst_dev_addr, MAX_ADDR_LEN);
dev_addr->bound_dev_if = dev->ifindex;
}
EXPORT_SYMBOL(rdma_copy_addr);
int rdma_translate_ip(const struct sockaddr *addr,
struct rdma_dev_addr *dev_addr)
{
struct net_device *dev;
if (dev_addr->bound_dev_if) {
dev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
if (!dev)
return -ENODEV;
rdma_copy_addr(dev_addr, dev, NULL);
dev_put(dev);
return 0;
}
switch (addr->sa_family) {
case AF_INET:
dev = ip_dev_find(dev_addr->net,
((const struct sockaddr_in *)addr)->sin_addr.s_addr);
if (!dev)
return -EADDRNOTAVAIL;
rdma_copy_addr(dev_addr, dev, NULL);
dev_put(dev);
break;
#if IS_ENABLED(CONFIG_IPV6)
case AF_INET6:
rcu_read_lock();
for_each_netdev_rcu(dev_addr->net, dev) {
if (ipv6_chk_addr(dev_addr->net,
&((const struct sockaddr_in6 *)addr)->sin6_addr,
dev, 1)) {
rdma_copy_addr(dev_addr, dev, NULL);
break;
}
}
rcu_read_unlock();
break;
#endif
}
return 0;
}
EXPORT_SYMBOL(rdma_translate_ip);
static void set_timeout(struct addr_req *req, unsigned long time)
{
unsigned long delay;
delay = time - jiffies;
if ((long)delay < 0)
delay = 0;
mod_delayed_work(addr_wq, &req->work, delay);
}
static void queue_req(struct addr_req *req)
{
spin_lock_bh(&lock);
list_add_tail(&req->list, &req_list);
set_timeout(req, req->timeout);
spin_unlock_bh(&lock);
}
static int ib_nl_fetch_ha(const struct dst_entry *dst,
struct rdma_dev_addr *dev_addr,
const void *daddr, u32 seq, u16 family)
{
if (rdma_nl_chk_listeners(RDMA_NL_GROUP_LS))
return -EADDRNOTAVAIL;
/* We fill in what we can, the response will fill the rest */
rdma_copy_addr(dev_addr, dst->dev, NULL);
return ib_nl_ip_send_msg(dev_addr, daddr, seq, family);
}
static int dst_fetch_ha(const struct dst_entry *dst,
struct rdma_dev_addr *dev_addr,
const void *daddr)
{
struct neighbour *n;
int ret = 0;
n = dst_neigh_lookup(dst, daddr);
if (!n)
return -ENODATA;
if (!(n->nud_state & NUD_VALID)) {
neigh_event_send(n, NULL);
ret = -ENODATA;
} else {
rdma_copy_addr(dev_addr, dst->dev, n->ha);
}
neigh_release(n);
return ret;
}
static bool has_gateway(const struct dst_entry *dst, sa_family_t family)
{
struct rtable *rt;
struct rt6_info *rt6;
if (family == AF_INET) {
rt = container_of(dst, struct rtable, dst);
return rt->rt_uses_gateway;
}
rt6 = container_of(dst, struct rt6_info, dst);
return rt6->rt6i_flags & RTF_GATEWAY;
}
static int fetch_ha(const struct dst_entry *dst, struct rdma_dev_addr *dev_addr,
const struct sockaddr *dst_in, u32 seq)
{
const struct sockaddr_in *dst_in4 =
(const struct sockaddr_in *)dst_in;
const struct sockaddr_in6 *dst_in6 =
(const struct sockaddr_in6 *)dst_in;
const void *daddr = (dst_in->sa_family == AF_INET) ?
(const void *)&dst_in4->sin_addr.s_addr :
(const void *)&dst_in6->sin6_addr;
sa_family_t family = dst_in->sa_family;
/* Gateway + ARPHRD_INFINIBAND -> IB router */
if (has_gateway(dst, family) && dst->dev->type == ARPHRD_INFINIBAND)
return ib_nl_fetch_ha(dst, dev_addr, daddr, seq, family);
else
return dst_fetch_ha(dst, dev_addr, daddr);
}
static int addr4_resolve(struct sockaddr_in *src_in,
const struct sockaddr_in *dst_in,
struct rdma_dev_addr *addr,
struct rtable **prt)
{
__be32 src_ip = src_in->sin_addr.s_addr;
__be32 dst_ip = dst_in->sin_addr.s_addr;
struct rtable *rt;
struct flowi4 fl4;
int ret;
memset(&fl4, 0, sizeof(fl4));
fl4.daddr = dst_ip;
fl4.saddr = src_ip;
fl4.flowi4_oif = addr->bound_dev_if;
rt = ip_route_output_key(addr->net, &fl4);
ret = PTR_ERR_OR_ZERO(rt);
if (ret)
return ret;
src_in->sin_family = AF_INET;
src_in->sin_addr.s_addr = fl4.saddr;
/* If there's a gateway and type of device not ARPHRD_INFINIBAND, we're
* definitely in RoCE v2 (as RoCE v1 isn't routable) set the network
* type accordingly.
*/
if (rt->rt_uses_gateway && rt->dst.dev->type != ARPHRD_INFINIBAND)
addr->network = RDMA_NETWORK_IPV4;
addr->hoplimit = ip4_dst_hoplimit(&rt->dst);
*prt = rt;
return 0;
}
#if IS_ENABLED(CONFIG_IPV6)
static int addr6_resolve(struct sockaddr_in6 *src_in,
const struct sockaddr_in6 *dst_in,
struct rdma_dev_addr *addr,
struct dst_entry **pdst)
{
struct flowi6 fl6;
struct dst_entry *dst;
struct rt6_info *rt;
memset(&fl6, 0, sizeof fl6);
fl6.daddr = dst_in->sin6_addr;
fl6.saddr = src_in->sin6_addr;
fl6.flowi6_oif = addr->bound_dev_if;
dst = ipv6_stub->ipv6_dst_lookup_flow(addr->net, NULL, &fl6, NULL);
if (IS_ERR(dst))
return PTR_ERR(dst);
rt = (struct rt6_info *)dst;
if (ipv6_addr_any(&src_in->sin6_addr)) {
src_in->sin6_family = AF_INET6;
src_in->sin6_addr = fl6.saddr;
}
/* If there's a gateway and type of device not ARPHRD_INFINIBAND, we're
* definitely in RoCE v2 (as RoCE v1 isn't routable) set the network
* type accordingly.
*/
if (rt->rt6i_flags & RTF_GATEWAY &&
ip6_dst_idev(dst)->dev->type != ARPHRD_INFINIBAND)
addr->network = RDMA_NETWORK_IPV6;
addr->hoplimit = ip6_dst_hoplimit(dst);
*pdst = dst;
return 0;
}
#else
static int addr6_resolve(struct sockaddr_in6 *src_in,
const struct sockaddr_in6 *dst_in,
struct rdma_dev_addr *addr,
struct dst_entry **pdst)
{
return -EADDRNOTAVAIL;
}
#endif
static int addr_resolve_neigh(const struct dst_entry *dst,
const struct sockaddr *dst_in,
struct rdma_dev_addr *addr,
u32 seq)
{
if (dst->dev->flags & IFF_LOOPBACK) {
int ret;
ret = rdma_translate_ip(dst_in, addr);
if (!ret)
memcpy(addr->dst_dev_addr, addr->src_dev_addr,
MAX_ADDR_LEN);
return ret;
}
/* If the device doesn't do ARP internally */
if (!(dst->dev->flags & IFF_NOARP))
return fetch_ha(dst, addr, dst_in, seq);
rdma_copy_addr(addr, dst->dev, NULL);
return 0;
}
static int addr_resolve(struct sockaddr *src_in,
const struct sockaddr *dst_in,
struct rdma_dev_addr *addr,
bool resolve_neigh,
u32 seq)
{
struct net_device *ndev;
struct dst_entry *dst;
int ret;
if (!addr->net) {
pr_warn_ratelimited("%s: missing namespace\n", __func__);
return -EINVAL;
}
if (src_in->sa_family == AF_INET) {
struct rtable *rt = NULL;
const struct sockaddr_in *dst_in4 =
(const struct sockaddr_in *)dst_in;
ret = addr4_resolve((struct sockaddr_in *)src_in,
dst_in4, addr, &rt);
if (ret)
return ret;
if (resolve_neigh)
ret = addr_resolve_neigh(&rt->dst, dst_in, addr, seq);
if (addr->bound_dev_if) {
ndev = dev_get_by_index(addr->net, addr->bound_dev_if);
} else {
ndev = rt->dst.dev;
dev_hold(ndev);
}
ip_rt_put(rt);
} else {
const struct sockaddr_in6 *dst_in6 =
(const struct sockaddr_in6 *)dst_in;
ret = addr6_resolve((struct sockaddr_in6 *)src_in,
dst_in6, addr,
&dst);
if (ret)
return ret;
if (resolve_neigh)
ret = addr_resolve_neigh(dst, dst_in, addr, seq);
if (addr->bound_dev_if) {
ndev = dev_get_by_index(addr->net, addr->bound_dev_if);
} else {
ndev = dst->dev;
dev_hold(ndev);
}
dst_release(dst);
}
if (ndev) {
if (ndev->flags & IFF_LOOPBACK)
ret = rdma_translate_ip(dst_in, addr);
else
addr->bound_dev_if = ndev->ifindex;
dev_put(ndev);
}
return ret;
}
static void process_one_req(struct work_struct *_work)
{
struct addr_req *req;
struct sockaddr *src_in, *dst_in;
req = container_of(_work, struct addr_req, work.work);
if (req->status == -ENODATA) {
src_in = (struct sockaddr *)&req->src_addr;
dst_in = (struct sockaddr *)&req->dst_addr;
req->status = addr_resolve(src_in, dst_in, req->addr,
true, req->seq);
if (req->status && time_after_eq(jiffies, req->timeout)) {
req->status = -ETIMEDOUT;
} else if (req->status == -ENODATA) {
/* requeue the work for retrying again */
spin_lock_bh(&lock);
if (!list_empty(&req->list))
set_timeout(req, req->timeout);
spin_unlock_bh(&lock);
return;
}
}
req->callback(req->status, (struct sockaddr *)&req->src_addr,
req->addr, req->context);
req->callback = NULL;
spin_lock_bh(&lock);
/*
* Although the work will normally have been canceled by the workqueue,
* it can still be requeued as long as it is on the req_list.
*/
cancel_delayed_work(&req->work);
if (!list_empty(&req->list)) {
list_del_init(&req->list);
kfree(req);
}
spin_unlock_bh(&lock);
}
int rdma_resolve_ip(struct sockaddr *src_addr, const struct sockaddr *dst_addr,
struct rdma_dev_addr *addr, int timeout_ms,
void (*callback)(int status, struct sockaddr *src_addr,
struct rdma_dev_addr *addr, void *context),
void *context)
{
struct sockaddr *src_in, *dst_in;
struct addr_req *req;
int ret = 0;
req = kzalloc(sizeof *req, GFP_KERNEL);
if (!req)
return -ENOMEM;
src_in = (struct sockaddr *) &req->src_addr;
dst_in = (struct sockaddr *) &req->dst_addr;
if (src_addr) {
if (src_addr->sa_family != dst_addr->sa_family) {
ret = -EINVAL;
goto err;
}
memcpy(src_in, src_addr, rdma_addr_size(src_addr));
} else {
src_in->sa_family = dst_addr->sa_family;
}
memcpy(dst_in, dst_addr, rdma_addr_size(dst_addr));
req->addr = addr;
req->callback = callback;
req->context = context;
INIT_DELAYED_WORK(&req->work, process_one_req);
req->seq = (u32)atomic_inc_return(&ib_nl_addr_request_seq);
req->status = addr_resolve(src_in, dst_in, addr, true, req->seq);
switch (req->status) {
case 0:
req->timeout = jiffies;
queue_req(req);
break;
case -ENODATA:
req->timeout = msecs_to_jiffies(timeout_ms) + jiffies;
queue_req(req);
break;
default:
ret = req->status;
goto err;
}
return ret;
err:
kfree(req);
return ret;
}
EXPORT_SYMBOL(rdma_resolve_ip);
int rdma_resolve_ip_route(struct sockaddr *src_addr,
const struct sockaddr *dst_addr,
struct rdma_dev_addr *addr)
{
struct sockaddr_storage ssrc_addr = {};
struct sockaddr *src_in = (struct sockaddr *)&ssrc_addr;
if (src_addr) {
if (src_addr->sa_family != dst_addr->sa_family)
return -EINVAL;
memcpy(src_in, src_addr, rdma_addr_size(src_addr));
} else {
src_in->sa_family = dst_addr->sa_family;
}
return addr_resolve(src_in, dst_addr, addr, false, 0);
}
void rdma_addr_cancel(struct rdma_dev_addr *addr)
{
struct addr_req *req, *temp_req;
struct addr_req *found = NULL;
spin_lock_bh(&lock);
list_for_each_entry_safe(req, temp_req, &req_list, list) {
if (req->addr == addr) {
/*
* Removing from the list means we take ownership of
* the req
*/
list_del_init(&req->list);
found = req;
break;
}
}
spin_unlock_bh(&lock);
if (!found)
return;
/*
* sync canceling the work after removing it from the req_list
* guarentees no work is running and none will be started.
*/
cancel_delayed_work_sync(&found->work);
if (found->callback)
found->callback(-ECANCELED, (struct sockaddr *)&found->src_addr,
found->addr, found->context);
kfree(found);
}
EXPORT_SYMBOL(rdma_addr_cancel);
struct resolve_cb_context {
struct completion comp;
int status;
};
static void resolve_cb(int status, struct sockaddr *src_addr,
struct rdma_dev_addr *addr, void *context)
{
((struct resolve_cb_context *)context)->status = status;
complete(&((struct resolve_cb_context *)context)->comp);
}
int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
const union ib_gid *dgid,
u8 *dmac, const struct net_device *ndev,
int *hoplimit)
{
struct rdma_dev_addr dev_addr;
struct resolve_cb_context ctx;
union {
struct sockaddr_in _sockaddr_in;
struct sockaddr_in6 _sockaddr_in6;
} sgid_addr, dgid_addr;
int ret;
rdma_gid2ip((struct sockaddr *)&sgid_addr, sgid);
rdma_gid2ip((struct sockaddr *)&dgid_addr, dgid);
memset(&dev_addr, 0, sizeof(dev_addr));
dev_addr.bound_dev_if = ndev->ifindex;
dev_addr.net = &init_net;
init_completion(&ctx.comp);
ret = rdma_resolve_ip((struct sockaddr *)&sgid_addr,
(struct sockaddr *)&dgid_addr, &dev_addr, 1000,
resolve_cb, &ctx);
if (ret)
return ret;
wait_for_completion(&ctx.comp);
ret = ctx.status;
if (ret)
return ret;
memcpy(dmac, dev_addr.dst_dev_addr, ETH_ALEN);
*hoplimit = dev_addr.hoplimit;
return 0;
}
static int netevent_callback(struct notifier_block *self, unsigned long event,
void *ctx)
{
struct addr_req *req;
if (event == NETEVENT_NEIGH_UPDATE) {
struct neighbour *neigh = ctx;
if (neigh->nud_state & NUD_VALID) {
spin_lock_bh(&lock);
list_for_each_entry(req, &req_list, list)
set_timeout(req, jiffies);
spin_unlock_bh(&lock);
}
}
return 0;
}
static struct notifier_block nb = {
.notifier_call = netevent_callback
};
int addr_init(void)
{
addr_wq = alloc_ordered_workqueue("ib_addr", 0);
if (!addr_wq)
return -ENOMEM;
register_netevent_notifier(&nb);
return 0;
}
void addr_cleanup(void)
{
unregister_netevent_notifier(&nb);
destroy_workqueue(addr_wq);
WARN_ON(!list_empty(&req_list));
}
+221
View File
@@ -0,0 +1,221 @@
/*
* Copyright (c) 2004, 2005 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2004, 2005 Infinicon Corporation. All rights reserved.
* Copyright (c) 2004, 2005 Intel Corporation. All rights reserved.
* Copyright (c) 2004, 2005 Topspin Corporation. All rights reserved.
* Copyright (c) 2004-2007 Voltaire Corporation. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
#include <linux/slab.h>
#include <linux/string.h>
#include "agent.h"
#include "smi.h"
#include "mad_priv.h"
#define SPFX "ib_agent: "
struct ib_agent_port_private {
struct list_head port_list;
struct ib_mad_agent *agent[2];
};
static DEFINE_SPINLOCK(ib_agent_port_list_lock);
static LIST_HEAD(ib_agent_port_list);
static struct ib_agent_port_private *
__ib_get_agent_port(const struct ib_device *device, int port_num)
{
struct ib_agent_port_private *entry;
list_for_each_entry(entry, &ib_agent_port_list, port_list) {
if (entry->agent[1]->device == device &&
entry->agent[1]->port_num == port_num)
return entry;
}
return NULL;
}
static struct ib_agent_port_private *
ib_get_agent_port(const struct ib_device *device, int port_num)
{
struct ib_agent_port_private *entry;
unsigned long flags;
spin_lock_irqsave(&ib_agent_port_list_lock, flags);
entry = __ib_get_agent_port(device, port_num);
spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
return entry;
}
void agent_send_response(const struct ib_mad_hdr *mad_hdr, const struct ib_grh *grh,
const struct ib_wc *wc, const struct ib_device *device,
int port_num, int qpn, size_t resp_mad_len, bool opa)
{
struct ib_agent_port_private *port_priv;
struct ib_mad_agent *agent;
struct ib_mad_send_buf *send_buf;
struct ib_ah *ah;
struct ib_mad_send_wr_private *mad_send_wr;
if (rdma_cap_ib_switch(device))
port_priv = ib_get_agent_port(device, 0);
else
port_priv = ib_get_agent_port(device, port_num);
if (!port_priv) {
dev_err(&device->dev, "Unable to find port agent\n");
return;
}
agent = port_priv->agent[qpn];
ah = ib_create_ah_from_wc(agent->qp->pd, wc, grh, port_num);
if (IS_ERR(ah)) {
dev_err(&device->dev, "ib_create_ah_from_wc error %ld\n",
PTR_ERR(ah));
return;
}
if (opa && mad_hdr->base_version != OPA_MGMT_BASE_VERSION)
resp_mad_len = IB_MGMT_MAD_SIZE;
send_buf = ib_create_send_mad(agent, wc->src_qp, wc->pkey_index, 0,
IB_MGMT_MAD_HDR,
resp_mad_len - IB_MGMT_MAD_HDR,
GFP_KERNEL,
mad_hdr->base_version);
if (IS_ERR(send_buf)) {
dev_err(&device->dev, "ib_create_send_mad error\n");
goto err1;
}
memcpy(send_buf->mad, mad_hdr, resp_mad_len);
send_buf->ah = ah;
if (rdma_cap_ib_switch(device)) {
mad_send_wr = container_of(send_buf,
struct ib_mad_send_wr_private,
send_buf);
mad_send_wr->send_wr.port_num = port_num;
}
if (ib_post_send_mad(send_buf, NULL)) {
dev_err(&device->dev, "ib_post_send_mad error\n");
goto err2;
}
return;
err2:
ib_free_send_mad(send_buf);
err1:
rdma_destroy_ah(ah);
}
static void agent_send_handler(struct ib_mad_agent *mad_agent,
struct ib_mad_send_wc *mad_send_wc)
{
rdma_destroy_ah(mad_send_wc->send_buf->ah);
ib_free_send_mad(mad_send_wc->send_buf);
}
int ib_agent_port_open(struct ib_device *device, int port_num)
{
struct ib_agent_port_private *port_priv;
unsigned long flags;
int ret;
/* Create new device info */
port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL);
if (!port_priv) {
ret = -ENOMEM;
goto error1;
}
if (rdma_cap_ib_smi(device, port_num)) {
/* Obtain send only MAD agent for SMI QP */
port_priv->agent[0] = ib_register_mad_agent(device, port_num,
IB_QPT_SMI, NULL, 0,
&agent_send_handler,
NULL, NULL, 0);
if (IS_ERR(port_priv->agent[0])) {
ret = PTR_ERR(port_priv->agent[0]);
goto error2;
}
}
/* Obtain send only MAD agent for GSI QP */
port_priv->agent[1] = ib_register_mad_agent(device, port_num,
IB_QPT_GSI, NULL, 0,
&agent_send_handler,
NULL, NULL, 0);
if (IS_ERR(port_priv->agent[1])) {
ret = PTR_ERR(port_priv->agent[1]);
goto error3;
}
spin_lock_irqsave(&ib_agent_port_list_lock, flags);
list_add_tail(&port_priv->port_list, &ib_agent_port_list);
spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
return 0;
error3:
if (port_priv->agent[0])
ib_unregister_mad_agent(port_priv->agent[0]);
error2:
kfree(port_priv);
error1:
return ret;
}
int ib_agent_port_close(struct ib_device *device, int port_num)
{
struct ib_agent_port_private *port_priv;
unsigned long flags;
spin_lock_irqsave(&ib_agent_port_list_lock, flags);
port_priv = __ib_get_agent_port(device, port_num);
if (port_priv == NULL) {
spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
dev_err(&device->dev, "Port %d not found\n", port_num);
return -ENODEV;
}
list_del(&port_priv->port_list);
spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
ib_unregister_mad_agent(port_priv->agent[1]);
if (port_priv->agent[0])
ib_unregister_mad_agent(port_priv->agent[0]);
kfree(port_priv);
return 0;
}
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
* Copyright (c) 2004 Intel Corporation. All rights reserved.
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef __AGENT_H_
#define __AGENT_H_
#include <linux/err.h>
#include <rdma/ib_mad.h>
extern int ib_agent_port_open(struct ib_device *device, int port_num);
extern int ib_agent_port_close(struct ib_device *device, int port_num);
extern void agent_send_response(const struct ib_mad_hdr *mad_hdr, const struct ib_grh *grh,
const struct ib_wc *wc, const struct ib_device *device,
int port_num, int qpn, size_t resp_mad_len, bool opa);
#endif /* __AGENT_H_ */
File diff suppressed because it is too large. Load diff
+62
View File
@@ -0,0 +1,62 @@
/*
* Copyright (C) 2016 Parav Pandit <pandit.parav@gmail.com>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include "core_priv.h"
/**
* ib_device_register_rdmacg - register with rdma cgroup.
* @device: device to register to participate in resource
* accounting by rdma cgroup.
*
* Register with the rdma cgroup. Should be called before
* exposing rdma device to user space applications to avoid
* resource accounting leak.
* Returns 0 on success or otherwise failure code.
*/
int ib_device_register_rdmacg(struct ib_device *device)
{
device->cg_device.name = device->name;
return rdmacg_register_device(&device->cg_device);
}
/**
* ib_device_unregister_rdmacg - unregister with rdma cgroup.
* @device: device to unregister.
*
* Unregister with the rdma cgroup. Should be called after
* all the resources are deallocated, and after a stage when any
* other resource allocation by user application cannot be done
* for this device to avoid any leak in accounting.
*/
void ib_device_unregister_rdmacg(struct ib_device *device)
{
rdmacg_unregister_device(&device->cg_device);
}
int ib_rdmacg_try_charge(struct ib_rdmacg_object *cg_obj,
struct ib_device *device,
enum rdmacg_resource_type resource_index)
{
return rdmacg_try_charge(&cg_obj->cg, &device->cg_device,
resource_index);
}
EXPORT_SYMBOL(ib_rdmacg_try_charge);
void ib_rdmacg_uncharge(struct ib_rdmacg_object *cg_obj,
struct ib_device *device,
enum rdmacg_resource_type resource_index)
{
rdmacg_uncharge(cg_obj->cg, &device->cg_device,
resource_index);
}
EXPORT_SYMBOL(ib_rdmacg_uncharge);
File diff suppressed because it is too large. Load diff
+829
View File
@@ -0,0 +1,829 @@
/*
* Copyright (c) 2004, 2011 Intel Corporation. All rights reserved.
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
* Copyright (c) 2004 Voltaire Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING the madirectory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use source and binary forms, with or
* withmodification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retathe above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHWARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS THE
* SOFTWARE.
*/
#if !defined(CM_MSGS_H)
#define CM_MSGS_H
#include <rdma/ib_mad.h>
#include <rdma/ib_cm.h>
/*
* Parameters to routines below should be in network-byte order, and values
* are returned in network-byte order.
*/
#define IB_CM_CLASS_VERSION 2 /* IB specification 1.2 */
struct cm_req_msg {
struct ib_mad_hdr hdr;
__be32 local_comm_id;
__be32 rsvd4;
__be64 service_id;
__be64 local_ca_guid;
__be32 rsvd24;
__be32 local_qkey;
/* local QPN:24, responder resources:8 */
__be32 offset32;
/* local EECN:24, initiator depth:8 */
__be32 offset36;
/*
* remote EECN:24, remote CM response timeout:5,
* transport service type:2, end-to-end flow control:1
*/
__be32 offset40;
/* starting PSN:24, local CM response timeout:5, retry count:3 */
__be32 offset44;
__be16 pkey;
/* path MTU:4, RDC exists:1, RNR retry count:3. */
u8 offset50;
/* max CM Retries:4, SRQ:1, extended transport type:3 */
u8 offset51;
__be16 primary_local_lid;
__be16 primary_remote_lid;
union ib_gid primary_local_gid;
union ib_gid primary_remote_gid;
/* flow label:20, rsvd:6, packet rate:6 */
__be32 primary_offset88;
u8 primary_traffic_class;
u8 primary_hop_limit;
/* SL:4, subnet local:1, rsvd:3 */
u8 primary_offset94;
/* local ACK timeout:5, rsvd:3 */
u8 primary_offset95;
__be16 alt_local_lid;
__be16 alt_remote_lid;
union ib_gid alt_local_gid;
union ib_gid alt_remote_gid;
/* flow label:20, rsvd:6, packet rate:6 */
__be32 alt_offset132;
u8 alt_traffic_class;
u8 alt_hop_limit;
/* SL:4, subnet local:1, rsvd:3 */
u8 alt_offset138;
/* local ACK timeout:5, rsvd:3 */
u8 alt_offset139;
u32 private_data[IB_CM_REQ_PRIVATE_DATA_SIZE / sizeof(u32)];
} __attribute__ ((packed));
static inline __be32 cm_req_get_local_qpn(struct cm_req_msg *req_msg)
{
return cpu_to_be32(be32_to_cpu(req_msg->offset32) >> 8);
}
static inline void cm_req_set_local_qpn(struct cm_req_msg *req_msg, __be32 qpn)
{
req_msg->offset32 = cpu_to_be32((be32_to_cpu(qpn) << 8) |
(be32_to_cpu(req_msg->offset32) &
0x000000FF));
}
static inline u8 cm_req_get_resp_res(struct cm_req_msg *req_msg)
{
return (u8) be32_to_cpu(req_msg->offset32);
}
static inline void cm_req_set_resp_res(struct cm_req_msg *req_msg, u8 resp_res)
{
req_msg->offset32 = cpu_to_be32(resp_res |
(be32_to_cpu(req_msg->offset32) &
0xFFFFFF00));
}
static inline u8 cm_req_get_init_depth(struct cm_req_msg *req_msg)
{
return (u8) be32_to_cpu(req_msg->offset36);
}
static inline void cm_req_set_init_depth(struct cm_req_msg *req_msg,
u8 init_depth)
{
req_msg->offset36 = cpu_to_be32(init_depth |
(be32_to_cpu(req_msg->offset36) &
0xFFFFFF00));
}
static inline u8 cm_req_get_remote_resp_timeout(struct cm_req_msg *req_msg)
{
return (u8) ((be32_to_cpu(req_msg->offset40) & 0xF8) >> 3);
}
static inline void cm_req_set_remote_resp_timeout(struct cm_req_msg *req_msg,
u8 resp_timeout)
{
req_msg->offset40 = cpu_to_be32((resp_timeout << 3) |
(be32_to_cpu(req_msg->offset40) &
0xFFFFFF07));
}
static inline enum ib_qp_type cm_req_get_qp_type(struct cm_req_msg *req_msg)
{
u8 transport_type = (u8) (be32_to_cpu(req_msg->offset40) & 0x06) >> 1;
switch(transport_type) {
case 0: return IB_QPT_RC;
case 1: return IB_QPT_UC;
case 3:
switch (req_msg->offset51 & 0x7) {
case 1: return IB_QPT_XRC_TGT;
default: return 0;
}
default: return 0;
}
}
static inline void cm_req_set_qp_type(struct cm_req_msg *req_msg,
enum ib_qp_type qp_type)
{
switch(qp_type) {
case IB_QPT_UC:
req_msg->offset40 = cpu_to_be32((be32_to_cpu(
req_msg->offset40) &
0xFFFFFFF9) | 0x2);
break;
case IB_QPT_XRC_INI:
req_msg->offset40 = cpu_to_be32((be32_to_cpu(
req_msg->offset40) &
0xFFFFFFF9) | 0x6);
req_msg->offset51 = (req_msg->offset51 & 0xF8) | 1;
break;
default:
req_msg->offset40 = cpu_to_be32(be32_to_cpu(
req_msg->offset40) &
0xFFFFFFF9);
}
}
static inline u8 cm_req_get_flow_ctrl(struct cm_req_msg *req_msg)
{
return be32_to_cpu(req_msg->offset40) & 0x1;
}
static inline void cm_req_set_flow_ctrl(struct cm_req_msg *req_msg,
u8 flow_ctrl)
{
req_msg->offset40 = cpu_to_be32((flow_ctrl & 0x1) |
(be32_to_cpu(req_msg->offset40) &
0xFFFFFFFE));
}
static inline __be32 cm_req_get_starting_psn(struct cm_req_msg *req_msg)
{
return cpu_to_be32(be32_to_cpu(req_msg->offset44) >> 8);
}
static inline void cm_req_set_starting_psn(struct cm_req_msg *req_msg,
__be32 starting_psn)
{
req_msg->offset44 = cpu_to_be32((be32_to_cpu(starting_psn) << 8) |
(be32_to_cpu(req_msg->offset44) & 0x000000FF));
}
static inline u8 cm_req_get_local_resp_timeout(struct cm_req_msg *req_msg)
{
return (u8) ((be32_to_cpu(req_msg->offset44) & 0xF8) >> 3);
}
static inline void cm_req_set_local_resp_timeout(struct cm_req_msg *req_msg,
u8 resp_timeout)
{
req_msg->offset44 = cpu_to_be32((resp_timeout << 3) |
(be32_to_cpu(req_msg->offset44) & 0xFFFFFF07));
}
static inline u8 cm_req_get_retry_count(struct cm_req_msg *req_msg)
{
return (u8) (be32_to_cpu(req_msg->offset44) & 0x7);
}
static inline void cm_req_set_retry_count(struct cm_req_msg *req_msg,
u8 retry_count)
{
req_msg->offset44 = cpu_to_be32((retry_count & 0x7) |
(be32_to_cpu(req_msg->offset44) & 0xFFFFFFF8));
}
static inline u8 cm_req_get_path_mtu(struct cm_req_msg *req_msg)
{
return req_msg->offset50 >> 4;
}
static inline void cm_req_set_path_mtu(struct cm_req_msg *req_msg, u8 path_mtu)
{
req_msg->offset50 = (u8) ((req_msg->offset50 & 0xF) | (path_mtu << 4));
}
static inline u8 cm_req_get_rnr_retry_count(struct cm_req_msg *req_msg)
{
return req_msg->offset50 & 0x7;
}
static inline void cm_req_set_rnr_retry_count(struct cm_req_msg *req_msg,
u8 rnr_retry_count)
{
req_msg->offset50 = (u8) ((req_msg->offset50 & 0xF8) |
(rnr_retry_count & 0x7));
}
static inline u8 cm_req_get_max_cm_retries(struct cm_req_msg *req_msg)
{
return req_msg->offset51 >> 4;
}
static inline void cm_req_set_max_cm_retries(struct cm_req_msg *req_msg,
u8 retries)
{
req_msg->offset51 = (u8) ((req_msg->offset51 & 0xF) | (retries << 4));
}
static inline u8 cm_req_get_srq(struct cm_req_msg *req_msg)
{
return (req_msg->offset51 & 0x8) >> 3;
}
static inline void cm_req_set_srq(struct cm_req_msg *req_msg, u8 srq)
{
req_msg->offset51 = (u8) ((req_msg->offset51 & 0xF7) |
((srq & 0x1) << 3));
}
static inline __be32 cm_req_get_primary_flow_label(struct cm_req_msg *req_msg)
{
return cpu_to_be32(be32_to_cpu(req_msg->primary_offset88) >> 12);
}
static inline void cm_req_set_primary_flow_label(struct cm_req_msg *req_msg,
__be32 flow_label)
{
req_msg->primary_offset88 = cpu_to_be32(
(be32_to_cpu(req_msg->primary_offset88) &
0x00000FFF) |
(be32_to_cpu(flow_label) << 12));
}
static inline u8 cm_req_get_primary_packet_rate(struct cm_req_msg *req_msg)
{
return (u8) (be32_to_cpu(req_msg->primary_offset88) & 0x3F);
}
static inline void cm_req_set_primary_packet_rate(struct cm_req_msg *req_msg,
u8 rate)
{
req_msg->primary_offset88 = cpu_to_be32(
(be32_to_cpu(req_msg->primary_offset88) &
0xFFFFFFC0) | (rate & 0x3F));
}
static inline u8 cm_req_get_primary_sl(struct cm_req_msg *req_msg)
{
return (u8) (req_msg->primary_offset94 >> 4);
}
static inline void cm_req_set_primary_sl(struct cm_req_msg *req_msg, u8 sl)
{
req_msg->primary_offset94 = (u8) ((req_msg->primary_offset94 & 0x0F) |
(sl << 4));
}
static inline u8 cm_req_get_primary_subnet_local(struct cm_req_msg *req_msg)
{
return (u8) ((req_msg->primary_offset94 & 0x08) >> 3);
}
static inline void cm_req_set_primary_subnet_local(struct cm_req_msg *req_msg,
u8 subnet_local)
{
req_msg->primary_offset94 = (u8) ((req_msg->primary_offset94 & 0xF7) |
((subnet_local & 0x1) << 3));
}
static inline u8 cm_req_get_primary_local_ack_timeout(struct cm_req_msg *req_msg)
{
return (u8) (req_msg->primary_offset95 >> 3);
}
static inline void cm_req_set_primary_local_ack_timeout(struct cm_req_msg *req_msg,
u8 local_ack_timeout)
{
req_msg->primary_offset95 = (u8) ((req_msg->primary_offset95 & 0x07) |
(local_ack_timeout << 3));
}
static inline __be32 cm_req_get_alt_flow_label(struct cm_req_msg *req_msg)
{
return cpu_to_be32(be32_to_cpu(req_msg->alt_offset132) >> 12);
}
static inline void cm_req_set_alt_flow_label(struct cm_req_msg *req_msg,
__be32 flow_label)
{
req_msg->alt_offset132 = cpu_to_be32(
(be32_to_cpu(req_msg->alt_offset132) &
0x00000FFF) |
(be32_to_cpu(flow_label) << 12));
}
static inline u8 cm_req_get_alt_packet_rate(struct cm_req_msg *req_msg)
{
return (u8) (be32_to_cpu(req_msg->alt_offset132) & 0x3F);
}
static inline void cm_req_set_alt_packet_rate(struct cm_req_msg *req_msg,
u8 rate)
{
req_msg->alt_offset132 = cpu_to_be32(
(be32_to_cpu(req_msg->alt_offset132) &
0xFFFFFFC0) | (rate & 0x3F));
}
static inline u8 cm_req_get_alt_sl(struct cm_req_msg *req_msg)
{
return (u8) (req_msg->alt_offset138 >> 4);
}
static inline void cm_req_set_alt_sl(struct cm_req_msg *req_msg, u8 sl)
{
req_msg->alt_offset138 = (u8) ((req_msg->alt_offset138 & 0x0F) |
(sl << 4));
}
static inline u8 cm_req_get_alt_subnet_local(struct cm_req_msg *req_msg)
{
return (u8) ((req_msg->alt_offset138 & 0x08) >> 3);
}
static inline void cm_req_set_alt_subnet_local(struct cm_req_msg *req_msg,
u8 subnet_local)
{
req_msg->alt_offset138 = (u8) ((req_msg->alt_offset138 & 0xF7) |
((subnet_local & 0x1) << 3));
}
static inline u8 cm_req_get_alt_local_ack_timeout(struct cm_req_msg *req_msg)
{
return (u8) (req_msg->alt_offset139 >> 3);
}
static inline void cm_req_set_alt_local_ack_timeout(struct cm_req_msg *req_msg,
u8 local_ack_timeout)
{
req_msg->alt_offset139 = (u8) ((req_msg->alt_offset139 & 0x07) |
(local_ack_timeout << 3));
}
/* Message REJected or MRAed */
enum cm_msg_response {
CM_MSG_RESPONSE_REQ = 0x0,
CM_MSG_RESPONSE_REP = 0x1,
CM_MSG_RESPONSE_OTHER = 0x2
};
struct cm_mra_msg {
struct ib_mad_hdr hdr;
__be32 local_comm_id;
__be32 remote_comm_id;
/* message MRAed:2, rsvd:6 */
u8 offset8;
/* service timeout:5, rsvd:3 */
u8 offset9;
u8 private_data[IB_CM_MRA_PRIVATE_DATA_SIZE];
} __attribute__ ((packed));
static inline u8 cm_mra_get_msg_mraed(struct cm_mra_msg *mra_msg)
{
return (u8) (mra_msg->offset8 >> 6);
}
static inline void cm_mra_set_msg_mraed(struct cm_mra_msg *mra_msg, u8 msg)
{
mra_msg->offset8 = (u8) ((mra_msg->offset8 & 0x3F) | (msg << 6));
}
static inline u8 cm_mra_get_service_timeout(struct cm_mra_msg *mra_msg)
{
return (u8) (mra_msg->offset9 >> 3);
}
static inline void cm_mra_set_service_timeout(struct cm_mra_msg *mra_msg,
u8 service_timeout)
{
mra_msg->offset9 = (u8) ((mra_msg->offset9 & 0x07) |
(service_timeout << 3));
}
struct cm_rej_msg {
struct ib_mad_hdr hdr;
__be32 local_comm_id;
__be32 remote_comm_id;
/* message REJected:2, rsvd:6 */
u8 offset8;
/* reject info length:7, rsvd:1. */
u8 offset9;
__be16 reason;
u8 ari[IB_CM_REJ_ARI_LENGTH];
u8 private_data[IB_CM_REJ_PRIVATE_DATA_SIZE];
} __attribute__ ((packed));
static inline u8 cm_rej_get_msg_rejected(struct cm_rej_msg *rej_msg)
{
return (u8) (rej_msg->offset8 >> 6);
}
static inline void cm_rej_set_msg_rejected(struct cm_rej_msg *rej_msg, u8 msg)
{
rej_msg->offset8 = (u8) ((rej_msg->offset8 & 0x3F) | (msg << 6));
}
static inline u8 cm_rej_get_reject_info_len(struct cm_rej_msg *rej_msg)
{
return (u8) (rej_msg->offset9 >> 1);
}
static inline void cm_rej_set_reject_info_len(struct cm_rej_msg *rej_msg,
u8 len)
{
rej_msg->offset9 = (u8) ((rej_msg->offset9 & 0x1) | (len << 1));
}
struct cm_rep_msg {
struct ib_mad_hdr hdr;
__be32 local_comm_id;
__be32 remote_comm_id;
__be32 local_qkey;
/* local QPN:24, rsvd:8 */
__be32 offset12;
/* local EECN:24, rsvd:8 */
__be32 offset16;
/* starting PSN:24 rsvd:8 */
__be32 offset20;
u8 resp_resources;
u8 initiator_depth;
/* target ACK delay:5, failover accepted:2, end-to-end flow control:1 */
u8 offset26;
/* RNR retry count:3, SRQ:1, rsvd:5 */
u8 offset27;
__be64 local_ca_guid;
u8 private_data[IB_CM_REP_PRIVATE_DATA_SIZE];
} __attribute__ ((packed));
static inline __be32 cm_rep_get_local_qpn(struct cm_rep_msg *rep_msg)
{
return cpu_to_be32(be32_to_cpu(rep_msg->offset12) >> 8);
}
static inline void cm_rep_set_local_qpn(struct cm_rep_msg *rep_msg, __be32 qpn)
{
rep_msg->offset12 = cpu_to_be32((be32_to_cpu(qpn) << 8) |
(be32_to_cpu(rep_msg->offset12) & 0x000000FF));
}
static inline __be32 cm_rep_get_local_eecn(struct cm_rep_msg *rep_msg)
{
return cpu_to_be32(be32_to_cpu(rep_msg->offset16) >> 8);
}
static inline void cm_rep_set_local_eecn(struct cm_rep_msg *rep_msg, __be32 eecn)
{
rep_msg->offset16 = cpu_to_be32((be32_to_cpu(eecn) << 8) |
(be32_to_cpu(rep_msg->offset16) & 0x000000FF));
}
static inline __be32 cm_rep_get_qpn(struct cm_rep_msg *rep_msg, enum ib_qp_type qp_type)
{
return (qp_type == IB_QPT_XRC_INI) ?
cm_rep_get_local_eecn(rep_msg) : cm_rep_get_local_qpn(rep_msg);
}
static inline __be32 cm_rep_get_starting_psn(struct cm_rep_msg *rep_msg)
{
return cpu_to_be32(be32_to_cpu(rep_msg->offset20) >> 8);
}
static inline void cm_rep_set_starting_psn(struct cm_rep_msg *rep_msg,
__be32 starting_psn)
{
rep_msg->offset20 = cpu_to_be32((be32_to_cpu(starting_psn) << 8) |
(be32_to_cpu(rep_msg->offset20) & 0x000000FF));
}
static inline u8 cm_rep_get_target_ack_delay(struct cm_rep_msg *rep_msg)
{
return (u8) (rep_msg->offset26 >> 3);
}
static inline void cm_rep_set_target_ack_delay(struct cm_rep_msg *rep_msg,
u8 target_ack_delay)
{
rep_msg->offset26 = (u8) ((rep_msg->offset26 & 0x07) |
(target_ack_delay << 3));
}
static inline u8 cm_rep_get_failover(struct cm_rep_msg *rep_msg)
{
return (u8) ((rep_msg->offset26 & 0x06) >> 1);
}
static inline void cm_rep_set_failover(struct cm_rep_msg *rep_msg, u8 failover)
{
rep_msg->offset26 = (u8) ((rep_msg->offset26 & 0xF9) |
((failover & 0x3) << 1));
}
static inline u8 cm_rep_get_flow_ctrl(struct cm_rep_msg *rep_msg)
{
return (u8) (rep_msg->offset26 & 0x01);
}
static inline void cm_rep_set_flow_ctrl(struct cm_rep_msg *rep_msg,
u8 flow_ctrl)
{
rep_msg->offset26 = (u8) ((rep_msg->offset26 & 0xFE) |
(flow_ctrl & 0x1));
}
static inline u8 cm_rep_get_rnr_retry_count(struct cm_rep_msg *rep_msg)
{
return (u8) (rep_msg->offset27 >> 5);
}
static inline void cm_rep_set_rnr_retry_count(struct cm_rep_msg *rep_msg,
u8 rnr_retry_count)
{
rep_msg->offset27 = (u8) ((rep_msg->offset27 & 0x1F) |
(rnr_retry_count << 5));
}
static inline u8 cm_rep_get_srq(struct cm_rep_msg *rep_msg)
{
return (u8) ((rep_msg->offset27 >> 4) & 0x1);
}
static inline void cm_rep_set_srq(struct cm_rep_msg *rep_msg, u8 srq)
{
rep_msg->offset27 = (u8) ((rep_msg->offset27 & 0xEF) |
((srq & 0x1) << 4));
}
struct cm_rtu_msg {
struct ib_mad_hdr hdr;
__be32 local_comm_id;
__be32 remote_comm_id;
u8 private_data[IB_CM_RTU_PRIVATE_DATA_SIZE];
} __attribute__ ((packed));
struct cm_dreq_msg {
struct ib_mad_hdr hdr;
__be32 local_comm_id;
__be32 remote_comm_id;
/* remote QPN/EECN:24, rsvd:8 */
__be32 offset8;
u8 private_data[IB_CM_DREQ_PRIVATE_DATA_SIZE];
} __attribute__ ((packed));
static inline __be32 cm_dreq_get_remote_qpn(struct cm_dreq_msg *dreq_msg)
{
return cpu_to_be32(be32_to_cpu(dreq_msg->offset8) >> 8);
}
static inline void cm_dreq_set_remote_qpn(struct cm_dreq_msg *dreq_msg, __be32 qpn)
{
dreq_msg->offset8 = cpu_to_be32((be32_to_cpu(qpn) << 8) |
(be32_to_cpu(dreq_msg->offset8) & 0x000000FF));
}
struct cm_drep_msg {
struct ib_mad_hdr hdr;
__be32 local_comm_id;
__be32 remote_comm_id;
u8 private_data[IB_CM_DREP_PRIVATE_DATA_SIZE];
} __attribute__ ((packed));
struct cm_lap_msg {
struct ib_mad_hdr hdr;
__be32 local_comm_id;
__be32 remote_comm_id;
__be32 rsvd8;
/* remote QPN/EECN:24, remote CM response timeout:5, rsvd:3 */
__be32 offset12;
__be32 rsvd16;
__be16 alt_local_lid;
__be16 alt_remote_lid;
union ib_gid alt_local_gid;
union ib_gid alt_remote_gid;
/* flow label:20, rsvd:4, traffic class:8 */
__be32 offset56;
u8 alt_hop_limit;
/* rsvd:2, packet rate:6 */
u8 offset61;
/* SL:4, subnet local:1, rsvd:3 */
u8 offset62;
/* local ACK timeout:5, rsvd:3 */
u8 offset63;
u8 private_data[IB_CM_LAP_PRIVATE_DATA_SIZE];
} __attribute__ ((packed));
static inline __be32 cm_lap_get_remote_qpn(struct cm_lap_msg *lap_msg)
{
return cpu_to_be32(be32_to_cpu(lap_msg->offset12) >> 8);
}
static inline void cm_lap_set_remote_qpn(struct cm_lap_msg *lap_msg, __be32 qpn)
{
lap_msg->offset12 = cpu_to_be32((be32_to_cpu(qpn) << 8) |
(be32_to_cpu(lap_msg->offset12) &
0x000000FF));
}
static inline u8 cm_lap_get_remote_resp_timeout(struct cm_lap_msg *lap_msg)
{
return (u8) ((be32_to_cpu(lap_msg->offset12) & 0xF8) >> 3);
}
static inline void cm_lap_set_remote_resp_timeout(struct cm_lap_msg *lap_msg,
u8 resp_timeout)
{
lap_msg->offset12 = cpu_to_be32((resp_timeout << 3) |
(be32_to_cpu(lap_msg->offset12) &
0xFFFFFF07));
}
static inline __be32 cm_lap_get_flow_label(struct cm_lap_msg *lap_msg)
{
return cpu_to_be32(be32_to_cpu(lap_msg->offset56) >> 12);
}
static inline void cm_lap_set_flow_label(struct cm_lap_msg *lap_msg,
__be32 flow_label)
{
lap_msg->offset56 = cpu_to_be32(
(be32_to_cpu(lap_msg->offset56) & 0x00000FFF) |
(be32_to_cpu(flow_label) << 12));
}
static inline u8 cm_lap_get_traffic_class(struct cm_lap_msg *lap_msg)
{
return (u8) be32_to_cpu(lap_msg->offset56);
}
static inline void cm_lap_set_traffic_class(struct cm_lap_msg *lap_msg,
u8 traffic_class)
{
lap_msg->offset56 = cpu_to_be32(traffic_class |
(be32_to_cpu(lap_msg->offset56) &
0xFFFFFF00));
}
static inline u8 cm_lap_get_packet_rate(struct cm_lap_msg *lap_msg)
{
return lap_msg->offset61 & 0x3F;
}
static inline void cm_lap_set_packet_rate(struct cm_lap_msg *lap_msg,
u8 packet_rate)
{
lap_msg->offset61 = (packet_rate & 0x3F) | (lap_msg->offset61 & 0xC0);
}
static inline u8 cm_lap_get_sl(struct cm_lap_msg *lap_msg)
{
return lap_msg->offset62 >> 4;
}
static inline void cm_lap_set_sl(struct cm_lap_msg *lap_msg, u8 sl)
{
lap_msg->offset62 = (sl << 4) | (lap_msg->offset62 & 0x0F);
}
static inline u8 cm_lap_get_subnet_local(struct cm_lap_msg *lap_msg)
{
return (lap_msg->offset62 >> 3) & 0x1;
}
static inline void cm_lap_set_subnet_local(struct cm_lap_msg *lap_msg,
u8 subnet_local)
{
lap_msg->offset62 = ((subnet_local & 0x1) << 3) |
(lap_msg->offset61 & 0xF7);
}
static inline u8 cm_lap_get_local_ack_timeout(struct cm_lap_msg *lap_msg)
{
return lap_msg->offset63 >> 3;
}
static inline void cm_lap_set_local_ack_timeout(struct cm_lap_msg *lap_msg,
u8 local_ack_timeout)
{
lap_msg->offset63 = (local_ack_timeout << 3) |
(lap_msg->offset63 & 0x07);
}
struct cm_apr_msg {
struct ib_mad_hdr hdr;
__be32 local_comm_id;
__be32 remote_comm_id;
u8 info_length;
u8 ap_status;
__be16 rsvd;
u8 info[IB_CM_APR_INFO_LENGTH];
u8 private_data[IB_CM_APR_PRIVATE_DATA_SIZE];
} __attribute__ ((packed));
struct cm_sidr_req_msg {
struct ib_mad_hdr hdr;
__be32 request_id;
__be16 pkey;
__be16 rsvd;
__be64 service_id;
u32 private_data[IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE / sizeof(u32)];
} __attribute__ ((packed));
struct cm_sidr_rep_msg {
struct ib_mad_hdr hdr;
__be32 request_id;
u8 status;
u8 info_length;
__be16 rsvd;
/* QPN:24, rsvd:8 */
__be32 offset8;
__be64 service_id;
__be32 qkey;
u8 info[IB_CM_SIDR_REP_INFO_LENGTH];
u8 private_data[IB_CM_SIDR_REP_PRIVATE_DATA_SIZE];
} __attribute__ ((packed));
static inline __be32 cm_sidr_rep_get_qpn(struct cm_sidr_rep_msg *sidr_rep_msg)
{
return cpu_to_be32(be32_to_cpu(sidr_rep_msg->offset8) >> 8);
}
static inline void cm_sidr_rep_set_qpn(struct cm_sidr_rep_msg *sidr_rep_msg,
__be32 qpn)
{
sidr_rep_msg->offset8 = cpu_to_be32((be32_to_cpu(qpn) << 8) |
(be32_to_cpu(sidr_rep_msg->offset8) &
0x000000FF));
}
#endif /* CM_MSGS_H */
File diff suppressed because it is too large. Load diff
+363
View File
@@ -0,0 +1,363 @@
/*
* Copyright (c) 2015, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/module.h>
#include <linux/configfs.h>
#include <rdma/ib_verbs.h>
#include "core_priv.h"
struct cma_device;
struct cma_dev_group;
struct cma_dev_port_group {
unsigned int port_num;
struct cma_dev_group *cma_dev_group;
struct config_group group;
};
struct cma_dev_group {
char name[IB_DEVICE_NAME_MAX];
struct config_group device_group;
struct config_group ports_group;
struct cma_dev_port_group *ports;
};
static struct cma_dev_port_group *to_dev_port_group(struct config_item *item)
{
struct config_group *group;
if (!item)
return NULL;
group = container_of(item, struct config_group, cg_item);
return container_of(group, struct cma_dev_port_group, group);
}
static bool filter_by_name(struct ib_device *ib_dev, void *cookie)
{
return !strcmp(ib_dev->name, cookie);
}
static int cma_configfs_params_get(struct config_item *item,
struct cma_device **pcma_dev,
struct cma_dev_port_group **pgroup)
{
struct cma_dev_port_group *group = to_dev_port_group(item);
struct cma_device *cma_dev;
if (!group)
return -ENODEV;
cma_dev = cma_enum_devices_by_ibdev(filter_by_name,
group->cma_dev_group->name);
if (!cma_dev)
return -ENODEV;
*pcma_dev = cma_dev;
*pgroup = group;
return 0;
}
static void cma_configfs_params_put(struct cma_device *cma_dev)
{
cma_deref_dev(cma_dev);
}
static ssize_t default_roce_mode_show(struct config_item *item,
char *buf)
{
struct cma_device *cma_dev;
struct cma_dev_port_group *group;
int gid_type;
ssize_t ret;
ret = cma_configfs_params_get(item, &cma_dev, &group);
if (ret)
return ret;
gid_type = cma_get_default_gid_type(cma_dev, group->port_num);
cma_configfs_params_put(cma_dev);
if (gid_type < 0)
return gid_type;
return sprintf(buf, "%s\n", ib_cache_gid_type_str(gid_type));
}
static ssize_t default_roce_mode_store(struct config_item *item,
const char *buf, size_t count)
{
struct cma_device *cma_dev;
struct cma_dev_port_group *group;
int gid_type = ib_cache_gid_parse_type_str(buf);
ssize_t ret;
if (gid_type < 0)
return -EINVAL;
ret = cma_configfs_params_get(item, &cma_dev, &group);
if (ret)
return ret;
ret = cma_set_default_gid_type(cma_dev, group->port_num, gid_type);
cma_configfs_params_put(cma_dev);
return !ret ? strnlen(buf, count) : ret;
}
CONFIGFS_ATTR(, default_roce_mode);
static ssize_t default_roce_tos_show(struct config_item *item, char *buf)
{
struct cma_device *cma_dev;
struct cma_dev_port_group *group;
ssize_t ret;
u8 tos;
ret = cma_configfs_params_get(item, &cma_dev, &group);
if (ret)
return ret;
tos = cma_get_default_roce_tos(cma_dev, group->port_num);
cma_configfs_params_put(cma_dev);
return sprintf(buf, "%u\n", tos);
}
static ssize_t default_roce_tos_store(struct config_item *item,
const char *buf, size_t count)
{
struct cma_device *cma_dev;
struct cma_dev_port_group *group;
ssize_t ret;
u8 tos;
ret = kstrtou8(buf, 0, &tos);
if (ret)
return ret;
ret = cma_configfs_params_get(item, &cma_dev, &group);
if (ret)
return ret;
ret = cma_set_default_roce_tos(cma_dev, group->port_num, tos);
cma_configfs_params_put(cma_dev);
return ret ? ret : strnlen(buf, count);
}
CONFIGFS_ATTR(, default_roce_tos);
static struct configfs_attribute *cma_configfs_attributes[] = {
&attr_default_roce_mode,
&attr_default_roce_tos,
NULL,
};
static const struct config_item_type cma_port_group_type = {
.ct_attrs = cma_configfs_attributes,
.ct_owner = THIS_MODULE
};
static int make_cma_ports(struct cma_dev_group *cma_dev_group,
struct cma_device *cma_dev)
{
struct ib_device *ibdev;
unsigned int i;
unsigned int ports_num;
struct cma_dev_port_group *ports;
int err;
ibdev = cma_get_ib_dev(cma_dev);
if (!ibdev)
return -ENODEV;
ports_num = ibdev->phys_port_cnt;
ports = kcalloc(ports_num, sizeof(*cma_dev_group->ports),
GFP_KERNEL);
if (!ports) {
err = -ENOMEM;
goto free;
}
for (i = 0; i < ports_num; i++) {
char port_str[10];
ports[i].port_num = i + 1;
snprintf(port_str, sizeof(port_str), "%u", i + 1);
ports[i].cma_dev_group = cma_dev_group;
config_group_init_type_name(&ports[i].group,
port_str,
&cma_port_group_type);
configfs_add_default_group(&ports[i].group,
&cma_dev_group->ports_group);
}
cma_dev_group->ports = ports;
return 0;
free:
kfree(ports);
cma_dev_group->ports = NULL;
return err;
}
static void release_cma_dev(struct config_item *item)
{
struct config_group *group = container_of(item, struct config_group,
cg_item);
struct cma_dev_group *cma_dev_group = container_of(group,
struct cma_dev_group,
device_group);
kfree(cma_dev_group);
};
static void release_cma_ports_group(struct config_item *item)
{
struct config_group *group = container_of(item, struct config_group,
cg_item);
struct cma_dev_group *cma_dev_group = container_of(group,
struct cma_dev_group,
ports_group);
kfree(cma_dev_group->ports);
cma_dev_group->ports = NULL;
};
static struct configfs_item_operations cma_ports_item_ops = {
.release = release_cma_ports_group
};
static const struct config_item_type cma_ports_group_type = {
.ct_item_ops = &cma_ports_item_ops,
.ct_owner = THIS_MODULE
};
static struct configfs_item_operations cma_device_item_ops = {
.release = release_cma_dev
};
static const struct config_item_type cma_device_group_type = {
.ct_item_ops = &cma_device_item_ops,
.ct_owner = THIS_MODULE
};
static struct config_group *make_cma_dev(struct config_group *group,
const char *name)
{
int err = -ENODEV;
struct cma_device *cma_dev = cma_enum_devices_by_ibdev(filter_by_name,
(void *)name);
struct cma_dev_group *cma_dev_group = NULL;
if (!cma_dev)
goto fail;
cma_dev_group = kzalloc(sizeof(*cma_dev_group), GFP_KERNEL);
if (!cma_dev_group) {
err = -ENOMEM;
goto fail;
}
strlcpy(cma_dev_group->name, name, sizeof(cma_dev_group->name));
config_group_init_type_name(&cma_dev_group->ports_group, "ports",
&cma_ports_group_type);
err = make_cma_ports(cma_dev_group, cma_dev);
if (err)
goto fail;
config_group_init_type_name(&cma_dev_group->device_group, name,
&cma_device_group_type);
configfs_add_default_group(&cma_dev_group->ports_group,
&cma_dev_group->device_group);
cma_deref_dev(cma_dev);
return &cma_dev_group->device_group;
fail:
if (cma_dev)
cma_deref_dev(cma_dev);
kfree(cma_dev_group);
return ERR_PTR(err);
}
static void drop_cma_dev(struct config_group *cgroup, struct config_item *item)
{
struct config_group *group =
container_of(item, struct config_group, cg_item);
struct cma_dev_group *cma_dev_group =
container_of(group, struct cma_dev_group, device_group);
configfs_remove_default_groups(&cma_dev_group->ports_group);
configfs_remove_default_groups(&cma_dev_group->device_group);
config_item_put(item);
}
static struct configfs_group_operations cma_subsys_group_ops = {
.make_group = make_cma_dev,
.drop_item = drop_cma_dev,
};
static const struct config_item_type cma_subsys_type = {
.ct_group_ops = &cma_subsys_group_ops,
.ct_owner = THIS_MODULE,
};
static struct configfs_subsystem cma_subsys = {
.su_group = {
.cg_item = {
.ci_namebuf = "rdma_cm",
.ci_type = &cma_subsys_type,
},
},
};
int __init cma_configfs_init(void)
{
config_group_init(&cma_subsys.su_group);
mutex_init(&cma_subsys.su_mutex);
return configfs_register_subsystem(&cma_subsys);
}
void __exit cma_configfs_exit(void)
{
configfs_unregister_subsystem(&cma_subsys);
}
+97
View File
@@ -0,0 +1,97 @@
/*
* Copyright (c) 2005 Voltaire Inc. All rights reserved.
* Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
* Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
* Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef _CMA_PRIV_H
#define _CMA_PRIV_H
enum rdma_cm_state {
RDMA_CM_IDLE,
RDMA_CM_ADDR_QUERY,
RDMA_CM_ADDR_RESOLVED,
RDMA_CM_ROUTE_QUERY,
RDMA_CM_ROUTE_RESOLVED,
RDMA_CM_CONNECT,
RDMA_CM_DISCONNECT,
RDMA_CM_ADDR_BOUND,
RDMA_CM_LISTEN,
RDMA_CM_DEVICE_REMOVAL,
RDMA_CM_DESTROYING
};
struct rdma_id_private {
struct rdma_cm_id id;
struct rdma_bind_list *bind_list;
struct hlist_node node;
struct list_head list; /* listen_any_list or cma_device.list */
struct list_head listen_list; /* per device listens */
struct cma_device *cma_dev;
struct list_head mc_list;
int internal_id;
enum rdma_cm_state state;
spinlock_t lock;
struct mutex qp_mutex;
struct completion comp;
atomic_t refcount;
struct mutex handler_mutex;
int backlog;
int timeout_ms;
struct ib_sa_query *query;
int query_id;
union {
struct ib_cm_id *ib;
struct iw_cm_id *iw;
} cm_id;
u32 seq_num;
u32 qkey;
u32 qp_num;
u32 options;
u8 srq;
u8 tos;
bool tos_set;
u8 reuseaddr;
u8 afonly;
enum ib_gid_type gid_type;
/*
* Internal to RDMA/core, don't use in the drivers
*/
struct rdma_restrack_entry res;
};
#endif /* _CMA_PRIV_H */
+344
View File
@@ -0,0 +1,344 @@
/*
* Copyright (c) 2004 Topspin Communications. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef _CORE_PRIV_H
#define _CORE_PRIV_H
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/cgroup_rdma.h>
#include <rdma/ib_verbs.h>
#include <rdma/opa_addr.h>
#include <rdma/ib_mad.h>
#include <rdma/restrack.h>
#include "mad_priv.h"
/* Total number of ports combined across all struct ib_devices's */
#define RDMA_MAX_PORTS 1024
struct pkey_index_qp_list {
struct list_head pkey_index_list;
u16 pkey_index;
/* Lock to hold while iterating the qp_list. */
spinlock_t qp_list_lock;
struct list_head qp_list;
};
#if IS_ENABLED(CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS)
int cma_configfs_init(void);
void cma_configfs_exit(void);
#else
static inline int cma_configfs_init(void)
{
return 0;
}
static inline void cma_configfs_exit(void)
{
}
#endif
struct cma_device;
void cma_ref_dev(struct cma_device *cma_dev);
void cma_deref_dev(struct cma_device *cma_dev);
typedef bool (*cma_device_filter)(struct ib_device *, void *);
struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter filter,
void *cookie);
int cma_get_default_gid_type(struct cma_device *cma_dev,
unsigned int port);
int cma_set_default_gid_type(struct cma_device *cma_dev,
unsigned int port,
enum ib_gid_type default_gid_type);
int cma_get_default_roce_tos(struct cma_device *cma_dev, unsigned int port);
int cma_set_default_roce_tos(struct cma_device *a_dev, unsigned int port,
u8 default_roce_tos);
struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev);
int ib_device_register_sysfs(struct ib_device *device,
int (*port_callback)(struct ib_device *,
u8, struct kobject *));
void ib_device_unregister_sysfs(struct ib_device *device);
typedef void (*roce_netdev_callback)(struct ib_device *device, u8 port,
struct net_device *idev, void *cookie);
typedef bool (*roce_netdev_filter)(struct ib_device *device, u8 port,
struct net_device *idev, void *cookie);
void ib_enum_roce_netdev(struct ib_device *ib_dev,
roce_netdev_filter filter,
void *filter_cookie,
roce_netdev_callback cb,
void *cookie);
void ib_enum_all_roce_netdevs(roce_netdev_filter filter,
void *filter_cookie,
roce_netdev_callback cb,
void *cookie);
typedef int (*nldev_callback)(struct ib_device *device,
struct sk_buff *skb,
struct netlink_callback *cb,
unsigned int idx);
int ib_enum_all_devs(nldev_callback nldev_cb, struct sk_buff *skb,
struct netlink_callback *cb);
enum ib_cache_gid_default_mode {
IB_CACHE_GID_DEFAULT_MODE_SET,
IB_CACHE_GID_DEFAULT_MODE_DELETE
};
int ib_cache_gid_parse_type_str(const char *buf);
const char *ib_cache_gid_type_str(enum ib_gid_type gid_type);
void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
struct net_device *ndev,
unsigned long gid_type_mask,
enum ib_cache_gid_default_mode mode);
int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
union ib_gid *gid, struct ib_gid_attr *attr);
int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
union ib_gid *gid, struct ib_gid_attr *attr);
int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,
struct net_device *ndev);
int roce_gid_mgmt_init(void);
void roce_gid_mgmt_cleanup(void);
unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port);
int ib_cache_setup_one(struct ib_device *device);
void ib_cache_cleanup_one(struct ib_device *device);
void ib_cache_release_one(struct ib_device *device);
#ifdef CONFIG_CGROUP_RDMA
int ib_device_register_rdmacg(struct ib_device *device);
void ib_device_unregister_rdmacg(struct ib_device *device);
int ib_rdmacg_try_charge(struct ib_rdmacg_object *cg_obj,
struct ib_device *device,
enum rdmacg_resource_type resource_index);
void ib_rdmacg_uncharge(struct ib_rdmacg_object *cg_obj,
struct ib_device *device,
enum rdmacg_resource_type resource_index);
#else
static inline int ib_device_register_rdmacg(struct ib_device *device)
{ return 0; }
static inline void ib_device_unregister_rdmacg(struct ib_device *device)
{ }
static inline int ib_rdmacg_try_charge(struct ib_rdmacg_object *cg_obj,
struct ib_device *device,
enum rdmacg_resource_type resource_index)
{ return 0; }
static inline void ib_rdmacg_uncharge(struct ib_rdmacg_object *cg_obj,
struct ib_device *device,
enum rdmacg_resource_type resource_index)
{ }
#endif
static inline bool rdma_is_upper_dev_rcu(struct net_device *dev,
struct net_device *upper)
{
return netdev_has_upper_dev_all_rcu(dev, upper);
}
int addr_init(void);
void addr_cleanup(void);
int ib_mad_init(void);
void ib_mad_cleanup(void);
int ib_sa_init(void);
void ib_sa_cleanup(void);
int rdma_nl_init(void);
void rdma_nl_exit(void);
int ib_nl_handle_resolve_resp(struct sk_buff *skb,
struct nlmsghdr *nlh,
struct netlink_ext_ack *extack);
int ib_nl_handle_set_timeout(struct sk_buff *skb,
struct nlmsghdr *nlh,
struct netlink_ext_ack *extack);
int ib_nl_handle_ip_res_resp(struct sk_buff *skb,
struct nlmsghdr *nlh,
struct netlink_ext_ack *extack);
int ib_get_cached_subnet_prefix(struct ib_device *device,
u8 port_num,
u64 *sn_pfx);
#ifdef CONFIG_SECURITY_INFINIBAND
void ib_security_destroy_port_pkey_list(struct ib_device *device);
void ib_security_cache_change(struct ib_device *device,
u8 port_num,
u64 subnet_prefix);
int ib_security_modify_qp(struct ib_qp *qp,
struct ib_qp_attr *qp_attr,
int qp_attr_mask,
struct ib_udata *udata);
int ib_create_qp_security(struct ib_qp *qp, struct ib_device *dev);
void ib_destroy_qp_security_begin(struct ib_qp_security *sec);
void ib_destroy_qp_security_abort(struct ib_qp_security *sec);
void ib_destroy_qp_security_end(struct ib_qp_security *sec);
int ib_open_shared_qp_security(struct ib_qp *qp, struct ib_device *dev);
void ib_close_shared_qp_security(struct ib_qp_security *sec);
int ib_mad_agent_security_setup(struct ib_mad_agent *agent,
enum ib_qp_type qp_type);
void ib_mad_agent_security_cleanup(struct ib_mad_agent *agent);
int ib_mad_enforce_security(struct ib_mad_agent_private *map, u16 pkey_index);
#else
static inline void ib_security_destroy_port_pkey_list(struct ib_device *device)
{
}
static inline void ib_security_cache_change(struct ib_device *device,
u8 port_num,
u64 subnet_prefix)
{
}
static inline int ib_security_modify_qp(struct ib_qp *qp,
struct ib_qp_attr *qp_attr,
int qp_attr_mask,
struct ib_udata *udata)
{
return qp->device->modify_qp(qp->real_qp,
qp_attr,
qp_attr_mask,
udata);
}
static inline int ib_create_qp_security(struct ib_qp *qp,
struct ib_device *dev)
{
return 0;
}
static inline void ib_destroy_qp_security_begin(struct ib_qp_security *sec)
{
}
static inline void ib_destroy_qp_security_abort(struct ib_qp_security *sec)
{
}
static inline void ib_destroy_qp_security_end(struct ib_qp_security *sec)
{
}
static inline int ib_open_shared_qp_security(struct ib_qp *qp,
struct ib_device *dev)
{
return 0;
}
static inline void ib_close_shared_qp_security(struct ib_qp_security *sec)
{
}
static inline int ib_mad_agent_security_setup(struct ib_mad_agent *agent,
enum ib_qp_type qp_type)
{
return 0;
}
static inline void ib_mad_agent_security_cleanup(struct ib_mad_agent *agent)
{
}
static inline int ib_mad_enforce_security(struct ib_mad_agent_private *map,
u16 pkey_index)
{
return 0;
}
#endif
struct ib_device *ib_device_get_by_index(u32 ifindex);
/* RDMA device netlink */
void nldev_init(void);
void nldev_exit(void);
static inline struct ib_qp *_ib_create_qp(struct ib_device *dev,
struct ib_pd *pd,
struct ib_qp_init_attr *attr,
struct ib_udata *udata,
struct ib_uobject *uobj)
{
struct ib_qp *qp;
if (!dev->create_qp)
return ERR_PTR(-EOPNOTSUPP);
qp = dev->create_qp(pd, attr, udata);
if (IS_ERR(qp))
return qp;
qp->device = dev;
qp->pd = pd;
qp->uobject = uobj;
/*
* We don't track XRC QPs for now, because they don't have PD
* and more importantly they are created internaly by driver,
* see mlx5 create_dev_resources() as an example.
*/
if (attr->qp_type < IB_QPT_XRC_INI) {
qp->res.type = RDMA_RESTRACK_QP;
rdma_restrack_add(&qp->res);
} else
qp->res.valid = false;
return qp;
}
struct rdma_dev_addr;
int rdma_resolve_ip_route(struct sockaddr *src_addr,
const struct sockaddr *dst_addr,
struct rdma_dev_addr *addr);
int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
const union ib_gid *dgid,
u8 *dmac, const struct net_device *ndev,
int *hoplimit);
#endif /* _CORE_PRIV_H */
+231
View File
@@ -0,0 +1,231 @@
/*
* Copyright (c) 2015 HGST, a Western Digital Company.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include <linux/module.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <rdma/ib_verbs.h>
/* # of WCs to poll for with a single call to ib_poll_cq */
#define IB_POLL_BATCH 16
#define IB_POLL_BATCH_DIRECT 8
/* # of WCs to iterate over before yielding */
#define IB_POLL_BUDGET_IRQ 256
#define IB_POLL_BUDGET_WORKQUEUE 65536
#define IB_POLL_FLAGS \
(IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS)
static int __ib_process_cq(struct ib_cq *cq, int budget, struct ib_wc *wcs,
int batch)
{
int i, n, completed = 0;
/*
* budget might be (-1) if the caller does not
* want to bound this call, thus we need unsigned
* minimum here.
*/
while ((n = ib_poll_cq(cq, min_t(u32, batch,
budget - completed), wcs)) > 0) {
for (i = 0; i < n; i++) {
struct ib_wc *wc = &wcs[i];
if (wc->wr_cqe)
wc->wr_cqe->done(cq, wc);
else
WARN_ON_ONCE(wc->status == IB_WC_SUCCESS);
}
completed += n;
if (n != batch || (budget != -1 && completed >= budget))
break;
}
return completed;
}
/**
* ib_process_direct_cq - process a CQ in caller context
* @cq: CQ to process
* @budget: number of CQEs to poll for
*
* This function is used to process all outstanding CQ entries.
* It does not offload CQ processing to a different context and does
* not ask for completion interrupts from the HCA.
* Using direct processing on CQ with non IB_POLL_DIRECT type may trigger
* concurrent processing.
*
* Note: do not pass -1 as %budget unless it is guaranteed that the number
* of completions that will be processed is small.
*/
int ib_process_cq_direct(struct ib_cq *cq, int budget)
{
struct ib_wc wcs[IB_POLL_BATCH_DIRECT];
return __ib_process_cq(cq, budget, wcs, IB_POLL_BATCH_DIRECT);
}
EXPORT_SYMBOL(ib_process_cq_direct);
static void ib_cq_completion_direct(struct ib_cq *cq, void *private)
{
WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq);
}
static int ib_poll_handler(struct irq_poll *iop, int budget)
{
struct ib_cq *cq = container_of(iop, struct ib_cq, iop);
int completed;
completed = __ib_process_cq(cq, budget, cq->wc, IB_POLL_BATCH);
if (completed < budget) {
irq_poll_complete(&cq->iop);
if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
irq_poll_sched(&cq->iop);
}
return completed;
}
static void ib_cq_completion_softirq(struct ib_cq *cq, void *private)
{
irq_poll_sched(&cq->iop);
}
static void ib_cq_poll_work(struct work_struct *work)
{
struct ib_cq *cq = container_of(work, struct ib_cq, work);
int completed;
completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE, cq->wc,
IB_POLL_BATCH);
if (completed >= IB_POLL_BUDGET_WORKQUEUE ||
ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0)
queue_work(cq->comp_wq, &cq->work);
}
static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
{
queue_work(cq->comp_wq, &cq->work);
}
/**
* __ib_alloc_cq - allocate a completion queue
* @dev: device to allocate the CQ for
* @private: driver private data, accessible from cq->cq_context
* @nr_cqe: number of CQEs to allocate
* @comp_vector: HCA completion vectors for this CQ
* @poll_ctx: context to poll the CQ from.
* @caller: module owner name.
*
* This is the proper interface to allocate a CQ for in-kernel users. A
* CQ allocated with this interface will automatically be polled from the
* specified context. The ULP must use wr->wr_cqe instead of wr->wr_id
* to use this CQ abstraction.
*/
struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private,
int nr_cqe, int comp_vector,
enum ib_poll_context poll_ctx, const char *caller)
{
struct ib_cq_init_attr cq_attr = {
.cqe = nr_cqe,
.comp_vector = comp_vector,
};
struct ib_cq *cq;
int ret = -ENOMEM;
cq = dev->create_cq(dev, &cq_attr, NULL, NULL);
if (IS_ERR(cq))
return cq;
cq->device = dev;
cq->uobject = NULL;
cq->event_handler = NULL;
cq->cq_context = private;
cq->poll_ctx = poll_ctx;
atomic_set(&cq->usecnt, 0);
cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL);
if (!cq->wc)
goto out_destroy_cq;
cq->res.type = RDMA_RESTRACK_CQ;
cq->res.kern_name = caller;
rdma_restrack_add(&cq->res);
switch (cq->poll_ctx) {
case IB_POLL_DIRECT:
cq->comp_handler = ib_cq_completion_direct;
break;
case IB_POLL_SOFTIRQ:
cq->comp_handler = ib_cq_completion_softirq;
irq_poll_init(&cq->iop, IB_POLL_BUDGET_IRQ, ib_poll_handler);
ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
break;
case IB_POLL_WORKQUEUE:
case IB_POLL_UNBOUND_WORKQUEUE:
cq->comp_handler = ib_cq_completion_workqueue;
INIT_WORK(&cq->work, ib_cq_poll_work);
ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
cq->comp_wq = (cq->poll_ctx == IB_POLL_WORKQUEUE) ?
ib_comp_wq : ib_comp_unbound_wq;
break;
default:
ret = -EINVAL;
goto out_free_wc;
}
return cq;
out_free_wc:
kfree(cq->wc);
rdma_restrack_del(&cq->res);
out_destroy_cq:
cq->device->destroy_cq(cq);
return ERR_PTR(ret);
}
EXPORT_SYMBOL(__ib_alloc_cq);
/**
* ib_free_cq - free a completion queue
* @cq: completion queue to free.
*/
void ib_free_cq(struct ib_cq *cq)
{
int ret;
if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
return;
switch (cq->poll_ctx) {
case IB_POLL_DIRECT:
break;
case IB_POLL_SOFTIRQ:
irq_poll_disable(&cq->iop);
break;
case IB_POLL_WORKQUEUE:
case IB_POLL_UNBOUND_WORKQUEUE:
cancel_work_sync(&cq->work);
break;
default:
WARN_ON_ONCE(1);
}
kfree(cq->wc);
rdma_restrack_del(&cq->res);
ret = cq->device->destroy_cq(cq);
WARN_ON_ONCE(ret);
}
EXPORT_SYMBOL(ib_free_cq);
File diff suppressed because it is too large. Load diff
+508
View File
@@ -0,0 +1,508 @@
/*
* Copyright (c) 2004 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/errno.h>
#include <linux/spinlock.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/jhash.h>
#include <linux/kthread.h>
#include <rdma/ib_fmr_pool.h>
#include "core_priv.h"
#define PFX "fmr_pool: "
enum {
IB_FMR_MAX_REMAPS = 32,
IB_FMR_HASH_BITS = 8,
IB_FMR_HASH_SIZE = 1 << IB_FMR_HASH_BITS,
IB_FMR_HASH_MASK = IB_FMR_HASH_SIZE - 1
};
/*
* If an FMR is not in use, then the list member will point to either
* its pool's free_list (if the FMR can be mapped again; that is,
* remap_count < pool->max_remaps) or its pool's dirty_list (if the
* FMR needs to be unmapped before being remapped). In either of
* these cases it is a bug if the ref_count is not 0. In other words,
* if ref_count is > 0, then the list member must not be linked into
* either free_list or dirty_list.
*
* The cache_node member is used to link the FMR into a cache bucket
* (if caching is enabled). This is independent of the reference
* count of the FMR. When a valid FMR is released, its ref_count is
* decremented, and if ref_count reaches 0, the FMR is placed in
* either free_list or dirty_list as appropriate. However, it is not
* removed from the cache and may be "revived" if a call to
* ib_fmr_register_physical() occurs before the FMR is remapped. In
* this case we just increment the ref_count and remove the FMR from
* free_list/dirty_list.
*
* Before we remap an FMR from free_list, we remove it from the cache
* (to prevent another user from obtaining a stale FMR). When an FMR
* is released, we add it to the tail of the free list, so that our
* cache eviction policy is "least recently used."
*
* All manipulation of ref_count, list and cache_node is protected by
* pool_lock to maintain consistency.
*/
struct ib_fmr_pool {
spinlock_t pool_lock;
int pool_size;
int max_pages;
int max_remaps;
int dirty_watermark;
int dirty_len;
struct list_head free_list;
struct list_head dirty_list;
struct hlist_head *cache_bucket;
void (*flush_function)(struct ib_fmr_pool *pool,
void * arg);
void *flush_arg;
struct kthread_worker *worker;
struct kthread_work work;
atomic_t req_ser;
atomic_t flush_ser;
wait_queue_head_t force_wait;
};
static inline u32 ib_fmr_hash(u64 first_page)
{
return jhash_2words((u32) first_page, (u32) (first_page >> 32), 0) &
(IB_FMR_HASH_SIZE - 1);
}
/* Caller must hold pool_lock */
static inline struct ib_pool_fmr *ib_fmr_cache_lookup(struct ib_fmr_pool *pool,
u64 *page_list,
int page_list_len,
u64 io_virtual_address)
{
struct hlist_head *bucket;
struct ib_pool_fmr *fmr;
if (!pool->cache_bucket)
return NULL;
bucket = pool->cache_bucket + ib_fmr_hash(*page_list);
hlist_for_each_entry(fmr, bucket, cache_node)
if (io_virtual_address == fmr->io_virtual_address &&
page_list_len == fmr->page_list_len &&
!memcmp(page_list, fmr->page_list,
page_list_len * sizeof *page_list))
return fmr;
return NULL;
}
static void ib_fmr_batch_release(struct ib_fmr_pool *pool)
{
int ret;
struct ib_pool_fmr *fmr;
LIST_HEAD(unmap_list);
LIST_HEAD(fmr_list);
spin_lock_irq(&pool->pool_lock);
list_for_each_entry(fmr, &pool->dirty_list, list) {
hlist_del_init(&fmr->cache_node);
fmr->remap_count = 0;
list_add_tail(&fmr->fmr->list, &fmr_list);
#ifdef DEBUG
if (fmr->ref_count !=0) {
pr_warn(PFX "Unmapping FMR 0x%08x with ref count %d\n",
fmr, fmr->ref_count);
}
#endif
}
list_splice_init(&pool->dirty_list, &unmap_list);
pool->dirty_len = 0;
spin_unlock_irq(&pool->pool_lock);
if (list_empty(&unmap_list)) {
return;
}
ret = ib_unmap_fmr(&fmr_list);
if (ret)
pr_warn(PFX "ib_unmap_fmr returned %d\n", ret);
spin_lock_irq(&pool->pool_lock);
list_splice(&unmap_list, &pool->free_list);
spin_unlock_irq(&pool->pool_lock);
}
static void ib_fmr_cleanup_func(struct kthread_work *work)
{
struct ib_fmr_pool *pool = container_of(work, struct ib_fmr_pool, work);
ib_fmr_batch_release(pool);
atomic_inc(&pool->flush_ser);
wake_up_interruptible(&pool->force_wait);
if (pool->flush_function)
pool->flush_function(pool, pool->flush_arg);
if (atomic_read(&pool->flush_ser) - atomic_read(&pool->req_ser) < 0)
kthread_queue_work(pool->worker, &pool->work);
}
/**
* ib_create_fmr_pool - Create an FMR pool
* @pd:Protection domain for FMRs
* @params:FMR pool parameters
*
* Create a pool of FMRs. Return value is pointer to new pool or
* error code if creation failed.
*/
struct ib_fmr_pool *ib_create_fmr_pool(struct ib_pd *pd,
struct ib_fmr_pool_param *params)
{
struct ib_device *device;
struct ib_fmr_pool *pool;
int i;
int ret;
int max_remaps;
if (!params)
return ERR_PTR(-EINVAL);
device = pd->device;
if (!device->alloc_fmr || !device->dealloc_fmr ||
!device->map_phys_fmr || !device->unmap_fmr) {
pr_info(PFX "Device %s does not support FMRs\n", device->name);
return ERR_PTR(-ENOSYS);
}
if (!device->attrs.max_map_per_fmr)
max_remaps = IB_FMR_MAX_REMAPS;
else
max_remaps = device->attrs.max_map_per_fmr;
pool = kmalloc(sizeof *pool, GFP_KERNEL);
if (!pool)
return ERR_PTR(-ENOMEM);
pool->cache_bucket = NULL;
pool->flush_function = params->flush_function;
pool->flush_arg = params->flush_arg;
INIT_LIST_HEAD(&pool->free_list);
INIT_LIST_HEAD(&pool->dirty_list);
if (params->cache) {
pool->cache_bucket =
kmalloc_array(IB_FMR_HASH_SIZE,
sizeof(*pool->cache_bucket),
GFP_KERNEL);
if (!pool->cache_bucket) {
ret = -ENOMEM;
goto out_free_pool;
}
for (i = 0; i < IB_FMR_HASH_SIZE; ++i)
INIT_HLIST_HEAD(pool->cache_bucket + i);
}
pool->pool_size = 0;
pool->max_pages = params->max_pages_per_fmr;
pool->max_remaps = max_remaps;
pool->dirty_watermark = params->dirty_watermark;
pool->dirty_len = 0;
spin_lock_init(&pool->pool_lock);
atomic_set(&pool->req_ser, 0);
atomic_set(&pool->flush_ser, 0);
init_waitqueue_head(&pool->force_wait);
pool->worker = kthread_create_worker(0, "ib_fmr(%s)", device->name);
if (IS_ERR(pool->worker)) {
pr_warn(PFX "couldn't start cleanup kthread worker\n");
ret = PTR_ERR(pool->worker);
goto out_free_pool;
}
kthread_init_work(&pool->work, ib_fmr_cleanup_func);
{
struct ib_pool_fmr *fmr;
struct ib_fmr_attr fmr_attr = {
.max_pages = params->max_pages_per_fmr,
.max_maps = pool->max_remaps,
.page_shift = params->page_shift
};
int bytes_per_fmr = sizeof *fmr;
if (pool->cache_bucket)
bytes_per_fmr += params->max_pages_per_fmr * sizeof (u64);
for (i = 0; i < params->pool_size; ++i) {
fmr = kmalloc(bytes_per_fmr, GFP_KERNEL);
if (!fmr)
goto out_fail;
fmr->pool = pool;
fmr->remap_count = 0;
fmr->ref_count = 0;
INIT_HLIST_NODE(&fmr->cache_node);
fmr->fmr = ib_alloc_fmr(pd, params->access, &fmr_attr);
if (IS_ERR(fmr->fmr)) {
pr_warn(PFX "fmr_create failed for FMR %d\n",
i);
kfree(fmr);
goto out_fail;
}
list_add_tail(&fmr->list, &pool->free_list);
++pool->pool_size;
}
}
return pool;
out_free_pool:
kfree(pool->cache_bucket);
kfree(pool);
return ERR_PTR(ret);
out_fail:
ib_destroy_fmr_pool(pool);
return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL(ib_create_fmr_pool);
/**
* ib_destroy_fmr_pool - Free FMR pool
* @pool:FMR pool to free
*
* Destroy an FMR pool and free all associated resources.
*/
void ib_destroy_fmr_pool(struct ib_fmr_pool *pool)
{
struct ib_pool_fmr *fmr;
struct ib_pool_fmr *tmp;
LIST_HEAD(fmr_list);
int i;
kthread_destroy_worker(pool->worker);
ib_fmr_batch_release(pool);
i = 0;
list_for_each_entry_safe(fmr, tmp, &pool->free_list, list) {
if (fmr->remap_count) {
INIT_LIST_HEAD(&fmr_list);
list_add_tail(&fmr->fmr->list, &fmr_list);
ib_unmap_fmr(&fmr_list);
}
ib_dealloc_fmr(fmr->fmr);
list_del(&fmr->list);
kfree(fmr);
++i;
}
if (i < pool->pool_size)
pr_warn(PFX "pool still has %d regions registered\n",
pool->pool_size - i);
kfree(pool->cache_bucket);
kfree(pool);
}
EXPORT_SYMBOL(ib_destroy_fmr_pool);
/**
* ib_flush_fmr_pool - Invalidate all unmapped FMRs
* @pool:FMR pool to flush
*
* Ensure that all unmapped FMRs are fully invalidated.
*/
int ib_flush_fmr_pool(struct ib_fmr_pool *pool)
{
int serial;
struct ib_pool_fmr *fmr, *next;
/*
* The free_list holds FMRs that may have been used
* but have not been remapped enough times to be dirty.
* Put them on the dirty list now so that the cleanup
* thread will reap them too.
*/
spin_lock_irq(&pool->pool_lock);
list_for_each_entry_safe(fmr, next, &pool->free_list, list) {
if (fmr->remap_count > 0)
list_move(&fmr->list, &pool->dirty_list);
}
spin_unlock_irq(&pool->pool_lock);
serial = atomic_inc_return(&pool->req_ser);
kthread_queue_work(pool->worker, &pool->work);
if (wait_event_interruptible(pool->force_wait,
atomic_read(&pool->flush_ser) - serial >= 0))
return -EINTR;
return 0;
}
EXPORT_SYMBOL(ib_flush_fmr_pool);
/**
* ib_fmr_pool_map_phys - Map an FMR from an FMR pool.
* @pool_handle: FMR pool to allocate FMR from
* @page_list: List of pages to map
* @list_len: Number of pages in @page_list
* @io_virtual_address: I/O virtual address for new FMR
*/
struct ib_pool_fmr *ib_fmr_pool_map_phys(struct ib_fmr_pool *pool_handle,
u64 *page_list,
int list_len,
u64 io_virtual_address)
{
struct ib_fmr_pool *pool = pool_handle;
struct ib_pool_fmr *fmr;
unsigned long flags;
int result;
if (list_len < 1 || list_len > pool->max_pages)
return ERR_PTR(-EINVAL);
spin_lock_irqsave(&pool->pool_lock, flags);
fmr = ib_fmr_cache_lookup(pool,
page_list,
list_len,
io_virtual_address);
if (fmr) {
/* found in cache */
++fmr->ref_count;
if (fmr->ref_count == 1) {
list_del(&fmr->list);
}
spin_unlock_irqrestore(&pool->pool_lock, flags);
return fmr;
}
if (list_empty(&pool->free_list)) {
spin_unlock_irqrestore(&pool->pool_lock, flags);
return ERR_PTR(-EAGAIN);
}
fmr = list_entry(pool->free_list.next, struct ib_pool_fmr, list);
list_del(&fmr->list);
hlist_del_init(&fmr->cache_node);
spin_unlock_irqrestore(&pool->pool_lock, flags);
result = ib_map_phys_fmr(fmr->fmr, page_list, list_len,
io_virtual_address);
if (result) {
spin_lock_irqsave(&pool->pool_lock, flags);
list_add(&fmr->list, &pool->free_list);
spin_unlock_irqrestore(&pool->pool_lock, flags);
pr_warn(PFX "fmr_map returns %d\n", result);
return ERR_PTR(result);
}
++fmr->remap_count;
fmr->ref_count = 1;
if (pool->cache_bucket) {
fmr->io_virtual_address = io_virtual_address;
fmr->page_list_len = list_len;
memcpy(fmr->page_list, page_list, list_len * sizeof(*page_list));
spin_lock_irqsave(&pool->pool_lock, flags);
hlist_add_head(&fmr->cache_node,
pool->cache_bucket + ib_fmr_hash(fmr->page_list[0]));
spin_unlock_irqrestore(&pool->pool_lock, flags);
}
return fmr;
}
EXPORT_SYMBOL(ib_fmr_pool_map_phys);
/**
* ib_fmr_pool_unmap - Unmap FMR
* @fmr:FMR to unmap
*
* Unmap an FMR. The FMR mapping may remain valid until the FMR is
* reused (or until ib_flush_fmr_pool() is called).
*/
int ib_fmr_pool_unmap(struct ib_pool_fmr *fmr)
{
struct ib_fmr_pool *pool;
unsigned long flags;
pool = fmr->pool;
spin_lock_irqsave(&pool->pool_lock, flags);
--fmr->ref_count;
if (!fmr->ref_count) {
if (fmr->remap_count < pool->max_remaps) {
list_add_tail(&fmr->list, &pool->free_list);
} else {
list_add_tail(&fmr->list, &pool->dirty_list);
if (++pool->dirty_len >= pool->dirty_watermark) {
atomic_inc(&pool->req_ser);
kthread_queue_work(pool->worker, &pool->work);
}
}
}
#ifdef DEBUG
if (fmr->ref_count < 0)
pr_warn(PFX "FMR %p has ref count %d < 0\n",
fmr, fmr->ref_count);
#endif
spin_unlock_irqrestore(&pool->pool_lock, flags);
return 0;
}
EXPORT_SYMBOL(ib_fmr_pool_unmap);
File diff suppressed because it is too large. Load diff
+62
View File
@@ -0,0 +1,62 @@
/*
* Copyright (c) 2005 Network Appliance, Inc. All rights reserved.
* Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef IWCM_H
#define IWCM_H
enum iw_cm_state {
IW_CM_STATE_IDLE, /* unbound, inactive */
IW_CM_STATE_LISTEN, /* listen waiting for connect */
IW_CM_STATE_CONN_RECV, /* inbound waiting for user accept */
IW_CM_STATE_CONN_SENT, /* outbound waiting for peer accept */
IW_CM_STATE_ESTABLISHED, /* established */
IW_CM_STATE_CLOSING, /* disconnect */
IW_CM_STATE_DESTROYING /* object being deleted */
};
struct iwcm_id_private {
struct iw_cm_id id;
enum iw_cm_state state;
unsigned long flags;
struct ib_qp *qp;
struct completion destroy_comp;
wait_queue_head_t connect_wait;
struct list_head work_list;
spinlock_t lock;
atomic_t refcount;
struct list_head work_free_list;
};
#define IWCM_F_DROP_EVENTS 1
#define IWCM_F_CONNECT_WAIT 2
#endif /* IWCM_H */
+750
View File
@@ -0,0 +1,750 @@
/*
* Copyright (c) 2014 Intel Corporation. All rights reserved.
* Copyright (c) 2014 Chelsio, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "iwpm_util.h"
static const char iwpm_ulib_name[IWPM_ULIBNAME_SIZE] = "iWarpPortMapperUser";
static int iwpm_ulib_version = 3;
static int iwpm_user_pid = IWPM_PID_UNDEFINED;
static atomic_t echo_nlmsg_seq;
int iwpm_valid_pid(void)
{
return iwpm_user_pid > 0;
}
/*
* iwpm_register_pid - Send a netlink query to user space
* for the iwarp port mapper pid
*
* nlmsg attributes:
* [IWPM_NLA_REG_PID_SEQ]
* [IWPM_NLA_REG_IF_NAME]
* [IWPM_NLA_REG_IBDEV_NAME]
* [IWPM_NLA_REG_ULIB_NAME]
*/
int iwpm_register_pid(struct iwpm_dev_data *pm_msg, u8 nl_client)
{
struct sk_buff *skb = NULL;
struct iwpm_nlmsg_request *nlmsg_request = NULL;
struct nlmsghdr *nlh;
u32 msg_seq;
const char *err_str = "";
int ret = -EINVAL;
if (!iwpm_valid_client(nl_client)) {
err_str = "Invalid port mapper client";
goto pid_query_error;
}
if (iwpm_check_registration(nl_client, IWPM_REG_VALID) ||
iwpm_user_pid == IWPM_PID_UNAVAILABLE)
return 0;
skb = iwpm_create_nlmsg(RDMA_NL_IWPM_REG_PID, &nlh, nl_client);
if (!skb) {
err_str = "Unable to create a nlmsg";
goto pid_query_error;
}
nlh->nlmsg_seq = iwpm_get_nlmsg_seq();
nlmsg_request = iwpm_get_nlmsg_request(nlh->nlmsg_seq, nl_client, GFP_KERNEL);
if (!nlmsg_request) {
err_str = "Unable to allocate netlink request";
goto pid_query_error;
}
msg_seq = atomic_read(&echo_nlmsg_seq);
/* fill in the pid request message */
err_str = "Unable to put attribute of the nlmsg";
ret = ibnl_put_attr(skb, nlh, sizeof(u32), &msg_seq, IWPM_NLA_REG_PID_SEQ);
if (ret)
goto pid_query_error;
ret = ibnl_put_attr(skb, nlh, IFNAMSIZ,
pm_msg->if_name, IWPM_NLA_REG_IF_NAME);
if (ret)
goto pid_query_error;
ret = ibnl_put_attr(skb, nlh, IWPM_DEVNAME_SIZE,
pm_msg->dev_name, IWPM_NLA_REG_IBDEV_NAME);
if (ret)
goto pid_query_error;
ret = ibnl_put_attr(skb, nlh, IWPM_ULIBNAME_SIZE,
(char *)iwpm_ulib_name, IWPM_NLA_REG_ULIB_NAME);
if (ret)
goto pid_query_error;
nlmsg_end(skb, nlh);
pr_debug("%s: Multicasting a nlmsg (dev = %s ifname = %s iwpm = %s)\n",
__func__, pm_msg->dev_name, pm_msg->if_name, iwpm_ulib_name);
ret = rdma_nl_multicast(skb, RDMA_NL_GROUP_IWPM, GFP_KERNEL);
if (ret) {
skb = NULL; /* skb is freed in the netlink send-op handling */
iwpm_user_pid = IWPM_PID_UNAVAILABLE;
err_str = "Unable to send a nlmsg";
goto pid_query_error;
}
nlmsg_request->req_buffer = pm_msg;
ret = iwpm_wait_complete_req(nlmsg_request);
return ret;
pid_query_error:
pr_info("%s: %s (client = %d)\n", __func__, err_str, nl_client);
if (skb)
dev_kfree_skb(skb);
if (nlmsg_request)
iwpm_free_nlmsg_request(&nlmsg_request->kref);
return ret;
}
/*
* iwpm_add_mapping - Send a netlink add mapping message
* to the port mapper
* nlmsg attributes:
* [IWPM_NLA_MANAGE_MAPPING_SEQ]
* [IWPM_NLA_MANAGE_ADDR]
*/
int iwpm_add_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)
{
struct sk_buff *skb = NULL;
struct iwpm_nlmsg_request *nlmsg_request = NULL;
struct nlmsghdr *nlh;
u32 msg_seq;
const char *err_str = "";
int ret = -EINVAL;
if (!iwpm_valid_client(nl_client)) {
err_str = "Invalid port mapper client";
goto add_mapping_error;
}
if (!iwpm_valid_pid())
return 0;
if (!iwpm_check_registration(nl_client, IWPM_REG_VALID)) {
err_str = "Unregistered port mapper client";
goto add_mapping_error;
}
skb = iwpm_create_nlmsg(RDMA_NL_IWPM_ADD_MAPPING, &nlh, nl_client);
if (!skb) {
err_str = "Unable to create a nlmsg";
goto add_mapping_error;
}
nlh->nlmsg_seq = iwpm_get_nlmsg_seq();
nlmsg_request = iwpm_get_nlmsg_request(nlh->nlmsg_seq, nl_client, GFP_KERNEL);
if (!nlmsg_request) {
err_str = "Unable to allocate netlink request";
goto add_mapping_error;
}
msg_seq = atomic_read(&echo_nlmsg_seq);
/* fill in the add mapping message */
err_str = "Unable to put attribute of the nlmsg";
ret = ibnl_put_attr(skb, nlh, sizeof(u32), &msg_seq,
IWPM_NLA_MANAGE_MAPPING_SEQ);
if (ret)
goto add_mapping_error;
ret = ibnl_put_attr(skb, nlh, sizeof(struct sockaddr_storage),
&pm_msg->loc_addr, IWPM_NLA_MANAGE_ADDR);
if (ret)
goto add_mapping_error;
nlmsg_end(skb, nlh);
nlmsg_request->req_buffer = pm_msg;
ret = rdma_nl_unicast_wait(skb, iwpm_user_pid);
if (ret) {
skb = NULL; /* skb is freed in the netlink send-op handling */
iwpm_user_pid = IWPM_PID_UNDEFINED;
err_str = "Unable to send a nlmsg";
goto add_mapping_error;
}
ret = iwpm_wait_complete_req(nlmsg_request);
return ret;
add_mapping_error:
pr_info("%s: %s (client = %d)\n", __func__, err_str, nl_client);
if (skb)
dev_kfree_skb(skb);
if (nlmsg_request)
iwpm_free_nlmsg_request(&nlmsg_request->kref);
return ret;
}
/*
* iwpm_add_and_query_mapping - Send a netlink add and query
* mapping message to the port mapper
* nlmsg attributes:
* [IWPM_NLA_QUERY_MAPPING_SEQ]
* [IWPM_NLA_QUERY_LOCAL_ADDR]
* [IWPM_NLA_QUERY_REMOTE_ADDR]
*/
int iwpm_add_and_query_mapping(struct iwpm_sa_data *pm_msg, u8 nl_client)
{
struct sk_buff *skb = NULL;
struct iwpm_nlmsg_request *nlmsg_request = NULL;
struct nlmsghdr *nlh;
u32 msg_seq;
const char *err_str = "";
int ret = -EINVAL;
if (!iwpm_valid_client(nl_client)) {
err_str = "Invalid port mapper client";
goto query_mapping_error;
}
if (!iwpm_valid_pid())
return 0;
if (!iwpm_check_registration(nl_client, IWPM_REG_VALID)) {
err_str = "Unregistered port mapper client";
goto query_mapping_error;
}
ret = -ENOMEM;
skb = iwpm_create_nlmsg(RDMA_NL_IWPM_QUERY_MAPPING, &nlh, nl_client);
if (!skb) {
err_str = "Unable to create a nlmsg";
goto query_mapping_error;
}
nlh->nlmsg_seq = iwpm_get_nlmsg_seq();
nlmsg_request = iwpm_get_nlmsg_request(nlh->nlmsg_seq,
nl_client, GFP_KERNEL);
if (!nlmsg_request) {
err_str = "Unable to allocate netlink request";
goto query_mapping_error;
}
msg_seq = atomic_read(&echo_nlmsg_seq);
/* fill in the query message */
err_str = "Unable to put attribute of the nlmsg";
ret = ibnl_put_attr(skb, nlh, sizeof(u32), &msg_seq,
IWPM_NLA_QUERY_MAPPING_SEQ);
if (ret)
goto query_mapping_error;
ret = ibnl_put_attr(skb, nlh, sizeof(struct sockaddr_storage),
&pm_msg->loc_addr, IWPM_NLA_QUERY_LOCAL_ADDR);
if (ret)
goto query_mapping_error;
ret = ibnl_put_attr(skb, nlh, sizeof(struct sockaddr_storage),
&pm_msg->rem_addr, IWPM_NLA_QUERY_REMOTE_ADDR);
if (ret)
goto query_mapping_error;
nlmsg_end(skb, nlh);
nlmsg_request->req_buffer = pm_msg;
ret = rdma_nl_unicast_wait(skb, iwpm_user_pid);
if (ret) {
skb = NULL; /* skb is freed in the netlink send-op handling */
err_str = "Unable to send a nlmsg";
goto query_mapping_error;
}
ret = iwpm_wait_complete_req(nlmsg_request);
return ret;
query_mapping_error:
pr_info("%s: %s (client = %d)\n", __func__, err_str, nl_client);
if (skb)
dev_kfree_skb(skb);
if (nlmsg_request)
iwpm_free_nlmsg_request(&nlmsg_request->kref);
return ret;
}
/*
* iwpm_remove_mapping - Send a netlink remove mapping message
* to the port mapper
* nlmsg attributes:
* [IWPM_NLA_MANAGE_MAPPING_SEQ]
* [IWPM_NLA_MANAGE_ADDR]
*/
int iwpm_remove_mapping(struct sockaddr_storage *local_addr, u8 nl_client)
{
struct sk_buff *skb = NULL;
struct nlmsghdr *nlh;
u32 msg_seq;
const char *err_str = "";
int ret = -EINVAL;
if (!iwpm_valid_client(nl_client)) {
err_str = "Invalid port mapper client";
goto remove_mapping_error;
}
if (!iwpm_valid_pid())
return 0;
if (iwpm_check_registration(nl_client, IWPM_REG_UNDEF)) {
err_str = "Unregistered port mapper client";
goto remove_mapping_error;
}
skb = iwpm_create_nlmsg(RDMA_NL_IWPM_REMOVE_MAPPING, &nlh, nl_client);
if (!skb) {
ret = -ENOMEM;
err_str = "Unable to create a nlmsg";
goto remove_mapping_error;
}
msg_seq = atomic_read(&echo_nlmsg_seq);
nlh->nlmsg_seq = iwpm_get_nlmsg_seq();
err_str = "Unable to put attribute of the nlmsg";
ret = ibnl_put_attr(skb, nlh, sizeof(u32), &msg_seq,
IWPM_NLA_MANAGE_MAPPING_SEQ);
if (ret)
goto remove_mapping_error;
ret = ibnl_put_attr(skb, nlh, sizeof(struct sockaddr_storage),
local_addr, IWPM_NLA_MANAGE_ADDR);
if (ret)
goto remove_mapping_error;
nlmsg_end(skb, nlh);
ret = rdma_nl_unicast_wait(skb, iwpm_user_pid);
if (ret) {
skb = NULL; /* skb is freed in the netlink send-op handling */
iwpm_user_pid = IWPM_PID_UNDEFINED;
err_str = "Unable to send a nlmsg";
goto remove_mapping_error;
}
iwpm_print_sockaddr(local_addr,
"remove_mapping: Local sockaddr:");
return 0;
remove_mapping_error:
pr_info("%s: %s (client = %d)\n", __func__, err_str, nl_client);
if (skb)
dev_kfree_skb_any(skb);
return ret;
}
/* netlink attribute policy for the received response to register pid request */
static const struct nla_policy resp_reg_policy[IWPM_NLA_RREG_PID_MAX] = {
[IWPM_NLA_RREG_PID_SEQ] = { .type = NLA_U32 },
[IWPM_NLA_RREG_IBDEV_NAME] = { .type = NLA_STRING,
.len = IWPM_DEVNAME_SIZE - 1 },
[IWPM_NLA_RREG_ULIB_NAME] = { .type = NLA_STRING,
.len = IWPM_ULIBNAME_SIZE - 1 },
[IWPM_NLA_RREG_ULIB_VER] = { .type = NLA_U16 },
[IWPM_NLA_RREG_PID_ERR] = { .type = NLA_U16 }
};
/*
* iwpm_register_pid_cb - Process a port mapper response to
* iwpm_register_pid()
*/
int iwpm_register_pid_cb(struct sk_buff *skb, struct netlink_callback *cb)
{
struct iwpm_nlmsg_request *nlmsg_request = NULL;
struct nlattr *nltb[IWPM_NLA_RREG_PID_MAX];
struct iwpm_dev_data *pm_msg;
char *dev_name, *iwpm_name;
u32 msg_seq;
u8 nl_client;
u16 iwpm_version;
const char *msg_type = "Register Pid response";
if (iwpm_parse_nlmsg(cb, IWPM_NLA_RREG_PID_MAX,
resp_reg_policy, nltb, msg_type))
return -EINVAL;
msg_seq = nla_get_u32(nltb[IWPM_NLA_RREG_PID_SEQ]);
nlmsg_request = iwpm_find_nlmsg_request(msg_seq);
if (!nlmsg_request) {
pr_info("%s: Could not find a matching request (seq = %u)\n",
__func__, msg_seq);
return -EINVAL;
}
pm_msg = nlmsg_request->req_buffer;
nl_client = nlmsg_request->nl_client;
dev_name = (char *)nla_data(nltb[IWPM_NLA_RREG_IBDEV_NAME]);
iwpm_name = (char *)nla_data(nltb[IWPM_NLA_RREG_ULIB_NAME]);
iwpm_version = nla_get_u16(nltb[IWPM_NLA_RREG_ULIB_VER]);
/* check device name, ulib name and version */
if (strcmp(pm_msg->dev_name, dev_name) ||
strcmp(iwpm_ulib_name, iwpm_name) ||
iwpm_version != iwpm_ulib_version) {
pr_info("%s: Incorrect info (dev = %s name = %s version = %d)\n",
__func__, dev_name, iwpm_name, iwpm_version);
nlmsg_request->err_code = IWPM_USER_LIB_INFO_ERR;
goto register_pid_response_exit;
}
iwpm_user_pid = cb->nlh->nlmsg_pid;
atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq);
pr_debug("%s: iWarp Port Mapper (pid = %d) is available!\n",
__func__, iwpm_user_pid);
if (iwpm_valid_client(nl_client))
iwpm_set_registration(nl_client, IWPM_REG_VALID);
register_pid_response_exit:
nlmsg_request->request_done = 1;
/* always for found nlmsg_request */
kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request);
barrier();
up(&nlmsg_request->sem);
return 0;
}
/* netlink attribute policy for the received response to add mapping request */
static const struct nla_policy resp_add_policy[IWPM_NLA_RMANAGE_MAPPING_MAX] = {
[IWPM_NLA_MANAGE_MAPPING_SEQ] = { .type = NLA_U32 },
[IWPM_NLA_MANAGE_ADDR] = { .len = sizeof(struct sockaddr_storage) },
[IWPM_NLA_MANAGE_MAPPED_LOC_ADDR] = { .len = sizeof(struct sockaddr_storage) },
[IWPM_NLA_RMANAGE_MAPPING_ERR] = { .type = NLA_U16 }
};
/*
* iwpm_add_mapping_cb - Process a port mapper response to
* iwpm_add_mapping()
*/
int iwpm_add_mapping_cb(struct sk_buff *skb, struct netlink_callback *cb)
{
struct iwpm_sa_data *pm_msg;
struct iwpm_nlmsg_request *nlmsg_request = NULL;
struct nlattr *nltb[IWPM_NLA_RMANAGE_MAPPING_MAX];
struct sockaddr_storage *local_sockaddr;
struct sockaddr_storage *mapped_sockaddr;
const char *msg_type;
u32 msg_seq;
msg_type = "Add Mapping response";
if (iwpm_parse_nlmsg(cb, IWPM_NLA_RMANAGE_MAPPING_MAX,
resp_add_policy, nltb, msg_type))
return -EINVAL;
atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq);
msg_seq = nla_get_u32(nltb[IWPM_NLA_MANAGE_MAPPING_SEQ]);
nlmsg_request = iwpm_find_nlmsg_request(msg_seq);
if (!nlmsg_request) {
pr_info("%s: Could not find a matching request (seq = %u)\n",
__func__, msg_seq);
return -EINVAL;
}
pm_msg = nlmsg_request->req_buffer;
local_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_MANAGE_ADDR]);
mapped_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_MANAGE_MAPPED_LOC_ADDR]);
if (iwpm_compare_sockaddr(local_sockaddr, &pm_msg->loc_addr)) {
nlmsg_request->err_code = IWPM_USER_LIB_INFO_ERR;
goto add_mapping_response_exit;
}
if (mapped_sockaddr->ss_family != local_sockaddr->ss_family) {
pr_info("%s: Sockaddr family doesn't match the requested one\n",
__func__);
nlmsg_request->err_code = IWPM_USER_LIB_INFO_ERR;
goto add_mapping_response_exit;
}
memcpy(&pm_msg->mapped_loc_addr, mapped_sockaddr,
sizeof(*mapped_sockaddr));
iwpm_print_sockaddr(&pm_msg->loc_addr,
"add_mapping: Local sockaddr:");
iwpm_print_sockaddr(&pm_msg->mapped_loc_addr,
"add_mapping: Mapped local sockaddr:");
add_mapping_response_exit:
nlmsg_request->request_done = 1;
/* always for found request */
kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request);
barrier();
up(&nlmsg_request->sem);
return 0;
}
/* netlink attribute policy for the response to add and query mapping request
* and response with remote address info */
static const struct nla_policy resp_query_policy[IWPM_NLA_RQUERY_MAPPING_MAX] = {
[IWPM_NLA_QUERY_MAPPING_SEQ] = { .type = NLA_U32 },
[IWPM_NLA_QUERY_LOCAL_ADDR] = { .len = sizeof(struct sockaddr_storage) },
[IWPM_NLA_QUERY_REMOTE_ADDR] = { .len = sizeof(struct sockaddr_storage) },
[IWPM_NLA_RQUERY_MAPPED_LOC_ADDR] = { .len = sizeof(struct sockaddr_storage) },
[IWPM_NLA_RQUERY_MAPPED_REM_ADDR] = { .len = sizeof(struct sockaddr_storage) },
[IWPM_NLA_RQUERY_MAPPING_ERR] = { .type = NLA_U16 }
};
/*
* iwpm_add_and_query_mapping_cb - Process a port mapper response to
* iwpm_add_and_query_mapping()
*/
int iwpm_add_and_query_mapping_cb(struct sk_buff *skb,
struct netlink_callback *cb)
{
struct iwpm_sa_data *pm_msg;
struct iwpm_nlmsg_request *nlmsg_request = NULL;
struct nlattr *nltb[IWPM_NLA_RQUERY_MAPPING_MAX];
struct sockaddr_storage *local_sockaddr, *remote_sockaddr;
struct sockaddr_storage *mapped_loc_sockaddr, *mapped_rem_sockaddr;
const char *msg_type;
u32 msg_seq;
u16 err_code;
msg_type = "Query Mapping response";
if (iwpm_parse_nlmsg(cb, IWPM_NLA_RQUERY_MAPPING_MAX,
resp_query_policy, nltb, msg_type))
return -EINVAL;
atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq);
msg_seq = nla_get_u32(nltb[IWPM_NLA_QUERY_MAPPING_SEQ]);
nlmsg_request = iwpm_find_nlmsg_request(msg_seq);
if (!nlmsg_request) {
pr_info("%s: Could not find a matching request (seq = %u)\n",
__func__, msg_seq);
return -EINVAL;
}
pm_msg = nlmsg_request->req_buffer;
local_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_QUERY_LOCAL_ADDR]);
remote_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_QUERY_REMOTE_ADDR]);
mapped_loc_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_RQUERY_MAPPED_LOC_ADDR]);
mapped_rem_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_RQUERY_MAPPED_REM_ADDR]);
err_code = nla_get_u16(nltb[IWPM_NLA_RQUERY_MAPPING_ERR]);
if (err_code == IWPM_REMOTE_QUERY_REJECT) {
pr_info("%s: Received a Reject (pid = %u, echo seq = %u)\n",
__func__, cb->nlh->nlmsg_pid, msg_seq);
nlmsg_request->err_code = IWPM_REMOTE_QUERY_REJECT;
}
if (iwpm_compare_sockaddr(local_sockaddr, &pm_msg->loc_addr) ||
iwpm_compare_sockaddr(remote_sockaddr, &pm_msg->rem_addr)) {
pr_info("%s: Incorrect local sockaddr\n", __func__);
nlmsg_request->err_code = IWPM_USER_LIB_INFO_ERR;
goto query_mapping_response_exit;
}
if (mapped_loc_sockaddr->ss_family != local_sockaddr->ss_family ||
mapped_rem_sockaddr->ss_family != remote_sockaddr->ss_family) {
pr_info("%s: Sockaddr family doesn't match the requested one\n",
__func__);
nlmsg_request->err_code = IWPM_USER_LIB_INFO_ERR;
goto query_mapping_response_exit;
}
memcpy(&pm_msg->mapped_loc_addr, mapped_loc_sockaddr,
sizeof(*mapped_loc_sockaddr));
memcpy(&pm_msg->mapped_rem_addr, mapped_rem_sockaddr,
sizeof(*mapped_rem_sockaddr));
iwpm_print_sockaddr(&pm_msg->loc_addr,
"query_mapping: Local sockaddr:");
iwpm_print_sockaddr(&pm_msg->mapped_loc_addr,
"query_mapping: Mapped local sockaddr:");
iwpm_print_sockaddr(&pm_msg->rem_addr,
"query_mapping: Remote sockaddr:");
iwpm_print_sockaddr(&pm_msg->mapped_rem_addr,
"query_mapping: Mapped remote sockaddr:");
query_mapping_response_exit:
nlmsg_request->request_done = 1;
/* always for found request */
kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request);
barrier();
up(&nlmsg_request->sem);
return 0;
}
/*
* iwpm_remote_info_cb - Process a port mapper message, containing
* the remote connecting peer address info
*/
int iwpm_remote_info_cb(struct sk_buff *skb, struct netlink_callback *cb)
{
struct nlattr *nltb[IWPM_NLA_RQUERY_MAPPING_MAX];
struct sockaddr_storage *local_sockaddr, *remote_sockaddr;
struct sockaddr_storage *mapped_loc_sockaddr, *mapped_rem_sockaddr;
struct iwpm_remote_info *rem_info;
const char *msg_type;
u8 nl_client;
int ret = -EINVAL;
msg_type = "Remote Mapping info";
if (iwpm_parse_nlmsg(cb, IWPM_NLA_RQUERY_MAPPING_MAX,
resp_query_policy, nltb, msg_type))
return ret;
nl_client = RDMA_NL_GET_CLIENT(cb->nlh->nlmsg_type);
if (!iwpm_valid_client(nl_client)) {
pr_info("%s: Invalid port mapper client = %d\n",
__func__, nl_client);
return ret;
}
atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq);
local_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_QUERY_LOCAL_ADDR]);
remote_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_QUERY_REMOTE_ADDR]);
mapped_loc_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_RQUERY_MAPPED_LOC_ADDR]);
mapped_rem_sockaddr = (struct sockaddr_storage *)
nla_data(nltb[IWPM_NLA_RQUERY_MAPPED_REM_ADDR]);
if (mapped_loc_sockaddr->ss_family != local_sockaddr->ss_family ||
mapped_rem_sockaddr->ss_family != remote_sockaddr->ss_family) {
pr_info("%s: Sockaddr family doesn't match the requested one\n",
__func__);
return ret;
}
rem_info = kzalloc(sizeof(struct iwpm_remote_info), GFP_ATOMIC);
if (!rem_info) {
ret = -ENOMEM;
return ret;
}
memcpy(&rem_info->mapped_loc_sockaddr, mapped_loc_sockaddr,
sizeof(struct sockaddr_storage));
memcpy(&rem_info->remote_sockaddr, remote_sockaddr,
sizeof(struct sockaddr_storage));
memcpy(&rem_info->mapped_rem_sockaddr, mapped_rem_sockaddr,
sizeof(struct sockaddr_storage));
rem_info->nl_client = nl_client;
iwpm_add_remote_info(rem_info);
iwpm_print_sockaddr(local_sockaddr,
"remote_info: Local sockaddr:");
iwpm_print_sockaddr(mapped_loc_sockaddr,
"remote_info: Mapped local sockaddr:");
iwpm_print_sockaddr(remote_sockaddr,
"remote_info: Remote sockaddr:");
iwpm_print_sockaddr(mapped_rem_sockaddr,
"remote_info: Mapped remote sockaddr:");
return ret;
}
/* netlink attribute policy for the received request for mapping info */
static const struct nla_policy resp_mapinfo_policy[IWPM_NLA_MAPINFO_REQ_MAX] = {
[IWPM_NLA_MAPINFO_ULIB_NAME] = { .type = NLA_STRING,
.len = IWPM_ULIBNAME_SIZE - 1 },
[IWPM_NLA_MAPINFO_ULIB_VER] = { .type = NLA_U16 }
};
/*
* iwpm_mapping_info_cb - Process a port mapper request for mapping info
*/
int iwpm_mapping_info_cb(struct sk_buff *skb, struct netlink_callback *cb)
{
struct nlattr *nltb[IWPM_NLA_MAPINFO_REQ_MAX];
const char *msg_type = "Mapping Info response";
u8 nl_client;
char *iwpm_name;
u16 iwpm_version;
int ret = -EINVAL;
if (iwpm_parse_nlmsg(cb, IWPM_NLA_MAPINFO_REQ_MAX,
resp_mapinfo_policy, nltb, msg_type)) {
pr_info("%s: Unable to parse nlmsg\n", __func__);
return ret;
}
iwpm_name = (char *)nla_data(nltb[IWPM_NLA_MAPINFO_ULIB_NAME]);
iwpm_version = nla_get_u16(nltb[IWPM_NLA_MAPINFO_ULIB_VER]);
if (strcmp(iwpm_ulib_name, iwpm_name) ||
iwpm_version != iwpm_ulib_version) {
pr_info("%s: Invalid port mapper name = %s version = %d\n",
__func__, iwpm_name, iwpm_version);
return ret;
}
nl_client = RDMA_NL_GET_CLIENT(cb->nlh->nlmsg_type);
if (!iwpm_valid_client(nl_client)) {
pr_info("%s: Invalid port mapper client = %d\n",
__func__, nl_client);
return ret;
}
iwpm_set_registration(nl_client, IWPM_REG_INCOMPL);
atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq);
iwpm_user_pid = cb->nlh->nlmsg_pid;
if (!iwpm_mapinfo_available())
return 0;
pr_debug("%s: iWarp Port Mapper (pid = %d) is available!\n",
__func__, iwpm_user_pid);
ret = iwpm_send_mapinfo(nl_client, iwpm_user_pid);
return ret;
}
/* netlink attribute policy for the received mapping info ack */
static const struct nla_policy ack_mapinfo_policy[IWPM_NLA_MAPINFO_NUM_MAX] = {
[IWPM_NLA_MAPINFO_SEQ] = { .type = NLA_U32 },
[IWPM_NLA_MAPINFO_SEND_NUM] = { .type = NLA_U32 },
[IWPM_NLA_MAPINFO_ACK_NUM] = { .type = NLA_U32 }
};
/*
* iwpm_ack_mapping_info_cb - Process a port mapper ack for
* the provided mapping info records
*/
int iwpm_ack_mapping_info_cb(struct sk_buff *skb, struct netlink_callback *cb)
{
struct nlattr *nltb[IWPM_NLA_MAPINFO_NUM_MAX];
u32 mapinfo_send, mapinfo_ack;
const char *msg_type = "Mapping Info Ack";
if (iwpm_parse_nlmsg(cb, IWPM_NLA_MAPINFO_NUM_MAX,
ack_mapinfo_policy, nltb, msg_type))
return -EINVAL;
mapinfo_send = nla_get_u32(nltb[IWPM_NLA_MAPINFO_SEND_NUM]);
mapinfo_ack = nla_get_u32(nltb[IWPM_NLA_MAPINFO_ACK_NUM]);
if (mapinfo_ack != mapinfo_send)
pr_info("%s: Invalid mapinfo number (sent = %u ack-ed = %u)\n",
__func__, mapinfo_send, mapinfo_ack);
atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq);
return 0;
}
/* netlink attribute policy for the received port mapper error message */
static const struct nla_policy map_error_policy[IWPM_NLA_ERR_MAX] = {
[IWPM_NLA_ERR_SEQ] = { .type = NLA_U32 },
[IWPM_NLA_ERR_CODE] = { .type = NLA_U16 },
};
/*
* iwpm_mapping_error_cb - Process a port mapper error message
*/
int iwpm_mapping_error_cb(struct sk_buff *skb, struct netlink_callback *cb)
{
struct iwpm_nlmsg_request *nlmsg_request = NULL;
int nl_client = RDMA_NL_GET_CLIENT(cb->nlh->nlmsg_type);
struct nlattr *nltb[IWPM_NLA_ERR_MAX];
u32 msg_seq;
u16 err_code;
const char *msg_type = "Mapping Error Msg";
if (iwpm_parse_nlmsg(cb, IWPM_NLA_ERR_MAX,
map_error_policy, nltb, msg_type))
return -EINVAL;
msg_seq = nla_get_u32(nltb[IWPM_NLA_ERR_SEQ]);
err_code = nla_get_u16(nltb[IWPM_NLA_ERR_CODE]);
pr_info("%s: Received msg seq = %u err code = %u client = %d\n",
__func__, msg_seq, err_code, nl_client);
/* look for nlmsg_request */
nlmsg_request = iwpm_find_nlmsg_request(msg_seq);
if (!nlmsg_request) {
/* not all errors have associated requests */
pr_debug("Could not find matching req (seq = %u)\n", msg_seq);
return 0;
}
atomic_set(&echo_nlmsg_seq, cb->nlh->nlmsg_seq);
nlmsg_request->err_code = err_code;
nlmsg_request->request_done = 1;
/* always for found request */
kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request);
barrier();
up(&nlmsg_request->sem);
return 0;
}
+756
View File
@@ -0,0 +1,756 @@
/*
* Copyright (c) 2014 Chelsio, Inc. All rights reserved.
* Copyright (c) 2014 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "iwpm_util.h"
#define IWPM_MAPINFO_HASH_SIZE 512
#define IWPM_MAPINFO_HASH_MASK (IWPM_MAPINFO_HASH_SIZE - 1)
#define IWPM_REMINFO_HASH_SIZE 64
#define IWPM_REMINFO_HASH_MASK (IWPM_REMINFO_HASH_SIZE - 1)
#define IWPM_MSG_SIZE 512
static LIST_HEAD(iwpm_nlmsg_req_list);
static DEFINE_SPINLOCK(iwpm_nlmsg_req_lock);
static struct hlist_head *iwpm_hash_bucket;
static DEFINE_SPINLOCK(iwpm_mapinfo_lock);
static struct hlist_head *iwpm_reminfo_bucket;
static DEFINE_SPINLOCK(iwpm_reminfo_lock);
static DEFINE_MUTEX(iwpm_admin_lock);
static struct iwpm_admin_data iwpm_admin;
int iwpm_init(u8 nl_client)
{
int ret = 0;
mutex_lock(&iwpm_admin_lock);
if (atomic_read(&iwpm_admin.refcount) == 0) {
iwpm_hash_bucket = kcalloc(IWPM_MAPINFO_HASH_SIZE,
sizeof(struct hlist_head),
GFP_KERNEL);
if (!iwpm_hash_bucket) {
ret = -ENOMEM;
goto init_exit;
}
iwpm_reminfo_bucket = kcalloc(IWPM_REMINFO_HASH_SIZE,
sizeof(struct hlist_head),
GFP_KERNEL);
if (!iwpm_reminfo_bucket) {
kfree(iwpm_hash_bucket);
ret = -ENOMEM;
goto init_exit;
}
}
atomic_inc(&iwpm_admin.refcount);
init_exit:
mutex_unlock(&iwpm_admin_lock);
if (!ret) {
iwpm_set_valid(nl_client, 1);
iwpm_set_registration(nl_client, IWPM_REG_UNDEF);
pr_debug("%s: Mapinfo and reminfo tables are created\n",
__func__);
}
return ret;
}
static void free_hash_bucket(void);
static void free_reminfo_bucket(void);
int iwpm_exit(u8 nl_client)
{
if (!iwpm_valid_client(nl_client))
return -EINVAL;
mutex_lock(&iwpm_admin_lock);
if (atomic_read(&iwpm_admin.refcount) == 0) {
mutex_unlock(&iwpm_admin_lock);
pr_err("%s Incorrect usage - negative refcount\n", __func__);
return -EINVAL;
}
if (atomic_dec_and_test(&iwpm_admin.refcount)) {
free_hash_bucket();
free_reminfo_bucket();
pr_debug("%s: Resources are destroyed\n", __func__);
}
mutex_unlock(&iwpm_admin_lock);
iwpm_set_valid(nl_client, 0);
iwpm_set_registration(nl_client, IWPM_REG_UNDEF);
return 0;
}
static struct hlist_head *get_mapinfo_hash_bucket(struct sockaddr_storage *,
struct sockaddr_storage *);
int iwpm_create_mapinfo(struct sockaddr_storage *local_sockaddr,
struct sockaddr_storage *mapped_sockaddr,
u8 nl_client)
{
struct hlist_head *hash_bucket_head = NULL;
struct iwpm_mapping_info *map_info;
unsigned long flags;
int ret = -EINVAL;
if (!iwpm_valid_client(nl_client))
return ret;
map_info = kzalloc(sizeof(struct iwpm_mapping_info), GFP_KERNEL);
if (!map_info)
return -ENOMEM;
memcpy(&map_info->local_sockaddr, local_sockaddr,
sizeof(struct sockaddr_storage));
memcpy(&map_info->mapped_sockaddr, mapped_sockaddr,
sizeof(struct sockaddr_storage));
map_info->nl_client = nl_client;
spin_lock_irqsave(&iwpm_mapinfo_lock, flags);
if (iwpm_hash_bucket) {
hash_bucket_head = get_mapinfo_hash_bucket(
&map_info->local_sockaddr,
&map_info->mapped_sockaddr);
if (hash_bucket_head) {
hlist_add_head(&map_info->hlist_node, hash_bucket_head);
ret = 0;
}
}
spin_unlock_irqrestore(&iwpm_mapinfo_lock, flags);
if (!hash_bucket_head)
kfree(map_info);
return ret;
}
int iwpm_remove_mapinfo(struct sockaddr_storage *local_sockaddr,
struct sockaddr_storage *mapped_local_addr)
{
struct hlist_node *tmp_hlist_node;
struct hlist_head *hash_bucket_head;
struct iwpm_mapping_info *map_info = NULL;
unsigned long flags;
int ret = -EINVAL;
spin_lock_irqsave(&iwpm_mapinfo_lock, flags);
if (iwpm_hash_bucket) {
hash_bucket_head = get_mapinfo_hash_bucket(
local_sockaddr,
mapped_local_addr);
if (!hash_bucket_head)
goto remove_mapinfo_exit;
hlist_for_each_entry_safe(map_info, tmp_hlist_node,
hash_bucket_head, hlist_node) {
if (!iwpm_compare_sockaddr(&map_info->mapped_sockaddr,
mapped_local_addr)) {
hlist_del_init(&map_info->hlist_node);
kfree(map_info);
ret = 0;
break;
}
}
}
remove_mapinfo_exit:
spin_unlock_irqrestore(&iwpm_mapinfo_lock, flags);
return ret;
}
static void free_hash_bucket(void)
{
struct hlist_node *tmp_hlist_node;
struct iwpm_mapping_info *map_info;
unsigned long flags;
int i;
/* remove all the mapinfo data from the list */
spin_lock_irqsave(&iwpm_mapinfo_lock, flags);
for (i = 0; i < IWPM_MAPINFO_HASH_SIZE; i++) {
hlist_for_each_entry_safe(map_info, tmp_hlist_node,
&iwpm_hash_bucket[i], hlist_node) {
hlist_del_init(&map_info->hlist_node);
kfree(map_info);
}
}
/* free the hash list */
kfree(iwpm_hash_bucket);
iwpm_hash_bucket = NULL;
spin_unlock_irqrestore(&iwpm_mapinfo_lock, flags);
}
static void free_reminfo_bucket(void)
{
struct hlist_node *tmp_hlist_node;
struct iwpm_remote_info *rem_info;
unsigned long flags;
int i;
/* remove all the remote info from the list */
spin_lock_irqsave(&iwpm_reminfo_lock, flags);
for (i = 0; i < IWPM_REMINFO_HASH_SIZE; i++) {
hlist_for_each_entry_safe(rem_info, tmp_hlist_node,
&iwpm_reminfo_bucket[i], hlist_node) {
hlist_del_init(&rem_info->hlist_node);
kfree(rem_info);
}
}
/* free the hash list */
kfree(iwpm_reminfo_bucket);
iwpm_reminfo_bucket = NULL;
spin_unlock_irqrestore(&iwpm_reminfo_lock, flags);
}
static struct hlist_head *get_reminfo_hash_bucket(struct sockaddr_storage *,
struct sockaddr_storage *);
void iwpm_add_remote_info(struct iwpm_remote_info *rem_info)
{
struct hlist_head *hash_bucket_head;
unsigned long flags;
spin_lock_irqsave(&iwpm_reminfo_lock, flags);
if (iwpm_reminfo_bucket) {
hash_bucket_head = get_reminfo_hash_bucket(
&rem_info->mapped_loc_sockaddr,
&rem_info->mapped_rem_sockaddr);
if (hash_bucket_head)
hlist_add_head(&rem_info->hlist_node, hash_bucket_head);
}
spin_unlock_irqrestore(&iwpm_reminfo_lock, flags);
}
int iwpm_get_remote_info(struct sockaddr_storage *mapped_loc_addr,
struct sockaddr_storage *mapped_rem_addr,
struct sockaddr_storage *remote_addr,
u8 nl_client)
{
struct hlist_node *tmp_hlist_node;
struct hlist_head *hash_bucket_head;
struct iwpm_remote_info *rem_info = NULL;
unsigned long flags;
int ret = -EINVAL;
if (!iwpm_valid_client(nl_client)) {
pr_info("%s: Invalid client = %d\n", __func__, nl_client);
return ret;
}
spin_lock_irqsave(&iwpm_reminfo_lock, flags);
if (iwpm_reminfo_bucket) {
hash_bucket_head = get_reminfo_hash_bucket(
mapped_loc_addr,
mapped_rem_addr);
if (!hash_bucket_head)
goto get_remote_info_exit;
hlist_for_each_entry_safe(rem_info, tmp_hlist_node,
hash_bucket_head, hlist_node) {
if (!iwpm_compare_sockaddr(&rem_info->mapped_loc_sockaddr,
mapped_loc_addr) &&
!iwpm_compare_sockaddr(&rem_info->mapped_rem_sockaddr,
mapped_rem_addr)) {
memcpy(remote_addr, &rem_info->remote_sockaddr,
sizeof(struct sockaddr_storage));
iwpm_print_sockaddr(remote_addr,
"get_remote_info: Remote sockaddr:");
hlist_del_init(&rem_info->hlist_node);
kfree(rem_info);
ret = 0;
break;
}
}
}
get_remote_info_exit:
spin_unlock_irqrestore(&iwpm_reminfo_lock, flags);
return ret;
}
struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq,
u8 nl_client, gfp_t gfp)
{
struct iwpm_nlmsg_request *nlmsg_request = NULL;
unsigned long flags;
nlmsg_request = kzalloc(sizeof(struct iwpm_nlmsg_request), gfp);
if (!nlmsg_request)
return NULL;
spin_lock_irqsave(&iwpm_nlmsg_req_lock, flags);
list_add_tail(&nlmsg_request->inprocess_list, &iwpm_nlmsg_req_list);
spin_unlock_irqrestore(&iwpm_nlmsg_req_lock, flags);
kref_init(&nlmsg_request->kref);
kref_get(&nlmsg_request->kref);
nlmsg_request->nlmsg_seq = nlmsg_seq;
nlmsg_request->nl_client = nl_client;
nlmsg_request->request_done = 0;
nlmsg_request->err_code = 0;
sema_init(&nlmsg_request->sem, 1);
down(&nlmsg_request->sem);
return nlmsg_request;
}
void iwpm_free_nlmsg_request(struct kref *kref)
{
struct iwpm_nlmsg_request *nlmsg_request;
unsigned long flags;
nlmsg_request = container_of(kref, struct iwpm_nlmsg_request, kref);
spin_lock_irqsave(&iwpm_nlmsg_req_lock, flags);
list_del_init(&nlmsg_request->inprocess_list);
spin_unlock_irqrestore(&iwpm_nlmsg_req_lock, flags);
if (!nlmsg_request->request_done)
pr_debug("%s Freeing incomplete nlmsg request (seq = %u).\n",
__func__, nlmsg_request->nlmsg_seq);
kfree(nlmsg_request);
}
struct iwpm_nlmsg_request *iwpm_find_nlmsg_request(__u32 echo_seq)
{
struct iwpm_nlmsg_request *nlmsg_request;
struct iwpm_nlmsg_request *found_request = NULL;
unsigned long flags;
spin_lock_irqsave(&iwpm_nlmsg_req_lock, flags);
list_for_each_entry(nlmsg_request, &iwpm_nlmsg_req_list,
inprocess_list) {
if (nlmsg_request->nlmsg_seq == echo_seq) {
found_request = nlmsg_request;
kref_get(&nlmsg_request->kref);
break;
}
}
spin_unlock_irqrestore(&iwpm_nlmsg_req_lock, flags);
return found_request;
}
int iwpm_wait_complete_req(struct iwpm_nlmsg_request *nlmsg_request)
{
int ret;
ret = down_timeout(&nlmsg_request->sem, IWPM_NL_TIMEOUT);
if (ret) {
ret = -EINVAL;
pr_info("%s: Timeout %d sec for netlink request (seq = %u)\n",
__func__, (IWPM_NL_TIMEOUT/HZ), nlmsg_request->nlmsg_seq);
} else {
ret = nlmsg_request->err_code;
}
kref_put(&nlmsg_request->kref, iwpm_free_nlmsg_request);
return ret;
}
int iwpm_get_nlmsg_seq(void)
{
return atomic_inc_return(&iwpm_admin.nlmsg_seq);
}
int iwpm_valid_client(u8 nl_client)
{
return iwpm_admin.client_list[nl_client];
}
void iwpm_set_valid(u8 nl_client, int valid)
{
iwpm_admin.client_list[nl_client] = valid;
}
/* valid client */
u32 iwpm_get_registration(u8 nl_client)
{
return iwpm_admin.reg_list[nl_client];
}
/* valid client */
void iwpm_set_registration(u8 nl_client, u32 reg)
{
iwpm_admin.reg_list[nl_client] = reg;
}
/* valid client */
u32 iwpm_check_registration(u8 nl_client, u32 reg)
{
return (iwpm_get_registration(nl_client) & reg);
}
int iwpm_compare_sockaddr(struct sockaddr_storage *a_sockaddr,
struct sockaddr_storage *b_sockaddr)
{
if (a_sockaddr->ss_family != b_sockaddr->ss_family)
return 1;
if (a_sockaddr->ss_family == AF_INET) {
struct sockaddr_in *a4_sockaddr =
(struct sockaddr_in *)a_sockaddr;
struct sockaddr_in *b4_sockaddr =
(struct sockaddr_in *)b_sockaddr;
if (!memcmp(&a4_sockaddr->sin_addr,
&b4_sockaddr->sin_addr, sizeof(struct in_addr))
&& a4_sockaddr->sin_port == b4_sockaddr->sin_port)
return 0;
} else if (a_sockaddr->ss_family == AF_INET6) {
struct sockaddr_in6 *a6_sockaddr =
(struct sockaddr_in6 *)a_sockaddr;
struct sockaddr_in6 *b6_sockaddr =
(struct sockaddr_in6 *)b_sockaddr;
if (!memcmp(&a6_sockaddr->sin6_addr,
&b6_sockaddr->sin6_addr, sizeof(struct in6_addr))
&& a6_sockaddr->sin6_port == b6_sockaddr->sin6_port)
return 0;
} else {
pr_err("%s: Invalid sockaddr family\n", __func__);
}
return 1;
}
struct sk_buff *iwpm_create_nlmsg(u32 nl_op, struct nlmsghdr **nlh,
int nl_client)
{
struct sk_buff *skb = NULL;
skb = dev_alloc_skb(IWPM_MSG_SIZE);
if (!skb)
goto create_nlmsg_exit;
if (!(ibnl_put_msg(skb, nlh, 0, 0, nl_client, nl_op,
NLM_F_REQUEST))) {
pr_warn("%s: Unable to put the nlmsg header\n", __func__);
dev_kfree_skb(skb);
skb = NULL;
}
create_nlmsg_exit:
return skb;
}
int iwpm_parse_nlmsg(struct netlink_callback *cb, int policy_max,
const struct nla_policy *nlmsg_policy,
struct nlattr *nltb[], const char *msg_type)
{
int nlh_len = 0;
int ret;
const char *err_str = "";
ret = nlmsg_validate(cb->nlh, nlh_len, policy_max - 1, nlmsg_policy,
NULL);
if (ret) {
err_str = "Invalid attribute";
goto parse_nlmsg_error;
}
ret = nlmsg_parse(cb->nlh, nlh_len, nltb, policy_max - 1,
nlmsg_policy, NULL);
if (ret) {
err_str = "Unable to parse the nlmsg";
goto parse_nlmsg_error;
}
ret = iwpm_validate_nlmsg_attr(nltb, policy_max);
if (ret) {
err_str = "Invalid NULL attribute";
goto parse_nlmsg_error;
}
return 0;
parse_nlmsg_error:
pr_warn("%s: %s (msg type %s ret = %d)\n",
__func__, err_str, msg_type, ret);
return ret;
}
void iwpm_print_sockaddr(struct sockaddr_storage *sockaddr, char *msg)
{
struct sockaddr_in6 *sockaddr_v6;
struct sockaddr_in *sockaddr_v4;
switch (sockaddr->ss_family) {
case AF_INET:
sockaddr_v4 = (struct sockaddr_in *)sockaddr;
pr_debug("%s IPV4 %pI4: %u(0x%04X)\n",
msg, &sockaddr_v4->sin_addr,
ntohs(sockaddr_v4->sin_port),
ntohs(sockaddr_v4->sin_port));
break;
case AF_INET6:
sockaddr_v6 = (struct sockaddr_in6 *)sockaddr;
pr_debug("%s IPV6 %pI6: %u(0x%04X)\n",
msg, &sockaddr_v6->sin6_addr,
ntohs(sockaddr_v6->sin6_port),
ntohs(sockaddr_v6->sin6_port));
break;
default:
break;
}
}
static u32 iwpm_ipv6_jhash(struct sockaddr_in6 *ipv6_sockaddr)
{
u32 ipv6_hash = jhash(&ipv6_sockaddr->sin6_addr, sizeof(struct in6_addr), 0);
u32 hash = jhash_2words(ipv6_hash, (__force u32) ipv6_sockaddr->sin6_port, 0);
return hash;
}
static u32 iwpm_ipv4_jhash(struct sockaddr_in *ipv4_sockaddr)
{
u32 ipv4_hash = jhash(&ipv4_sockaddr->sin_addr, sizeof(struct in_addr), 0);
u32 hash = jhash_2words(ipv4_hash, (__force u32) ipv4_sockaddr->sin_port, 0);
return hash;
}
static int get_hash_bucket(struct sockaddr_storage *a_sockaddr,
struct sockaddr_storage *b_sockaddr, u32 *hash)
{
u32 a_hash, b_hash;
if (a_sockaddr->ss_family == AF_INET) {
a_hash = iwpm_ipv4_jhash((struct sockaddr_in *) a_sockaddr);
b_hash = iwpm_ipv4_jhash((struct sockaddr_in *) b_sockaddr);
} else if (a_sockaddr->ss_family == AF_INET6) {
a_hash = iwpm_ipv6_jhash((struct sockaddr_in6 *) a_sockaddr);
b_hash = iwpm_ipv6_jhash((struct sockaddr_in6 *) b_sockaddr);
} else {
pr_err("%s: Invalid sockaddr family\n", __func__);
return -EINVAL;
}
if (a_hash == b_hash) /* if port mapper isn't available */
*hash = a_hash;
else
*hash = jhash_2words(a_hash, b_hash, 0);
return 0;
}
static struct hlist_head *get_mapinfo_hash_bucket(struct sockaddr_storage
*local_sockaddr, struct sockaddr_storage
*mapped_sockaddr)
{
u32 hash;
int ret;
ret = get_hash_bucket(local_sockaddr, mapped_sockaddr, &hash);
if (ret)
return NULL;
return &iwpm_hash_bucket[hash & IWPM_MAPINFO_HASH_MASK];
}
static struct hlist_head *get_reminfo_hash_bucket(struct sockaddr_storage
*mapped_loc_sockaddr, struct sockaddr_storage
*mapped_rem_sockaddr)
{
u32 hash;
int ret;
ret = get_hash_bucket(mapped_loc_sockaddr, mapped_rem_sockaddr, &hash);
if (ret)
return NULL;
return &iwpm_reminfo_bucket[hash & IWPM_REMINFO_HASH_MASK];
}
static int send_mapinfo_num(u32 mapping_num, u8 nl_client, int iwpm_pid)
{
struct sk_buff *skb = NULL;
struct nlmsghdr *nlh;
u32 msg_seq;
const char *err_str = "";
int ret = -EINVAL;
skb = iwpm_create_nlmsg(RDMA_NL_IWPM_MAPINFO_NUM, &nlh, nl_client);
if (!skb) {
err_str = "Unable to create a nlmsg";
goto mapinfo_num_error;
}
nlh->nlmsg_seq = iwpm_get_nlmsg_seq();
msg_seq = 0;
err_str = "Unable to put attribute of mapinfo number nlmsg";
ret = ibnl_put_attr(skb, nlh, sizeof(u32), &msg_seq, IWPM_NLA_MAPINFO_SEQ);
if (ret)
goto mapinfo_num_error;
ret = ibnl_put_attr(skb, nlh, sizeof(u32),
&mapping_num, IWPM_NLA_MAPINFO_SEND_NUM);
if (ret)
goto mapinfo_num_error;
nlmsg_end(skb, nlh);
ret = rdma_nl_unicast(skb, iwpm_pid);
if (ret) {
skb = NULL;
err_str = "Unable to send a nlmsg";
goto mapinfo_num_error;
}
pr_debug("%s: Sent mapping number = %d\n", __func__, mapping_num);
return 0;
mapinfo_num_error:
pr_info("%s: %s\n", __func__, err_str);
if (skb)
dev_kfree_skb(skb);
return ret;
}
static int send_nlmsg_done(struct sk_buff *skb, u8 nl_client, int iwpm_pid)
{
struct nlmsghdr *nlh = NULL;
int ret = 0;
if (!skb)
return ret;
if (!(ibnl_put_msg(skb, &nlh, 0, 0, nl_client,
RDMA_NL_IWPM_MAPINFO, NLM_F_MULTI))) {
pr_warn("%s Unable to put NLMSG_DONE\n", __func__);
dev_kfree_skb(skb);
return -ENOMEM;
}
nlh->nlmsg_type = NLMSG_DONE;
ret = rdma_nl_unicast(skb, iwpm_pid);
if (ret)
pr_warn("%s Unable to send a nlmsg\n", __func__);
return ret;
}
int iwpm_send_mapinfo(u8 nl_client, int iwpm_pid)
{
struct iwpm_mapping_info *map_info;
struct sk_buff *skb = NULL;
struct nlmsghdr *nlh;
int skb_num = 0, mapping_num = 0;
int i = 0, nlmsg_bytes = 0;
unsigned long flags;
const char *err_str = "";
int ret;
skb = dev_alloc_skb(NLMSG_GOODSIZE);
if (!skb) {
ret = -ENOMEM;
err_str = "Unable to allocate skb";
goto send_mapping_info_exit;
}
skb_num++;
spin_lock_irqsave(&iwpm_mapinfo_lock, flags);
ret = -EINVAL;
for (i = 0; i < IWPM_MAPINFO_HASH_SIZE; i++) {
hlist_for_each_entry(map_info, &iwpm_hash_bucket[i],
hlist_node) {
if (map_info->nl_client != nl_client)
continue;
nlh = NULL;
if (!(ibnl_put_msg(skb, &nlh, 0, 0, nl_client,
RDMA_NL_IWPM_MAPINFO, NLM_F_MULTI))) {
ret = -ENOMEM;
err_str = "Unable to put the nlmsg header";
goto send_mapping_info_unlock;
}
err_str = "Unable to put attribute of the nlmsg";
ret = ibnl_put_attr(skb, nlh,
sizeof(struct sockaddr_storage),
&map_info->local_sockaddr,
IWPM_NLA_MAPINFO_LOCAL_ADDR);
if (ret)
goto send_mapping_info_unlock;
ret = ibnl_put_attr(skb, nlh,
sizeof(struct sockaddr_storage),
&map_info->mapped_sockaddr,
IWPM_NLA_MAPINFO_MAPPED_ADDR);
if (ret)
goto send_mapping_info_unlock;
nlmsg_end(skb, nlh);
iwpm_print_sockaddr(&map_info->local_sockaddr,
"send_mapping_info: Local sockaddr:");
iwpm_print_sockaddr(&map_info->mapped_sockaddr,
"send_mapping_info: Mapped local sockaddr:");
mapping_num++;
nlmsg_bytes += nlh->nlmsg_len;
/* check if all mappings can fit in one skb */
if (NLMSG_GOODSIZE - nlmsg_bytes < nlh->nlmsg_len * 2) {
/* and leave room for NLMSG_DONE */
nlmsg_bytes = 0;
skb_num++;
spin_unlock_irqrestore(&iwpm_mapinfo_lock,
flags);
/* send the skb */
ret = send_nlmsg_done(skb, nl_client, iwpm_pid);
skb = NULL;
if (ret) {
err_str = "Unable to send map info";
goto send_mapping_info_exit;
}
if (skb_num == IWPM_MAPINFO_SKB_COUNT) {
ret = -ENOMEM;
err_str = "Insufficient skbs for map info";
goto send_mapping_info_exit;
}
skb = dev_alloc_skb(NLMSG_GOODSIZE);
if (!skb) {
ret = -ENOMEM;
err_str = "Unable to allocate skb";
goto send_mapping_info_exit;
}
spin_lock_irqsave(&iwpm_mapinfo_lock, flags);
}
}
}
send_mapping_info_unlock:
spin_unlock_irqrestore(&iwpm_mapinfo_lock, flags);
send_mapping_info_exit:
if (ret) {
pr_warn("%s: %s (ret = %d)\n", __func__, err_str, ret);
if (skb)
dev_kfree_skb(skb);
return ret;
}
send_nlmsg_done(skb, nl_client, iwpm_pid);
return send_mapinfo_num(mapping_num, nl_client, iwpm_pid);
}
int iwpm_mapinfo_available(void)
{
unsigned long flags;
int full_bucket = 0, i = 0;
spin_lock_irqsave(&iwpm_mapinfo_lock, flags);
if (iwpm_hash_bucket) {
for (i = 0; i < IWPM_MAPINFO_HASH_SIZE; i++) {
if (!hlist_empty(&iwpm_hash_bucket[i])) {
full_bucket = 1;
break;
}
}
}
spin_unlock_irqrestore(&iwpm_mapinfo_lock, flags);
return full_bucket;
}
+269
View File
@@ -0,0 +1,269 @@
/*
* Copyright (c) 2014 Intel Corporation. All rights reserved.
* Copyright (c) 2014 Chelsio, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef _IWPM_UTIL_H
#define _IWPM_UTIL_H
#include <linux/module.h>
#include <linux/io.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/spinlock.h>
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/delay.h>
#include <linux/workqueue.h>
#include <linux/mutex.h>
#include <linux/jhash.h>
#include <linux/kref.h>
#include <net/netlink.h>
#include <linux/errno.h>
#include <rdma/iw_portmap.h>
#include <rdma/rdma_netlink.h>
#define IWPM_NL_RETRANS 3
#define IWPM_NL_TIMEOUT (10*HZ)
#define IWPM_MAPINFO_SKB_COUNT 20
#define IWPM_PID_UNDEFINED -1
#define IWPM_PID_UNAVAILABLE -2
#define IWPM_REG_UNDEF 0x01
#define IWPM_REG_VALID 0x02
#define IWPM_REG_INCOMPL 0x04
struct iwpm_nlmsg_request {
struct list_head inprocess_list;
__u32 nlmsg_seq;
void *req_buffer;
u8 nl_client;
u8 request_done;
u16 err_code;
struct semaphore sem;
struct kref kref;
};
struct iwpm_mapping_info {
struct hlist_node hlist_node;
struct sockaddr_storage local_sockaddr;
struct sockaddr_storage mapped_sockaddr;
u8 nl_client;
};
struct iwpm_remote_info {
struct hlist_node hlist_node;
struct sockaddr_storage remote_sockaddr;
struct sockaddr_storage mapped_loc_sockaddr;
struct sockaddr_storage mapped_rem_sockaddr;
u8 nl_client;
};
struct iwpm_admin_data {
atomic_t refcount;
atomic_t nlmsg_seq;
int client_list[RDMA_NL_NUM_CLIENTS];
u32 reg_list[RDMA_NL_NUM_CLIENTS];
};
/**
* iwpm_get_nlmsg_request - Allocate and initialize netlink message request
* @nlmsg_seq: Sequence number of the netlink message
* @nl_client: The index of the netlink client
* @gfp: Indicates how the memory for the request should be allocated
*
* Returns the newly allocated netlink request object if successful,
* otherwise returns NULL
*/
struct iwpm_nlmsg_request *iwpm_get_nlmsg_request(__u32 nlmsg_seq,
u8 nl_client, gfp_t gfp);
/**
* iwpm_free_nlmsg_request - Deallocate netlink message request
* @kref: Holds reference of netlink message request
*/
void iwpm_free_nlmsg_request(struct kref *kref);
/**
* iwpm_find_nlmsg_request - Find netlink message request in the request list
* @echo_seq: Sequence number of the netlink request to find
*
* Returns the found netlink message request,
* if not found, returns NULL
*/
struct iwpm_nlmsg_request *iwpm_find_nlmsg_request(__u32 echo_seq);
/**
* iwpm_wait_complete_req - Block while servicing the netlink request
* @nlmsg_request: Netlink message request to service
*
* Wakes up, after the request is completed or expired
* Returns 0 if the request is complete without error
*/
int iwpm_wait_complete_req(struct iwpm_nlmsg_request *nlmsg_request);
/**
* iwpm_get_nlmsg_seq - Get the sequence number for a netlink
* message to send to the port mapper
*
* Returns the sequence number for the netlink message.
*/
int iwpm_get_nlmsg_seq(void);
/**
* iwpm_add_reminfo - Add remote address info of the connecting peer
* to the remote info hash table
* @reminfo: The remote info to be added
*/
void iwpm_add_remote_info(struct iwpm_remote_info *reminfo);
/**
* iwpm_valid_client - Check if the port mapper client is valid
* @nl_client: The index of the netlink client
*
* Valid clients need to call iwpm_init() before using
* the port mapper
*/
int iwpm_valid_client(u8 nl_client);
/**
* iwpm_set_valid - Set the port mapper client to valid or not
* @nl_client: The index of the netlink client
* @valid: 1 if valid or 0 if invalid
*/
void iwpm_set_valid(u8 nl_client, int valid);
/**
* iwpm_check_registration - Check if the client registration
* matches the given one
* @nl_client: The index of the netlink client
* @reg: The given registration type to compare with
*
* Call iwpm_register_pid() to register a client
* Returns true if the client registration matches reg,
* otherwise returns false
*/
u32 iwpm_check_registration(u8 nl_client, u32 reg);
/**
* iwpm_set_registration - Set the client registration
* @nl_client: The index of the netlink client
* @reg: Registration type to set
*/
void iwpm_set_registration(u8 nl_client, u32 reg);
/**
* iwpm_get_registration
* @nl_client: The index of the netlink client
*
* Returns the client registration type
*/
u32 iwpm_get_registration(u8 nl_client);
/**
* iwpm_send_mapinfo - Send local and mapped IPv4/IPv6 address info of
* a client to the user space port mapper
* @nl_client: The index of the netlink client
* @iwpm_pid: The pid of the user space port mapper
*
* If successful, returns the number of sent mapping info records
*/
int iwpm_send_mapinfo(u8 nl_client, int iwpm_pid);
/**
* iwpm_mapinfo_available - Check if any mapping info records is available
* in the hash table
*
* Returns 1 if mapping information is available, otherwise returns 0
*/
int iwpm_mapinfo_available(void);
/**
* iwpm_compare_sockaddr - Compare two sockaddr storage structs
*
* Returns 0 if they are holding the same ip/tcp address info,
* otherwise returns 1
*/
int iwpm_compare_sockaddr(struct sockaddr_storage *a_sockaddr,
struct sockaddr_storage *b_sockaddr);
/**
* iwpm_validate_nlmsg_attr - Check for NULL netlink attributes
* @nltb: Holds address of each netlink message attributes
* @nla_count: Number of netlink message attributes
*
* Returns error if any of the nla_count attributes is NULL
*/
static inline int iwpm_validate_nlmsg_attr(struct nlattr *nltb[],
int nla_count)
{
int i;
for (i = 1; i < nla_count; i++) {
if (!nltb[i])
return -EINVAL;
}
return 0;
}
/**
* iwpm_create_nlmsg - Allocate skb and form a netlink message
* @nl_op: Netlink message opcode
* @nlh: Holds address of the netlink message header in skb
* @nl_client: The index of the netlink client
*
* Returns the newly allcated skb, or NULL if the tailroom of the skb
* is insufficient to store the message header and payload
*/
struct sk_buff *iwpm_create_nlmsg(u32 nl_op, struct nlmsghdr **nlh,
int nl_client);
/**
* iwpm_parse_nlmsg - Validate and parse the received netlink message
* @cb: Netlink callback structure
* @policy_max: Maximum attribute type to be expected
* @nlmsg_policy: Validation policy
* @nltb: Array to store policy_max parsed elements
* @msg_type: Type of netlink message
*
* Returns 0 on success or a negative error code
*/
int iwpm_parse_nlmsg(struct netlink_callback *cb, int policy_max,
const struct nla_policy *nlmsg_policy,
struct nlattr *nltb[], const char *msg_type);
/**
* iwpm_print_sockaddr - Print IPv4/IPv6 address and TCP port
* @sockaddr: Socket address to print
* @msg: Message to print
*/
void iwpm_print_sockaddr(struct sockaddr_storage *sockaddr, char *msg);
#endif
File diff suppressed because it is too large. Load diff
+226
View File
@@ -0,0 +1,226 @@
/*
* Copyright (c) 2004, 2005, Voltaire, Inc. All rights reserved.
* Copyright (c) 2005 Intel Corporation. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009 HNR Consulting. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef __IB_MAD_PRIV_H__
#define __IB_MAD_PRIV_H__
#include <linux/completion.h>
#include <linux/err.h>
#include <linux/workqueue.h>
#include <rdma/ib_mad.h>
#include <rdma/ib_smi.h>
#include <rdma/opa_smi.h>
#define IB_MAD_QPS_CORE 2 /* Always QP0 and QP1 as a minimum */
/* QP and CQ parameters */
#define IB_MAD_QP_SEND_SIZE 128
#define IB_MAD_QP_RECV_SIZE 512
#define IB_MAD_QP_MIN_SIZE 64
#define IB_MAD_QP_MAX_SIZE 8192
#define IB_MAD_SEND_REQ_MAX_SG 2
#define IB_MAD_RECV_REQ_MAX_SG 1
#define IB_MAD_SEND_Q_PSN 0
/* Registration table sizes */
#define MAX_MGMT_CLASS 80
#define MAX_MGMT_VERSION 0x83
#define MAX_MGMT_OUI 8
#define MAX_MGMT_VENDOR_RANGE2 (IB_MGMT_CLASS_VENDOR_RANGE2_END - \
IB_MGMT_CLASS_VENDOR_RANGE2_START + 1)
struct ib_mad_list_head {
struct list_head list;
struct ib_cqe cqe;
struct ib_mad_queue *mad_queue;
};
struct ib_mad_private_header {
struct ib_mad_list_head mad_list;
struct ib_mad_recv_wc recv_wc;
struct ib_wc wc;
u64 mapping;
} __attribute__ ((packed));
struct ib_mad_private {
struct ib_mad_private_header header;
size_t mad_size;
struct ib_grh grh;
u8 mad[0];
} __attribute__ ((packed));
struct ib_rmpp_segment {
struct list_head list;
u32 num;
u8 data[0];
};
struct ib_mad_agent_private {
struct ib_mad_agent agent;
struct ib_mad_reg_req *reg_req;
struct ib_mad_qp_info *qp_info;
spinlock_t lock;
struct list_head send_list;
struct list_head wait_list;
struct list_head done_list;
struct delayed_work timed_work;
unsigned long timeout;
struct list_head local_list;
struct work_struct local_work;
struct list_head rmpp_list;
atomic_t refcount;
union {
struct completion comp;
struct rcu_head rcu;
};
};
struct ib_mad_snoop_private {
struct ib_mad_agent agent;
struct ib_mad_qp_info *qp_info;
int snoop_index;
int mad_snoop_flags;
atomic_t refcount;
struct completion comp;
};
struct ib_mad_send_wr_private {
struct ib_mad_list_head mad_list;
struct list_head agent_list;
struct ib_mad_agent_private *mad_agent_priv;
struct ib_mad_send_buf send_buf;
u64 header_mapping;
u64 payload_mapping;
struct ib_ud_wr send_wr;
struct ib_sge sg_list[IB_MAD_SEND_REQ_MAX_SG];
__be64 tid;
unsigned long timeout;
int max_retries;
int retries_left;
int retry;
int refcount;
enum ib_wc_status status;
/* RMPP control */
struct list_head rmpp_list;
struct ib_rmpp_segment *last_ack_seg;
struct ib_rmpp_segment *cur_seg;
int last_ack;
int seg_num;
int newwin;
int pad;
};
struct ib_mad_local_private {
struct list_head completion_list;
struct ib_mad_private *mad_priv;
struct ib_mad_agent_private *recv_mad_agent;
struct ib_mad_send_wr_private *mad_send_wr;
size_t return_wc_byte_len;
};
struct ib_mad_mgmt_method_table {
struct ib_mad_agent_private *agent[IB_MGMT_MAX_METHODS];
};
struct ib_mad_mgmt_class_table {
struct ib_mad_mgmt_method_table *method_table[MAX_MGMT_CLASS];
};
struct ib_mad_mgmt_vendor_class {
u8 oui[MAX_MGMT_OUI][3];
struct ib_mad_mgmt_method_table *method_table[MAX_MGMT_OUI];
};
struct ib_mad_mgmt_vendor_class_table {
struct ib_mad_mgmt_vendor_class *vendor_class[MAX_MGMT_VENDOR_RANGE2];
};
struct ib_mad_mgmt_version_table {
struct ib_mad_mgmt_class_table *class;
struct ib_mad_mgmt_vendor_class_table *vendor;
};
struct ib_mad_queue {
spinlock_t lock;
struct list_head list;
int count;
int max_active;
struct ib_mad_qp_info *qp_info;
};
struct ib_mad_qp_info {
struct ib_mad_port_private *port_priv;
struct ib_qp *qp;
struct ib_mad_queue send_queue;
struct ib_mad_queue recv_queue;
struct list_head overflow_list;
spinlock_t snoop_lock;
struct ib_mad_snoop_private **snoop_table;
int snoop_table_size;
atomic_t snoop_count;
};
struct ib_mad_port_private {
struct list_head port_list;
struct ib_device *device;
int port_num;
struct ib_cq *cq;
struct ib_pd *pd;
spinlock_t reg_lock;
struct ib_mad_mgmt_version_table version[MAX_MGMT_VERSION];
struct workqueue_struct *wq;
struct ib_mad_qp_info qp_info[IB_MAD_QPS_CORE];
};
int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr);
struct ib_mad_send_wr_private *
ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv,
const struct ib_mad_recv_wc *mad_recv_wc);
void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_send_wc *mad_send_wc);
void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr);
void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr,
int timeout_ms);
#endif /* __IB_MAD_PRIV_H__ */
+968
View File
@@ -0,0 +1,968 @@
/*
* Copyright (c) 2005 Intel Inc. All rights reserved.
* Copyright (c) 2005-2006 Voltaire, Inc. All rights reserved.
* Copyright (c) 2014 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/slab.h>
#include "mad_priv.h"
#include "mad_rmpp.h"
enum rmpp_state {
RMPP_STATE_ACTIVE,
RMPP_STATE_TIMEOUT,
RMPP_STATE_COMPLETE,
RMPP_STATE_CANCELING
};
struct mad_rmpp_recv {
struct ib_mad_agent_private *agent;
struct list_head list;
struct delayed_work timeout_work;
struct delayed_work cleanup_work;
struct completion comp;
enum rmpp_state state;
spinlock_t lock;
atomic_t refcount;
struct ib_ah *ah;
struct ib_mad_recv_wc *rmpp_wc;
struct ib_mad_recv_buf *cur_seg_buf;
int last_ack;
int seg_num;
int newwin;
int repwin;
__be64 tid;
u32 src_qp;
u32 slid;
u8 mgmt_class;
u8 class_version;
u8 method;
u8 base_version;
};
static inline void deref_rmpp_recv(struct mad_rmpp_recv *rmpp_recv)
{
if (atomic_dec_and_test(&rmpp_recv->refcount))
complete(&rmpp_recv->comp);
}
static void destroy_rmpp_recv(struct mad_rmpp_recv *rmpp_recv)
{
deref_rmpp_recv(rmpp_recv);
wait_for_completion(&rmpp_recv->comp);
rdma_destroy_ah(rmpp_recv->ah);
kfree(rmpp_recv);
}
void ib_cancel_rmpp_recvs(struct ib_mad_agent_private *agent)
{
struct mad_rmpp_recv *rmpp_recv, *temp_rmpp_recv;
unsigned long flags;
spin_lock_irqsave(&agent->lock, flags);
list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
if (rmpp_recv->state != RMPP_STATE_COMPLETE)
ib_free_recv_mad(rmpp_recv->rmpp_wc);
rmpp_recv->state = RMPP_STATE_CANCELING;
}
spin_unlock_irqrestore(&agent->lock, flags);
list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
cancel_delayed_work(&rmpp_recv->timeout_work);
cancel_delayed_work(&rmpp_recv->cleanup_work);
}
flush_workqueue(agent->qp_info->port_priv->wq);
list_for_each_entry_safe(rmpp_recv, temp_rmpp_recv,
&agent->rmpp_list, list) {
list_del(&rmpp_recv->list);
destroy_rmpp_recv(rmpp_recv);
}
}
static void format_ack(struct ib_mad_send_buf *msg,
struct ib_rmpp_mad *data,
struct mad_rmpp_recv *rmpp_recv)
{
struct ib_rmpp_mad *ack = msg->mad;
unsigned long flags;
memcpy(ack, &data->mad_hdr, msg->hdr_len);
ack->mad_hdr.method ^= IB_MGMT_METHOD_RESP;
ack->rmpp_hdr.rmpp_type = IB_MGMT_RMPP_TYPE_ACK;
ib_set_rmpp_flags(&ack->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE);
spin_lock_irqsave(&rmpp_recv->lock, flags);
rmpp_recv->last_ack = rmpp_recv->seg_num;
ack->rmpp_hdr.seg_num = cpu_to_be32(rmpp_recv->seg_num);
ack->rmpp_hdr.paylen_newwin = cpu_to_be32(rmpp_recv->newwin);
spin_unlock_irqrestore(&rmpp_recv->lock, flags);
}
static void ack_recv(struct mad_rmpp_recv *rmpp_recv,
struct ib_mad_recv_wc *recv_wc)
{
struct ib_mad_send_buf *msg;
int ret, hdr_len;
hdr_len = ib_get_mad_data_offset(recv_wc->recv_buf.mad->mad_hdr.mgmt_class);
msg = ib_create_send_mad(&rmpp_recv->agent->agent, recv_wc->wc->src_qp,
recv_wc->wc->pkey_index, 1, hdr_len,
0, GFP_KERNEL,
IB_MGMT_BASE_VERSION);
if (IS_ERR(msg))
return;
format_ack(msg, (struct ib_rmpp_mad *) recv_wc->recv_buf.mad, rmpp_recv);
msg->ah = rmpp_recv->ah;
ret = ib_post_send_mad(msg, NULL);
if (ret)
ib_free_send_mad(msg);
}
static struct ib_mad_send_buf *alloc_response_msg(struct ib_mad_agent *agent,
struct ib_mad_recv_wc *recv_wc)
{
struct ib_mad_send_buf *msg;
struct ib_ah *ah;
int hdr_len;
ah = ib_create_ah_from_wc(agent->qp->pd, recv_wc->wc,
recv_wc->recv_buf.grh, agent->port_num);
if (IS_ERR(ah))
return (void *) ah;
hdr_len = ib_get_mad_data_offset(recv_wc->recv_buf.mad->mad_hdr.mgmt_class);
msg = ib_create_send_mad(agent, recv_wc->wc->src_qp,
recv_wc->wc->pkey_index, 1,
hdr_len, 0, GFP_KERNEL,
IB_MGMT_BASE_VERSION);
if (IS_ERR(msg))
rdma_destroy_ah(ah);
else {
msg->ah = ah;
msg->context[0] = ah;
}
return msg;
}
static void ack_ds_ack(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *recv_wc)
{
struct ib_mad_send_buf *msg;
struct ib_rmpp_mad *rmpp_mad;
int ret;
msg = alloc_response_msg(&agent->agent, recv_wc);
if (IS_ERR(msg))
return;
rmpp_mad = msg->mad;
memcpy(rmpp_mad, recv_wc->recv_buf.mad, msg->hdr_len);
rmpp_mad->mad_hdr.method ^= IB_MGMT_METHOD_RESP;
ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE);
rmpp_mad->rmpp_hdr.seg_num = 0;
rmpp_mad->rmpp_hdr.paylen_newwin = cpu_to_be32(1);
ret = ib_post_send_mad(msg, NULL);
if (ret) {
rdma_destroy_ah(msg->ah);
ib_free_send_mad(msg);
}
}
void ib_rmpp_send_handler(struct ib_mad_send_wc *mad_send_wc)
{
if (mad_send_wc->send_buf->context[0] == mad_send_wc->send_buf->ah)
rdma_destroy_ah(mad_send_wc->send_buf->ah);
ib_free_send_mad(mad_send_wc->send_buf);
}
static void nack_recv(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *recv_wc, u8 rmpp_status)
{
struct ib_mad_send_buf *msg;
struct ib_rmpp_mad *rmpp_mad;
int ret;
msg = alloc_response_msg(&agent->agent, recv_wc);
if (IS_ERR(msg))
return;
rmpp_mad = msg->mad;
memcpy(rmpp_mad, recv_wc->recv_buf.mad, msg->hdr_len);
rmpp_mad->mad_hdr.method ^= IB_MGMT_METHOD_RESP;
rmpp_mad->rmpp_hdr.rmpp_version = IB_MGMT_RMPP_VERSION;
rmpp_mad->rmpp_hdr.rmpp_type = IB_MGMT_RMPP_TYPE_ABORT;
ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE);
rmpp_mad->rmpp_hdr.rmpp_status = rmpp_status;
rmpp_mad->rmpp_hdr.seg_num = 0;
rmpp_mad->rmpp_hdr.paylen_newwin = 0;
ret = ib_post_send_mad(msg, NULL);
if (ret) {
rdma_destroy_ah(msg->ah);
ib_free_send_mad(msg);
}
}
static void recv_timeout_handler(struct work_struct *work)
{
struct mad_rmpp_recv *rmpp_recv =
container_of(work, struct mad_rmpp_recv, timeout_work.work);
struct ib_mad_recv_wc *rmpp_wc;
unsigned long flags;
spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
if (rmpp_recv->state != RMPP_STATE_ACTIVE) {
spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
return;
}
rmpp_recv->state = RMPP_STATE_TIMEOUT;
list_del(&rmpp_recv->list);
spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
rmpp_wc = rmpp_recv->rmpp_wc;
nack_recv(rmpp_recv->agent, rmpp_wc, IB_MGMT_RMPP_STATUS_T2L);
destroy_rmpp_recv(rmpp_recv);
ib_free_recv_mad(rmpp_wc);
}
static void recv_cleanup_handler(struct work_struct *work)
{
struct mad_rmpp_recv *rmpp_recv =
container_of(work, struct mad_rmpp_recv, cleanup_work.work);
unsigned long flags;
spin_lock_irqsave(&rmpp_recv->agent->lock, flags);
if (rmpp_recv->state == RMPP_STATE_CANCELING) {
spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
return;
}
list_del(&rmpp_recv->list);
spin_unlock_irqrestore(&rmpp_recv->agent->lock, flags);
destroy_rmpp_recv(rmpp_recv);
}
static struct mad_rmpp_recv *
create_rmpp_recv(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct mad_rmpp_recv *rmpp_recv;
struct ib_mad_hdr *mad_hdr;
rmpp_recv = kmalloc(sizeof *rmpp_recv, GFP_KERNEL);
if (!rmpp_recv)
return NULL;
rmpp_recv->ah = ib_create_ah_from_wc(agent->agent.qp->pd,
mad_recv_wc->wc,
mad_recv_wc->recv_buf.grh,
agent->agent.port_num);
if (IS_ERR(rmpp_recv->ah))
goto error;
rmpp_recv->agent = agent;
init_completion(&rmpp_recv->comp);
INIT_DELAYED_WORK(&rmpp_recv->timeout_work, recv_timeout_handler);
INIT_DELAYED_WORK(&rmpp_recv->cleanup_work, recv_cleanup_handler);
spin_lock_init(&rmpp_recv->lock);
rmpp_recv->state = RMPP_STATE_ACTIVE;
atomic_set(&rmpp_recv->refcount, 1);
rmpp_recv->rmpp_wc = mad_recv_wc;
rmpp_recv->cur_seg_buf = &mad_recv_wc->recv_buf;
rmpp_recv->newwin = 1;
rmpp_recv->seg_num = 1;
rmpp_recv->last_ack = 0;
rmpp_recv->repwin = 1;
mad_hdr = &mad_recv_wc->recv_buf.mad->mad_hdr;
rmpp_recv->tid = mad_hdr->tid;
rmpp_recv->src_qp = mad_recv_wc->wc->src_qp;
rmpp_recv->slid = mad_recv_wc->wc->slid;
rmpp_recv->mgmt_class = mad_hdr->mgmt_class;
rmpp_recv->class_version = mad_hdr->class_version;
rmpp_recv->method = mad_hdr->method;
rmpp_recv->base_version = mad_hdr->base_version;
return rmpp_recv;
error: kfree(rmpp_recv);
return NULL;
}
static struct mad_rmpp_recv *
find_rmpp_recv(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct mad_rmpp_recv *rmpp_recv;
struct ib_mad_hdr *mad_hdr = &mad_recv_wc->recv_buf.mad->mad_hdr;
list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
if (rmpp_recv->tid == mad_hdr->tid &&
rmpp_recv->src_qp == mad_recv_wc->wc->src_qp &&
rmpp_recv->slid == mad_recv_wc->wc->slid &&
rmpp_recv->mgmt_class == mad_hdr->mgmt_class &&
rmpp_recv->class_version == mad_hdr->class_version &&
rmpp_recv->method == mad_hdr->method)
return rmpp_recv;
}
return NULL;
}
static struct mad_rmpp_recv *
acquire_rmpp_recv(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct mad_rmpp_recv *rmpp_recv;
unsigned long flags;
spin_lock_irqsave(&agent->lock, flags);
rmpp_recv = find_rmpp_recv(agent, mad_recv_wc);
if (rmpp_recv)
atomic_inc(&rmpp_recv->refcount);
spin_unlock_irqrestore(&agent->lock, flags);
return rmpp_recv;
}
static struct mad_rmpp_recv *
insert_rmpp_recv(struct ib_mad_agent_private *agent,
struct mad_rmpp_recv *rmpp_recv)
{
struct mad_rmpp_recv *cur_rmpp_recv;
cur_rmpp_recv = find_rmpp_recv(agent, rmpp_recv->rmpp_wc);
if (!cur_rmpp_recv)
list_add_tail(&rmpp_recv->list, &agent->rmpp_list);
return cur_rmpp_recv;
}
static inline int get_last_flag(struct ib_mad_recv_buf *seg)
{
struct ib_rmpp_mad *rmpp_mad;
rmpp_mad = (struct ib_rmpp_mad *) seg->mad;
return ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & IB_MGMT_RMPP_FLAG_LAST;
}
static inline int get_seg_num(struct ib_mad_recv_buf *seg)
{
struct ib_rmpp_mad *rmpp_mad;
rmpp_mad = (struct ib_rmpp_mad *) seg->mad;
return be32_to_cpu(rmpp_mad->rmpp_hdr.seg_num);
}
static inline struct ib_mad_recv_buf * get_next_seg(struct list_head *rmpp_list,
struct ib_mad_recv_buf *seg)
{
if (seg->list.next == rmpp_list)
return NULL;
return container_of(seg->list.next, struct ib_mad_recv_buf, list);
}
static inline int window_size(struct ib_mad_agent_private *agent)
{
return max(agent->qp_info->recv_queue.max_active >> 3, 1);
}
static struct ib_mad_recv_buf * find_seg_location(struct list_head *rmpp_list,
int seg_num)
{
struct ib_mad_recv_buf *seg_buf;
int cur_seg_num;
list_for_each_entry_reverse(seg_buf, rmpp_list, list) {
cur_seg_num = get_seg_num(seg_buf);
if (seg_num > cur_seg_num)
return seg_buf;
if (seg_num == cur_seg_num)
break;
}
return NULL;
}
static void update_seg_num(struct mad_rmpp_recv *rmpp_recv,
struct ib_mad_recv_buf *new_buf)
{
struct list_head *rmpp_list = &rmpp_recv->rmpp_wc->rmpp_list;
while (new_buf && (get_seg_num(new_buf) == rmpp_recv->seg_num + 1)) {
rmpp_recv->cur_seg_buf = new_buf;
rmpp_recv->seg_num++;
new_buf = get_next_seg(rmpp_list, new_buf);
}
}
static inline int get_mad_len(struct mad_rmpp_recv *rmpp_recv)
{
struct ib_rmpp_mad *rmpp_mad;
int hdr_size, data_size, pad;
bool opa = rdma_cap_opa_mad(rmpp_recv->agent->qp_info->port_priv->device,
rmpp_recv->agent->qp_info->port_priv->port_num);
rmpp_mad = (struct ib_rmpp_mad *)rmpp_recv->cur_seg_buf->mad;
hdr_size = ib_get_mad_data_offset(rmpp_mad->mad_hdr.mgmt_class);
if (opa && rmpp_recv->base_version == OPA_MGMT_BASE_VERSION) {
data_size = sizeof(struct opa_rmpp_mad) - hdr_size;
pad = OPA_MGMT_RMPP_DATA - be32_to_cpu(rmpp_mad->rmpp_hdr.paylen_newwin);
if (pad > OPA_MGMT_RMPP_DATA || pad < 0)
pad = 0;
} else {
data_size = sizeof(struct ib_rmpp_mad) - hdr_size;
pad = IB_MGMT_RMPP_DATA - be32_to_cpu(rmpp_mad->rmpp_hdr.paylen_newwin);
if (pad > IB_MGMT_RMPP_DATA || pad < 0)
pad = 0;
}
return hdr_size + rmpp_recv->seg_num * data_size - pad;
}
static struct ib_mad_recv_wc * complete_rmpp(struct mad_rmpp_recv *rmpp_recv)
{
struct ib_mad_recv_wc *rmpp_wc;
ack_recv(rmpp_recv, rmpp_recv->rmpp_wc);
if (rmpp_recv->seg_num > 1)
cancel_delayed_work(&rmpp_recv->timeout_work);
rmpp_wc = rmpp_recv->rmpp_wc;
rmpp_wc->mad_len = get_mad_len(rmpp_recv);
/* 10 seconds until we can find the packet lifetime */
queue_delayed_work(rmpp_recv->agent->qp_info->port_priv->wq,
&rmpp_recv->cleanup_work, msecs_to_jiffies(10000));
return rmpp_wc;
}
static struct ib_mad_recv_wc *
continue_rmpp(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct mad_rmpp_recv *rmpp_recv;
struct ib_mad_recv_buf *prev_buf;
struct ib_mad_recv_wc *done_wc;
int seg_num;
unsigned long flags;
rmpp_recv = acquire_rmpp_recv(agent, mad_recv_wc);
if (!rmpp_recv)
goto drop1;
seg_num = get_seg_num(&mad_recv_wc->recv_buf);
spin_lock_irqsave(&rmpp_recv->lock, flags);
if ((rmpp_recv->state == RMPP_STATE_TIMEOUT) ||
(seg_num > rmpp_recv->newwin))
goto drop3;
if ((seg_num <= rmpp_recv->last_ack) ||
(rmpp_recv->state == RMPP_STATE_COMPLETE)) {
spin_unlock_irqrestore(&rmpp_recv->lock, flags);
ack_recv(rmpp_recv, mad_recv_wc);
goto drop2;
}
prev_buf = find_seg_location(&rmpp_recv->rmpp_wc->rmpp_list, seg_num);
if (!prev_buf)
goto drop3;
done_wc = NULL;
list_add(&mad_recv_wc->recv_buf.list, &prev_buf->list);
if (rmpp_recv->cur_seg_buf == prev_buf) {
update_seg_num(rmpp_recv, &mad_recv_wc->recv_buf);
if (get_last_flag(rmpp_recv->cur_seg_buf)) {
rmpp_recv->state = RMPP_STATE_COMPLETE;
spin_unlock_irqrestore(&rmpp_recv->lock, flags);
done_wc = complete_rmpp(rmpp_recv);
goto out;
} else if (rmpp_recv->seg_num == rmpp_recv->newwin) {
rmpp_recv->newwin += window_size(agent);
spin_unlock_irqrestore(&rmpp_recv->lock, flags);
ack_recv(rmpp_recv, mad_recv_wc);
goto out;
}
}
spin_unlock_irqrestore(&rmpp_recv->lock, flags);
out:
deref_rmpp_recv(rmpp_recv);
return done_wc;
drop3: spin_unlock_irqrestore(&rmpp_recv->lock, flags);
drop2: deref_rmpp_recv(rmpp_recv);
drop1: ib_free_recv_mad(mad_recv_wc);
return NULL;
}
static struct ib_mad_recv_wc *
start_rmpp(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct mad_rmpp_recv *rmpp_recv;
unsigned long flags;
rmpp_recv = create_rmpp_recv(agent, mad_recv_wc);
if (!rmpp_recv) {
ib_free_recv_mad(mad_recv_wc);
return NULL;
}
spin_lock_irqsave(&agent->lock, flags);
if (insert_rmpp_recv(agent, rmpp_recv)) {
spin_unlock_irqrestore(&agent->lock, flags);
/* duplicate first MAD */
destroy_rmpp_recv(rmpp_recv);
return continue_rmpp(agent, mad_recv_wc);
}
atomic_inc(&rmpp_recv->refcount);
if (get_last_flag(&mad_recv_wc->recv_buf)) {
rmpp_recv->state = RMPP_STATE_COMPLETE;
spin_unlock_irqrestore(&agent->lock, flags);
complete_rmpp(rmpp_recv);
} else {
spin_unlock_irqrestore(&agent->lock, flags);
/* 40 seconds until we can find the packet lifetimes */
queue_delayed_work(agent->qp_info->port_priv->wq,
&rmpp_recv->timeout_work,
msecs_to_jiffies(40000));
rmpp_recv->newwin += window_size(agent);
ack_recv(rmpp_recv, mad_recv_wc);
mad_recv_wc = NULL;
}
deref_rmpp_recv(rmpp_recv);
return mad_recv_wc;
}
static int send_next_seg(struct ib_mad_send_wr_private *mad_send_wr)
{
struct ib_rmpp_mad *rmpp_mad;
int timeout;
u32 paylen = 0;
rmpp_mad = mad_send_wr->send_buf.mad;
ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE);
rmpp_mad->rmpp_hdr.seg_num = cpu_to_be32(++mad_send_wr->seg_num);
if (mad_send_wr->seg_num == 1) {
rmpp_mad->rmpp_hdr.rmpp_rtime_flags |= IB_MGMT_RMPP_FLAG_FIRST;
paylen = (mad_send_wr->send_buf.seg_count *
mad_send_wr->send_buf.seg_rmpp_size) -
mad_send_wr->pad;
}
if (mad_send_wr->seg_num == mad_send_wr->send_buf.seg_count) {
rmpp_mad->rmpp_hdr.rmpp_rtime_flags |= IB_MGMT_RMPP_FLAG_LAST;
paylen = mad_send_wr->send_buf.seg_rmpp_size - mad_send_wr->pad;
}
rmpp_mad->rmpp_hdr.paylen_newwin = cpu_to_be32(paylen);
/* 2 seconds for an ACK until we can find the packet lifetime */
timeout = mad_send_wr->send_buf.timeout_ms;
if (!timeout || timeout > 2000)
mad_send_wr->timeout = msecs_to_jiffies(2000);
return ib_send_mad(mad_send_wr);
}
static void abort_send(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc, u8 rmpp_status)
{
struct ib_mad_send_wr_private *mad_send_wr;
struct ib_mad_send_wc wc;
unsigned long flags;
spin_lock_irqsave(&agent->lock, flags);
mad_send_wr = ib_find_send_mad(agent, mad_recv_wc);
if (!mad_send_wr)
goto out; /* Unmatched send */
if ((mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count) ||
(!mad_send_wr->timeout) || (mad_send_wr->status != IB_WC_SUCCESS))
goto out; /* Send is already done */
ib_mark_mad_done(mad_send_wr);
spin_unlock_irqrestore(&agent->lock, flags);
wc.status = IB_WC_REM_ABORT_ERR;
wc.vendor_err = rmpp_status;
wc.send_buf = &mad_send_wr->send_buf;
ib_mad_complete_send_wr(mad_send_wr, &wc);
return;
out:
spin_unlock_irqrestore(&agent->lock, flags);
}
static inline void adjust_last_ack(struct ib_mad_send_wr_private *wr,
int seg_num)
{
struct list_head *list;
wr->last_ack = seg_num;
list = &wr->last_ack_seg->list;
list_for_each_entry(wr->last_ack_seg, list, list)
if (wr->last_ack_seg->num == seg_num)
break;
}
static void process_ds_ack(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc, int newwin)
{
struct mad_rmpp_recv *rmpp_recv;
rmpp_recv = find_rmpp_recv(agent, mad_recv_wc);
if (rmpp_recv && rmpp_recv->state == RMPP_STATE_COMPLETE)
rmpp_recv->repwin = newwin;
}
static void process_rmpp_ack(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct ib_mad_send_wr_private *mad_send_wr;
struct ib_rmpp_mad *rmpp_mad;
unsigned long flags;
int seg_num, newwin, ret;
rmpp_mad = (struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad;
if (rmpp_mad->rmpp_hdr.rmpp_status) {
abort_send(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_BAD_STATUS);
nack_recv(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_BAD_STATUS);
return;
}
seg_num = be32_to_cpu(rmpp_mad->rmpp_hdr.seg_num);
newwin = be32_to_cpu(rmpp_mad->rmpp_hdr.paylen_newwin);
if (newwin < seg_num) {
abort_send(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_W2S);
nack_recv(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_W2S);
return;
}
spin_lock_irqsave(&agent->lock, flags);
mad_send_wr = ib_find_send_mad(agent, mad_recv_wc);
if (!mad_send_wr) {
if (!seg_num)
process_ds_ack(agent, mad_recv_wc, newwin);
goto out; /* Unmatched or DS RMPP ACK */
}
if ((mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count) &&
(mad_send_wr->timeout)) {
spin_unlock_irqrestore(&agent->lock, flags);
ack_ds_ack(agent, mad_recv_wc);
return; /* Repeated ACK for DS RMPP transaction */
}
if ((mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count) ||
(!mad_send_wr->timeout) || (mad_send_wr->status != IB_WC_SUCCESS))
goto out; /* Send is already done */
if (seg_num > mad_send_wr->send_buf.seg_count ||
seg_num > mad_send_wr->newwin) {
spin_unlock_irqrestore(&agent->lock, flags);
abort_send(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_S2B);
nack_recv(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_S2B);
return;
}
if (newwin < mad_send_wr->newwin || seg_num < mad_send_wr->last_ack)
goto out; /* Old ACK */
if (seg_num > mad_send_wr->last_ack) {
adjust_last_ack(mad_send_wr, seg_num);
mad_send_wr->retries_left = mad_send_wr->max_retries;
}
mad_send_wr->newwin = newwin;
if (mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count) {
/* If no response is expected, the ACK completes the send */
if (!mad_send_wr->send_buf.timeout_ms) {
struct ib_mad_send_wc wc;
ib_mark_mad_done(mad_send_wr);
spin_unlock_irqrestore(&agent->lock, flags);
wc.status = IB_WC_SUCCESS;
wc.vendor_err = 0;
wc.send_buf = &mad_send_wr->send_buf;
ib_mad_complete_send_wr(mad_send_wr, &wc);
return;
}
if (mad_send_wr->refcount == 1)
ib_reset_mad_timeout(mad_send_wr,
mad_send_wr->send_buf.timeout_ms);
spin_unlock_irqrestore(&agent->lock, flags);
ack_ds_ack(agent, mad_recv_wc);
return;
} else if (mad_send_wr->refcount == 1 &&
mad_send_wr->seg_num < mad_send_wr->newwin &&
mad_send_wr->seg_num < mad_send_wr->send_buf.seg_count) {
/* Send failure will just result in a timeout/retry */
ret = send_next_seg(mad_send_wr);
if (ret)
goto out;
mad_send_wr->refcount++;
list_move_tail(&mad_send_wr->agent_list,
&mad_send_wr->mad_agent_priv->send_list);
}
out:
spin_unlock_irqrestore(&agent->lock, flags);
}
static struct ib_mad_recv_wc *
process_rmpp_data(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct ib_rmpp_hdr *rmpp_hdr;
u8 rmpp_status;
rmpp_hdr = &((struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad)->rmpp_hdr;
if (rmpp_hdr->rmpp_status) {
rmpp_status = IB_MGMT_RMPP_STATUS_BAD_STATUS;
goto bad;
}
if (rmpp_hdr->seg_num == cpu_to_be32(1)) {
if (!(ib_get_rmpp_flags(rmpp_hdr) & IB_MGMT_RMPP_FLAG_FIRST)) {
rmpp_status = IB_MGMT_RMPP_STATUS_BAD_SEG;
goto bad;
}
return start_rmpp(agent, mad_recv_wc);
} else {
if (ib_get_rmpp_flags(rmpp_hdr) & IB_MGMT_RMPP_FLAG_FIRST) {
rmpp_status = IB_MGMT_RMPP_STATUS_BAD_SEG;
goto bad;
}
return continue_rmpp(agent, mad_recv_wc);
}
bad:
nack_recv(agent, mad_recv_wc, rmpp_status);
ib_free_recv_mad(mad_recv_wc);
return NULL;
}
static void process_rmpp_stop(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct ib_rmpp_mad *rmpp_mad;
rmpp_mad = (struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad;
if (rmpp_mad->rmpp_hdr.rmpp_status != IB_MGMT_RMPP_STATUS_RESX) {
abort_send(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_BAD_STATUS);
nack_recv(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_BAD_STATUS);
} else
abort_send(agent, mad_recv_wc, rmpp_mad->rmpp_hdr.rmpp_status);
}
static void process_rmpp_abort(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct ib_rmpp_mad *rmpp_mad;
rmpp_mad = (struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad;
if (rmpp_mad->rmpp_hdr.rmpp_status < IB_MGMT_RMPP_STATUS_ABORT_MIN ||
rmpp_mad->rmpp_hdr.rmpp_status > IB_MGMT_RMPP_STATUS_ABORT_MAX) {
abort_send(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_BAD_STATUS);
nack_recv(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_BAD_STATUS);
} else
abort_send(agent, mad_recv_wc, rmpp_mad->rmpp_hdr.rmpp_status);
}
struct ib_mad_recv_wc *
ib_process_rmpp_recv_wc(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc)
{
struct ib_rmpp_mad *rmpp_mad;
rmpp_mad = (struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad;
if (!(rmpp_mad->rmpp_hdr.rmpp_rtime_flags & IB_MGMT_RMPP_FLAG_ACTIVE))
return mad_recv_wc;
if (rmpp_mad->rmpp_hdr.rmpp_version != IB_MGMT_RMPP_VERSION) {
abort_send(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_UNV);
nack_recv(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_UNV);
goto out;
}
switch (rmpp_mad->rmpp_hdr.rmpp_type) {
case IB_MGMT_RMPP_TYPE_DATA:
return process_rmpp_data(agent, mad_recv_wc);
case IB_MGMT_RMPP_TYPE_ACK:
process_rmpp_ack(agent, mad_recv_wc);
break;
case IB_MGMT_RMPP_TYPE_STOP:
process_rmpp_stop(agent, mad_recv_wc);
break;
case IB_MGMT_RMPP_TYPE_ABORT:
process_rmpp_abort(agent, mad_recv_wc);
break;
default:
abort_send(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_BADT);
nack_recv(agent, mad_recv_wc, IB_MGMT_RMPP_STATUS_BADT);
break;
}
out:
ib_free_recv_mad(mad_recv_wc);
return NULL;
}
static int init_newwin(struct ib_mad_send_wr_private *mad_send_wr)
{
struct ib_mad_agent_private *agent = mad_send_wr->mad_agent_priv;
struct ib_mad_hdr *mad_hdr = mad_send_wr->send_buf.mad;
struct mad_rmpp_recv *rmpp_recv;
struct rdma_ah_attr ah_attr;
unsigned long flags;
int newwin = 1;
if (!(mad_hdr->method & IB_MGMT_METHOD_RESP))
goto out;
spin_lock_irqsave(&agent->lock, flags);
list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
if (rmpp_recv->tid != mad_hdr->tid ||
rmpp_recv->mgmt_class != mad_hdr->mgmt_class ||
rmpp_recv->class_version != mad_hdr->class_version ||
(rmpp_recv->method & IB_MGMT_METHOD_RESP))
continue;
if (rdma_query_ah(mad_send_wr->send_buf.ah, &ah_attr))
continue;
if (rmpp_recv->slid == rdma_ah_get_dlid(&ah_attr)) {
newwin = rmpp_recv->repwin;
break;
}
}
spin_unlock_irqrestore(&agent->lock, flags);
out:
return newwin;
}
int ib_send_rmpp_mad(struct ib_mad_send_wr_private *mad_send_wr)
{
struct ib_rmpp_mad *rmpp_mad;
int ret;
rmpp_mad = mad_send_wr->send_buf.mad;
if (!(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
IB_MGMT_RMPP_FLAG_ACTIVE))
return IB_RMPP_RESULT_UNHANDLED;
if (rmpp_mad->rmpp_hdr.rmpp_type != IB_MGMT_RMPP_TYPE_DATA) {
mad_send_wr->seg_num = 1;
return IB_RMPP_RESULT_INTERNAL;
}
mad_send_wr->newwin = init_newwin(mad_send_wr);
/* We need to wait for the final ACK even if there isn't a response */
mad_send_wr->refcount += (mad_send_wr->timeout == 0);
ret = send_next_seg(mad_send_wr);
if (!ret)
return IB_RMPP_RESULT_CONSUMED;
return ret;
}
int ib_process_rmpp_send_wc(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_send_wc *mad_send_wc)
{
struct ib_rmpp_mad *rmpp_mad;
int ret;
rmpp_mad = mad_send_wr->send_buf.mad;
if (!(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
IB_MGMT_RMPP_FLAG_ACTIVE))
return IB_RMPP_RESULT_UNHANDLED; /* RMPP not active */
if (rmpp_mad->rmpp_hdr.rmpp_type != IB_MGMT_RMPP_TYPE_DATA)
return IB_RMPP_RESULT_INTERNAL; /* ACK, STOP, or ABORT */
if (mad_send_wc->status != IB_WC_SUCCESS ||
mad_send_wr->status != IB_WC_SUCCESS)
return IB_RMPP_RESULT_PROCESSED; /* Canceled or send error */
if (!mad_send_wr->timeout)
return IB_RMPP_RESULT_PROCESSED; /* Response received */
if (mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count) {
mad_send_wr->timeout =
msecs_to_jiffies(mad_send_wr->send_buf.timeout_ms);
return IB_RMPP_RESULT_PROCESSED; /* Send done */
}
if (mad_send_wr->seg_num == mad_send_wr->newwin ||
mad_send_wr->seg_num == mad_send_wr->send_buf.seg_count)
return IB_RMPP_RESULT_PROCESSED; /* Wait for ACK */
ret = send_next_seg(mad_send_wr);
if (ret) {
mad_send_wc->status = IB_WC_GENERAL_ERR;
return IB_RMPP_RESULT_PROCESSED;
}
return IB_RMPP_RESULT_CONSUMED;
}
int ib_retry_rmpp(struct ib_mad_send_wr_private *mad_send_wr)
{
struct ib_rmpp_mad *rmpp_mad;
int ret;
rmpp_mad = mad_send_wr->send_buf.mad;
if (!(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
IB_MGMT_RMPP_FLAG_ACTIVE))
return IB_RMPP_RESULT_UNHANDLED; /* RMPP not active */
if (mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count)
return IB_RMPP_RESULT_PROCESSED;
mad_send_wr->seg_num = mad_send_wr->last_ack;
mad_send_wr->cur_seg = mad_send_wr->last_ack_seg;
ret = send_next_seg(mad_send_wr);
if (ret)
return IB_RMPP_RESULT_PROCESSED;
return IB_RMPP_RESULT_CONSUMED;
}
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright (c) 2005 Intel Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef __MAD_RMPP_H__
#define __MAD_RMPP_H__
enum {
IB_RMPP_RESULT_PROCESSED,
IB_RMPP_RESULT_CONSUMED,
IB_RMPP_RESULT_INTERNAL,
IB_RMPP_RESULT_UNHANDLED
};
int ib_send_rmpp_mad(struct ib_mad_send_wr_private *mad_send_wr);
struct ib_mad_recv_wc *
ib_process_rmpp_recv_wc(struct ib_mad_agent_private *agent,
struct ib_mad_recv_wc *mad_recv_wc);
int ib_process_rmpp_send_wc(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_send_wc *mad_send_wc);
void ib_rmpp_send_handler(struct ib_mad_send_wc *mad_send_wc);
void ib_cancel_rmpp_recvs(struct ib_mad_agent_private *agent);
int ib_retry_rmpp(struct ib_mad_send_wr_private *mad_send_wr);
#endif /* __MAD_RMPP_H__ */
+86
View File
@@ -0,0 +1,86 @@
/*
* Copyright (c) 2016 HGST, a Western Digital Company.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include <rdma/ib_verbs.h>
#include <rdma/mr_pool.h>
struct ib_mr *ib_mr_pool_get(struct ib_qp *qp, struct list_head *list)
{
struct ib_mr *mr;
unsigned long flags;
spin_lock_irqsave(&qp->mr_lock, flags);
mr = list_first_entry_or_null(list, struct ib_mr, qp_entry);
if (mr) {
list_del(&mr->qp_entry);
qp->mrs_used++;
}
spin_unlock_irqrestore(&qp->mr_lock, flags);
return mr;
}
EXPORT_SYMBOL(ib_mr_pool_get);
void ib_mr_pool_put(struct ib_qp *qp, struct list_head *list, struct ib_mr *mr)
{
unsigned long flags;
spin_lock_irqsave(&qp->mr_lock, flags);
list_add(&mr->qp_entry, list);
qp->mrs_used--;
spin_unlock_irqrestore(&qp->mr_lock, flags);
}
EXPORT_SYMBOL(ib_mr_pool_put);
int ib_mr_pool_init(struct ib_qp *qp, struct list_head *list, int nr,
enum ib_mr_type type, u32 max_num_sg)
{
struct ib_mr *mr;
unsigned long flags;
int ret, i;
for (i = 0; i < nr; i++) {
mr = ib_alloc_mr(qp->pd, type, max_num_sg);
if (IS_ERR(mr)) {
ret = PTR_ERR(mr);
goto out;
}
spin_lock_irqsave(&qp->mr_lock, flags);
list_add_tail(&mr->qp_entry, list);
spin_unlock_irqrestore(&qp->mr_lock, flags);
}
return 0;
out:
ib_mr_pool_destroy(qp, list);
return ret;
}
EXPORT_SYMBOL(ib_mr_pool_init);
void ib_mr_pool_destroy(struct ib_qp *qp, struct list_head *list)
{
struct ib_mr *mr;
unsigned long flags;
spin_lock_irqsave(&qp->mr_lock, flags);
while (!list_empty(list)) {
mr = list_first_entry(list, struct ib_mr, qp_entry);
list_del(&mr->qp_entry);
spin_unlock_irqrestore(&qp->mr_lock, flags);
ib_dereg_mr(mr);
spin_lock_irqsave(&qp->mr_lock, flags);
}
spin_unlock_irqrestore(&qp->mr_lock, flags);
}
EXPORT_SYMBOL(ib_mr_pool_destroy);
+908
View File
@@ -0,0 +1,908 @@
/*
* Copyright (c) 2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/completion.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/interrupt.h>
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/bitops.h>
#include <linux/random.h>
#include <rdma/ib_cache.h>
#include "sa.h"
static void mcast_add_one(struct ib_device *device);
static void mcast_remove_one(struct ib_device *device, void *client_data);
static struct ib_client mcast_client = {
.name = "ib_multicast",
.add = mcast_add_one,
.remove = mcast_remove_one
};
static struct ib_sa_client sa_client;
static struct workqueue_struct *mcast_wq;
static union ib_gid mgid0;
struct mcast_device;
struct mcast_port {
struct mcast_device *dev;
spinlock_t lock;
struct rb_root table;
atomic_t refcount;
struct completion comp;
u8 port_num;
};
struct mcast_device {
struct ib_device *device;
struct ib_event_handler event_handler;
int start_port;
int end_port;
struct mcast_port port[0];
};
enum mcast_state {
MCAST_JOINING,
MCAST_MEMBER,
MCAST_ERROR,
};
enum mcast_group_state {
MCAST_IDLE,
MCAST_BUSY,
MCAST_GROUP_ERROR,
MCAST_PKEY_EVENT
};
enum {
MCAST_INVALID_PKEY_INDEX = 0xFFFF
};
struct mcast_member;
struct mcast_group {
struct ib_sa_mcmember_rec rec;
struct rb_node node;
struct mcast_port *port;
spinlock_t lock;
struct work_struct work;
struct list_head pending_list;
struct list_head active_list;
struct mcast_member *last_join;
int members[NUM_JOIN_MEMBERSHIP_TYPES];
atomic_t refcount;
enum mcast_group_state state;
struct ib_sa_query *query;
u16 pkey_index;
u8 leave_state;
int retries;
};
struct mcast_member {
struct ib_sa_multicast multicast;
struct ib_sa_client *client;
struct mcast_group *group;
struct list_head list;
enum mcast_state state;
atomic_t refcount;
struct completion comp;
};
static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
void *context);
static void leave_handler(int status, struct ib_sa_mcmember_rec *rec,
void *context);
static struct mcast_group *mcast_find(struct mcast_port *port,
union ib_gid *mgid)
{
struct rb_node *node = port->table.rb_node;
struct mcast_group *group;
int ret;
while (node) {
group = rb_entry(node, struct mcast_group, node);
ret = memcmp(mgid->raw, group->rec.mgid.raw, sizeof *mgid);
if (!ret)
return group;
if (ret < 0)
node = node->rb_left;
else
node = node->rb_right;
}
return NULL;
}
static struct mcast_group *mcast_insert(struct mcast_port *port,
struct mcast_group *group,
int allow_duplicates)
{
struct rb_node **link = &port->table.rb_node;
struct rb_node *parent = NULL;
struct mcast_group *cur_group;
int ret;
while (*link) {
parent = *link;
cur_group = rb_entry(parent, struct mcast_group, node);
ret = memcmp(group->rec.mgid.raw, cur_group->rec.mgid.raw,
sizeof group->rec.mgid);
if (ret < 0)
link = &(*link)->rb_left;
else if (ret > 0)
link = &(*link)->rb_right;
else if (allow_duplicates)
link = &(*link)->rb_left;
else
return cur_group;
}
rb_link_node(&group->node, parent, link);
rb_insert_color(&group->node, &port->table);
return NULL;
}
static void deref_port(struct mcast_port *port)
{
if (atomic_dec_and_test(&port->refcount))
complete(&port->comp);
}
static void release_group(struct mcast_group *group)
{
struct mcast_port *port = group->port;
unsigned long flags;
spin_lock_irqsave(&port->lock, flags);
if (atomic_dec_and_test(&group->refcount)) {
rb_erase(&group->node, &port->table);
spin_unlock_irqrestore(&port->lock, flags);
kfree(group);
deref_port(port);
} else
spin_unlock_irqrestore(&port->lock, flags);
}
static void deref_member(struct mcast_member *member)
{
if (atomic_dec_and_test(&member->refcount))
complete(&member->comp);
}
static void queue_join(struct mcast_member *member)
{
struct mcast_group *group = member->group;
unsigned long flags;
spin_lock_irqsave(&group->lock, flags);
list_add_tail(&member->list, &group->pending_list);
if (group->state == MCAST_IDLE) {
group->state = MCAST_BUSY;
atomic_inc(&group->refcount);
queue_work(mcast_wq, &group->work);
}
spin_unlock_irqrestore(&group->lock, flags);
}
/*
* A multicast group has four types of members: full member, non member,
* sendonly non member and sendonly full member.
* We need to keep track of the number of members of each
* type based on their join state. Adjust the number of members the belong to
* the specified join states.
*/
static void adjust_membership(struct mcast_group *group, u8 join_state, int inc)
{
int i;
for (i = 0; i < NUM_JOIN_MEMBERSHIP_TYPES; i++, join_state >>= 1)
if (join_state & 0x1)
group->members[i] += inc;
}
/*
* If a multicast group has zero members left for a particular join state, but
* the group is still a member with the SA, we need to leave that join state.
* Determine which join states we still belong to, but that do not have any
* active members.
*/
static u8 get_leave_state(struct mcast_group *group)
{
u8 leave_state = 0;
int i;
for (i = 0; i < NUM_JOIN_MEMBERSHIP_TYPES; i++)
if (!group->members[i])
leave_state |= (0x1 << i);
return leave_state & group->rec.join_state;
}
static int check_selector(ib_sa_comp_mask comp_mask,
ib_sa_comp_mask selector_mask,
ib_sa_comp_mask value_mask,
u8 selector, u8 src_value, u8 dst_value)
{
int err;
if (!(comp_mask & selector_mask) || !(comp_mask & value_mask))
return 0;
switch (selector) {
case IB_SA_GT:
err = (src_value <= dst_value);
break;
case IB_SA_LT:
err = (src_value >= dst_value);
break;
case IB_SA_EQ:
err = (src_value != dst_value);
break;
default:
err = 0;
break;
}
return err;
}
static int cmp_rec(struct ib_sa_mcmember_rec *src,
struct ib_sa_mcmember_rec *dst, ib_sa_comp_mask comp_mask)
{
/* MGID must already match */
if (comp_mask & IB_SA_MCMEMBER_REC_PORT_GID &&
memcmp(&src->port_gid, &dst->port_gid, sizeof src->port_gid))
return -EINVAL;
if (comp_mask & IB_SA_MCMEMBER_REC_QKEY && src->qkey != dst->qkey)
return -EINVAL;
if (comp_mask & IB_SA_MCMEMBER_REC_MLID && src->mlid != dst->mlid)
return -EINVAL;
if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_MTU_SELECTOR,
IB_SA_MCMEMBER_REC_MTU, dst->mtu_selector,
src->mtu, dst->mtu))
return -EINVAL;
if (comp_mask & IB_SA_MCMEMBER_REC_TRAFFIC_CLASS &&
src->traffic_class != dst->traffic_class)
return -EINVAL;
if (comp_mask & IB_SA_MCMEMBER_REC_PKEY && src->pkey != dst->pkey)
return -EINVAL;
if (check_selector(comp_mask, IB_SA_MCMEMBER_REC_RATE_SELECTOR,
IB_SA_MCMEMBER_REC_RATE, dst->rate_selector,
src->rate, dst->rate))
return -EINVAL;
if (check_selector(comp_mask,
IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME_SELECTOR,
IB_SA_MCMEMBER_REC_PACKET_LIFE_TIME,
dst->packet_life_time_selector,
src->packet_life_time, dst->packet_life_time))
return -EINVAL;
if (comp_mask & IB_SA_MCMEMBER_REC_SL && src->sl != dst->sl)
return -EINVAL;
if (comp_mask & IB_SA_MCMEMBER_REC_FLOW_LABEL &&
src->flow_label != dst->flow_label)
return -EINVAL;
if (comp_mask & IB_SA_MCMEMBER_REC_HOP_LIMIT &&
src->hop_limit != dst->hop_limit)
return -EINVAL;
if (comp_mask & IB_SA_MCMEMBER_REC_SCOPE && src->scope != dst->scope)
return -EINVAL;
/* join_state checked separately, proxy_join ignored */
return 0;
}
static int send_join(struct mcast_group *group, struct mcast_member *member)
{
struct mcast_port *port = group->port;
int ret;
group->last_join = member;
ret = ib_sa_mcmember_rec_query(&sa_client, port->dev->device,
port->port_num, IB_MGMT_METHOD_SET,
&member->multicast.rec,
member->multicast.comp_mask,
3000, GFP_KERNEL, join_handler, group,
&group->query);
return (ret > 0) ? 0 : ret;
}
static int send_leave(struct mcast_group *group, u8 leave_state)
{
struct mcast_port *port = group->port;
struct ib_sa_mcmember_rec rec;
int ret;
rec = group->rec;
rec.join_state = leave_state;
group->leave_state = leave_state;
ret = ib_sa_mcmember_rec_query(&sa_client, port->dev->device,
port->port_num, IB_SA_METHOD_DELETE, &rec,
IB_SA_MCMEMBER_REC_MGID |
IB_SA_MCMEMBER_REC_PORT_GID |
IB_SA_MCMEMBER_REC_JOIN_STATE,
3000, GFP_KERNEL, leave_handler,
group, &group->query);
return (ret > 0) ? 0 : ret;
}
static void join_group(struct mcast_group *group, struct mcast_member *member,
u8 join_state)
{
member->state = MCAST_MEMBER;
adjust_membership(group, join_state, 1);
group->rec.join_state |= join_state;
member->multicast.rec = group->rec;
member->multicast.rec.join_state = join_state;
list_move(&member->list, &group->active_list);
}
static int fail_join(struct mcast_group *group, struct mcast_member *member,
int status)
{
spin_lock_irq(&group->lock);
list_del_init(&member->list);
spin_unlock_irq(&group->lock);
return member->multicast.callback(status, &member->multicast);
}
static void process_group_error(struct mcast_group *group)
{
struct mcast_member *member;
int ret = 0;
u16 pkey_index;
if (group->state == MCAST_PKEY_EVENT)
ret = ib_find_pkey(group->port->dev->device,
group->port->port_num,
be16_to_cpu(group->rec.pkey), &pkey_index);
spin_lock_irq(&group->lock);
if (group->state == MCAST_PKEY_EVENT && !ret &&
group->pkey_index == pkey_index)
goto out;
while (!list_empty(&group->active_list)) {
member = list_entry(group->active_list.next,
struct mcast_member, list);
atomic_inc(&member->refcount);
list_del_init(&member->list);
adjust_membership(group, member->multicast.rec.join_state, -1);
member->state = MCAST_ERROR;
spin_unlock_irq(&group->lock);
ret = member->multicast.callback(-ENETRESET,
&member->multicast);
deref_member(member);
if (ret)
ib_sa_free_multicast(&member->multicast);
spin_lock_irq(&group->lock);
}
group->rec.join_state = 0;
out:
group->state = MCAST_BUSY;
spin_unlock_irq(&group->lock);
}
static void mcast_work_handler(struct work_struct *work)
{
struct mcast_group *group;
struct mcast_member *member;
struct ib_sa_multicast *multicast;
int status, ret;
u8 join_state;
group = container_of(work, typeof(*group), work);
retest:
spin_lock_irq(&group->lock);
while (!list_empty(&group->pending_list) ||
(group->state != MCAST_BUSY)) {
if (group->state != MCAST_BUSY) {
spin_unlock_irq(&group->lock);
process_group_error(group);
goto retest;
}
member = list_entry(group->pending_list.next,
struct mcast_member, list);
multicast = &member->multicast;
join_state = multicast->rec.join_state;
atomic_inc(&member->refcount);
if (join_state == (group->rec.join_state & join_state)) {
status = cmp_rec(&group->rec, &multicast->rec,
multicast->comp_mask);
if (!status)
join_group(group, member, join_state);
else
list_del_init(&member->list);
spin_unlock_irq(&group->lock);
ret = multicast->callback(status, multicast);
} else {
spin_unlock_irq(&group->lock);
status = send_join(group, member);
if (!status) {
deref_member(member);
return;
}
ret = fail_join(group, member, status);
}
deref_member(member);
if (ret)
ib_sa_free_multicast(&member->multicast);
spin_lock_irq(&group->lock);
}
join_state = get_leave_state(group);
if (join_state) {
group->rec.join_state &= ~join_state;
spin_unlock_irq(&group->lock);
if (send_leave(group, join_state))
goto retest;
} else {
group->state = MCAST_IDLE;
spin_unlock_irq(&group->lock);
release_group(group);
}
}
/*
* Fail a join request if it is still active - at the head of the pending queue.
*/
static void process_join_error(struct mcast_group *group, int status)
{
struct mcast_member *member;
int ret;
spin_lock_irq(&group->lock);
member = list_entry(group->pending_list.next,
struct mcast_member, list);
if (group->last_join == member) {
atomic_inc(&member->refcount);
list_del_init(&member->list);
spin_unlock_irq(&group->lock);
ret = member->multicast.callback(status, &member->multicast);
deref_member(member);
if (ret)
ib_sa_free_multicast(&member->multicast);
} else
spin_unlock_irq(&group->lock);
}
static void join_handler(int status, struct ib_sa_mcmember_rec *rec,
void *context)
{
struct mcast_group *group = context;
u16 pkey_index = MCAST_INVALID_PKEY_INDEX;
if (status)
process_join_error(group, status);
else {
int mgids_changed, is_mgid0;
if (ib_find_pkey(group->port->dev->device,
group->port->port_num, be16_to_cpu(rec->pkey),
&pkey_index))
pkey_index = MCAST_INVALID_PKEY_INDEX;
spin_lock_irq(&group->port->lock);
if (group->state == MCAST_BUSY &&
group->pkey_index == MCAST_INVALID_PKEY_INDEX)
group->pkey_index = pkey_index;
mgids_changed = memcmp(&rec->mgid, &group->rec.mgid,
sizeof(group->rec.mgid));
group->rec = *rec;
if (mgids_changed) {
rb_erase(&group->node, &group->port->table);
is_mgid0 = !memcmp(&mgid0, &group->rec.mgid,
sizeof(mgid0));
mcast_insert(group->port, group, is_mgid0);
}
spin_unlock_irq(&group->port->lock);
}
mcast_work_handler(&group->work);
}
static void leave_handler(int status, struct ib_sa_mcmember_rec *rec,
void *context)
{
struct mcast_group *group = context;
if (status && group->retries > 0 &&
!send_leave(group, group->leave_state))
group->retries--;
else
mcast_work_handler(&group->work);
}
static struct mcast_group *acquire_group(struct mcast_port *port,
union ib_gid *mgid, gfp_t gfp_mask)
{
struct mcast_group *group, *cur_group;
unsigned long flags;
int is_mgid0;
is_mgid0 = !memcmp(&mgid0, mgid, sizeof mgid0);
if (!is_mgid0) {
spin_lock_irqsave(&port->lock, flags);
group = mcast_find(port, mgid);
if (group)
goto found;
spin_unlock_irqrestore(&port->lock, flags);
}
group = kzalloc(sizeof *group, gfp_mask);
if (!group)
return NULL;
group->retries = 3;
group->port = port;
group->rec.mgid = *mgid;
group->pkey_index = MCAST_INVALID_PKEY_INDEX;
INIT_LIST_HEAD(&group->pending_list);
INIT_LIST_HEAD(&group->active_list);
INIT_WORK(&group->work, mcast_work_handler);
spin_lock_init(&group->lock);
spin_lock_irqsave(&port->lock, flags);
cur_group = mcast_insert(port, group, is_mgid0);
if (cur_group) {
kfree(group);
group = cur_group;
} else
atomic_inc(&port->refcount);
found:
atomic_inc(&group->refcount);
spin_unlock_irqrestore(&port->lock, flags);
return group;
}
/*
* We serialize all join requests to a single group to make our lives much
* easier. Otherwise, two users could try to join the same group
* simultaneously, with different configurations, one could leave while the
* join is in progress, etc., which makes locking around error recovery
* difficult.
*/
struct ib_sa_multicast *
ib_sa_join_multicast(struct ib_sa_client *client,
struct ib_device *device, u8 port_num,
struct ib_sa_mcmember_rec *rec,
ib_sa_comp_mask comp_mask, gfp_t gfp_mask,
int (*callback)(int status,
struct ib_sa_multicast *multicast),
void *context)
{
struct mcast_device *dev;
struct mcast_member *member;
struct ib_sa_multicast *multicast;
int ret;
dev = ib_get_client_data(device, &mcast_client);
if (!dev)
return ERR_PTR(-ENODEV);
member = kmalloc(sizeof *member, gfp_mask);
if (!member)
return ERR_PTR(-ENOMEM);
ib_sa_client_get(client);
member->client = client;
member->multicast.rec = *rec;
member->multicast.comp_mask = comp_mask;
member->multicast.callback = callback;
member->multicast.context = context;
init_completion(&member->comp);
atomic_set(&member->refcount, 1);
member->state = MCAST_JOINING;
member->group = acquire_group(&dev->port[port_num - dev->start_port],
&rec->mgid, gfp_mask);
if (!member->group) {
ret = -ENOMEM;
goto err;
}
/*
* The user will get the multicast structure in their callback. They
* could then free the multicast structure before we can return from
* this routine. So we save the pointer to return before queuing
* any callback.
*/
multicast = &member->multicast;
queue_join(member);
return multicast;
err:
ib_sa_client_put(client);
kfree(member);
return ERR_PTR(ret);
}
EXPORT_SYMBOL(ib_sa_join_multicast);
void ib_sa_free_multicast(struct ib_sa_multicast *multicast)
{
struct mcast_member *member;
struct mcast_group *group;
member = container_of(multicast, struct mcast_member, multicast);
group = member->group;
spin_lock_irq(&group->lock);
if (member->state == MCAST_MEMBER)
adjust_membership(group, multicast->rec.join_state, -1);
list_del_init(&member->list);
if (group->state == MCAST_IDLE) {
group->state = MCAST_BUSY;
spin_unlock_irq(&group->lock);
/* Continue to hold reference on group until callback */
queue_work(mcast_wq, &group->work);
} else {
spin_unlock_irq(&group->lock);
release_group(group);
}
deref_member(member);
wait_for_completion(&member->comp);
ib_sa_client_put(member->client);
kfree(member);
}
EXPORT_SYMBOL(ib_sa_free_multicast);
int ib_sa_get_mcmember_rec(struct ib_device *device, u8 port_num,
union ib_gid *mgid, struct ib_sa_mcmember_rec *rec)
{
struct mcast_device *dev;
struct mcast_port *port;
struct mcast_group *group;
unsigned long flags;
int ret = 0;
dev = ib_get_client_data(device, &mcast_client);
if (!dev)
return -ENODEV;
port = &dev->port[port_num - dev->start_port];
spin_lock_irqsave(&port->lock, flags);
group = mcast_find(port, mgid);
if (group)
*rec = group->rec;
else
ret = -EADDRNOTAVAIL;
spin_unlock_irqrestore(&port->lock, flags);
return ret;
}
EXPORT_SYMBOL(ib_sa_get_mcmember_rec);
/**
* ib_init_ah_from_mcmember - Initialize AH attribute from multicast
* member record and gid of the device.
* @device: RDMA device
* @port_num: Port of the rdma device to consider
* @ndev: Optional netdevice, applicable only for RoCE
* @gid_type: GID type to consider
* @ah_attr: AH attribute to fillup on successful completion
*
* ib_init_ah_from_mcmember() initializes AH attribute based on multicast
* member record and other device properties. On success the caller is
* responsible to call rdma_destroy_ah_attr on the ah_attr. Returns 0 on
* success or appropriate error code.
*
*/
int ib_init_ah_from_mcmember(struct ib_device *device, u8 port_num,
struct ib_sa_mcmember_rec *rec,
struct net_device *ndev,
enum ib_gid_type gid_type,
struct rdma_ah_attr *ah_attr)
{
const struct ib_gid_attr *sgid_attr;
/* GID table is not based on the netdevice for IB link layer,
* so ignore ndev during search.
*/
if (rdma_protocol_ib(device, port_num))
ndev = NULL;
else if (!rdma_protocol_roce(device, port_num))
return -EINVAL;
sgid_attr = rdma_find_gid_by_port(device, &rec->port_gid,
gid_type, port_num, ndev);
if (IS_ERR(sgid_attr))
return PTR_ERR(sgid_attr);
memset(ah_attr, 0, sizeof(*ah_attr));
ah_attr->type = rdma_ah_find_type(device, port_num);
rdma_ah_set_dlid(ah_attr, be16_to_cpu(rec->mlid));
rdma_ah_set_sl(ah_attr, rec->sl);
rdma_ah_set_port_num(ah_attr, port_num);
rdma_ah_set_static_rate(ah_attr, rec->rate);
rdma_move_grh_sgid_attr(ah_attr, &rec->mgid,
be32_to_cpu(rec->flow_label),
rec->hop_limit, rec->traffic_class,
sgid_attr);
return 0;
}
EXPORT_SYMBOL(ib_init_ah_from_mcmember);
static void mcast_groups_event(struct mcast_port *port,
enum mcast_group_state state)
{
struct mcast_group *group;
struct rb_node *node;
unsigned long flags;
spin_lock_irqsave(&port->lock, flags);
for (node = rb_first(&port->table); node; node = rb_next(node)) {
group = rb_entry(node, struct mcast_group, node);
spin_lock(&group->lock);
if (group->state == MCAST_IDLE) {
atomic_inc(&group->refcount);
queue_work(mcast_wq, &group->work);
}
if (group->state != MCAST_GROUP_ERROR)
group->state = state;
spin_unlock(&group->lock);
}
spin_unlock_irqrestore(&port->lock, flags);
}
static void mcast_event_handler(struct ib_event_handler *handler,
struct ib_event *event)
{
struct mcast_device *dev;
int index;
dev = container_of(handler, struct mcast_device, event_handler);
if (!rdma_cap_ib_mcast(dev->device, event->element.port_num))
return;
index = event->element.port_num - dev->start_port;
switch (event->event) {
case IB_EVENT_PORT_ERR:
case IB_EVENT_LID_CHANGE:
case IB_EVENT_SM_CHANGE:
case IB_EVENT_CLIENT_REREGISTER:
mcast_groups_event(&dev->port[index], MCAST_GROUP_ERROR);
break;
case IB_EVENT_PKEY_CHANGE:
mcast_groups_event(&dev->port[index], MCAST_PKEY_EVENT);
break;
default:
break;
}
}
static void mcast_add_one(struct ib_device *device)
{
struct mcast_device *dev;
struct mcast_port *port;
int i;
int count = 0;
dev = kmalloc(struct_size(dev, port, device->phys_port_cnt),
GFP_KERNEL);
if (!dev)
return;
dev->start_port = rdma_start_port(device);
dev->end_port = rdma_end_port(device);
for (i = 0; i <= dev->end_port - dev->start_port; i++) {
if (!rdma_cap_ib_mcast(device, dev->start_port + i))
continue;
port = &dev->port[i];
port->dev = dev;
port->port_num = dev->start_port + i;
spin_lock_init(&port->lock);
port->table = RB_ROOT;
init_completion(&port->comp);
atomic_set(&port->refcount, 1);
++count;
}
if (!count) {
kfree(dev);
return;
}
dev->device = device;
ib_set_client_data(device, &mcast_client, dev);
INIT_IB_EVENT_HANDLER(&dev->event_handler, device, mcast_event_handler);
ib_register_event_handler(&dev->event_handler);
}
static void mcast_remove_one(struct ib_device *device, void *client_data)
{
struct mcast_device *dev = client_data;
struct mcast_port *port;
int i;
if (!dev)
return;
ib_unregister_event_handler(&dev->event_handler);
flush_workqueue(mcast_wq);
for (i = 0; i <= dev->end_port - dev->start_port; i++) {
if (rdma_cap_ib_mcast(device, dev->start_port + i)) {
port = &dev->port[i];
deref_port(port);
wait_for_completion(&port->comp);
}
}
kfree(dev);
}
int mcast_init(void)
{
int ret;
mcast_wq = alloc_ordered_workqueue("ib_mcast", WQ_MEM_RECLAIM);
if (!mcast_wq)
return -ENOMEM;
ib_sa_register_client(&sa_client);
ret = ib_register_client(&mcast_client);
if (ret)
goto err;
return 0;
err:
ib_sa_unregister_client(&sa_client);
destroy_workqueue(mcast_wq);
return ret;
}
void mcast_cleanup(void)
{
ib_unregister_client(&mcast_client);
ib_sa_unregister_client(&sa_client);
destroy_workqueue(mcast_wq);
}
+311
View File
@@ -0,0 +1,311 @@
/*
* Copyright (c) 2017 Mellanox Technologies Inc. All rights reserved.
* Copyright (c) 2010 Voltaire Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__
#include <linux/export.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <net/sock.h>
#include <rdma/rdma_netlink.h>
#include <linux/module.h>
#include "core_priv.h"
static DEFINE_MUTEX(rdma_nl_mutex);
static struct sock *nls;
static struct {
const struct rdma_nl_cbs *cb_table;
} rdma_nl_types[RDMA_NL_NUM_CLIENTS];
int rdma_nl_chk_listeners(unsigned int group)
{
return (netlink_has_listeners(nls, group)) ? 0 : -1;
}
EXPORT_SYMBOL(rdma_nl_chk_listeners);
static bool is_nl_msg_valid(unsigned int type, unsigned int op)
{
static const unsigned int max_num_ops[RDMA_NL_NUM_CLIENTS] = {
[RDMA_NL_RDMA_CM] = RDMA_NL_RDMA_CM_NUM_OPS,
[RDMA_NL_IWCM] = RDMA_NL_IWPM_NUM_OPS,
[RDMA_NL_LS] = RDMA_NL_LS_NUM_OPS,
[RDMA_NL_NLDEV] = RDMA_NLDEV_NUM_OPS,
};
/*
* This BUILD_BUG_ON is intended to catch addition of new
* RDMA netlink protocol without updating the array above.
*/
BUILD_BUG_ON(RDMA_NL_NUM_CLIENTS != 6);
if (type >= RDMA_NL_NUM_CLIENTS)
return false;
return (op < max_num_ops[type]) ? true : false;
}
static bool is_nl_valid(unsigned int type, unsigned int op)
{
const struct rdma_nl_cbs *cb_table;
if (!is_nl_msg_valid(type, op))
return false;
if (!rdma_nl_types[type].cb_table) {
mutex_unlock(&rdma_nl_mutex);
request_module("rdma-netlink-subsys-%d", type);
mutex_lock(&rdma_nl_mutex);
}
cb_table = rdma_nl_types[type].cb_table;
if (!cb_table || (!cb_table[op].dump && !cb_table[op].doit))
return false;
return true;
}
void rdma_nl_register(unsigned int index,
const struct rdma_nl_cbs cb_table[])
{
mutex_lock(&rdma_nl_mutex);
if (!is_nl_msg_valid(index, 0)) {
/*
* All clients are not interesting in success/failure of
* this call. They want to see the print to error log and
* continue their initialization. Print warning for them,
* because it is programmer's error to be here.
*/
mutex_unlock(&rdma_nl_mutex);
WARN(true,
"The not-valid %u index was supplied to RDMA netlink\n",
index);
return;
}
if (rdma_nl_types[index].cb_table) {
mutex_unlock(&rdma_nl_mutex);
WARN(true,
"The %u index is already registered in RDMA netlink\n",
index);
return;
}
rdma_nl_types[index].cb_table = cb_table;
mutex_unlock(&rdma_nl_mutex);
}
EXPORT_SYMBOL(rdma_nl_register);
void rdma_nl_unregister(unsigned int index)
{
mutex_lock(&rdma_nl_mutex);
rdma_nl_types[index].cb_table = NULL;
mutex_unlock(&rdma_nl_mutex);
}
EXPORT_SYMBOL(rdma_nl_unregister);
void *ibnl_put_msg(struct sk_buff *skb, struct nlmsghdr **nlh, int seq,
int len, int client, int op, int flags)
{
*nlh = nlmsg_put(skb, 0, seq, RDMA_NL_GET_TYPE(client, op), len, flags);
if (!*nlh)
return NULL;
return nlmsg_data(*nlh);
}
EXPORT_SYMBOL(ibnl_put_msg);
int ibnl_put_attr(struct sk_buff *skb, struct nlmsghdr *nlh,
int len, void *data, int type)
{
if (nla_put(skb, type, len, data)) {
nlmsg_cancel(skb, nlh);
return -EMSGSIZE;
}
return 0;
}
EXPORT_SYMBOL(ibnl_put_attr);
static int rdma_nl_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack)
{
int type = nlh->nlmsg_type;
unsigned int index = RDMA_NL_GET_CLIENT(type);
unsigned int op = RDMA_NL_GET_OP(type);
const struct rdma_nl_cbs *cb_table;
if (!is_nl_valid(index, op))
return -EINVAL;
cb_table = rdma_nl_types[index].cb_table;
if ((cb_table[op].flags & RDMA_NL_ADMIN_PERM) &&
!netlink_capable(skb, CAP_NET_ADMIN))
return -EPERM;
/*
* LS responses overload the 0x100 (NLM_F_ROOT) flag. Don't
* mistakenly call the .dump() function.
*/
if (index == RDMA_NL_LS) {
if (cb_table[op].doit)
return cb_table[op].doit(skb, nlh, extack);
return -EINVAL;
}
/* FIXME: Convert IWCM to properly handle doit callbacks */
if ((nlh->nlmsg_flags & NLM_F_DUMP) || index == RDMA_NL_RDMA_CM ||
index == RDMA_NL_IWCM) {
struct netlink_dump_control c = {
.dump = cb_table[op].dump,
};
if (c.dump)
return netlink_dump_start(nls, skb, nlh, &c);
return -EINVAL;
}
if (cb_table[op].doit)
return cb_table[op].doit(skb, nlh, extack);
return 0;
}
/*
* This function is similar to netlink_rcv_skb with one exception:
* It calls to the callback for the netlink messages without NLM_F_REQUEST
* flag. These messages are intended for RDMA_NL_LS consumer, so it is allowed
* for that consumer only.
*/
static int rdma_nl_rcv_skb(struct sk_buff *skb, int (*cb)(struct sk_buff *,
struct nlmsghdr *,
struct netlink_ext_ack *))
{
struct netlink_ext_ack extack = {};
struct nlmsghdr *nlh;
int err;
while (skb->len >= nlmsg_total_size(0)) {
int msglen;
nlh = nlmsg_hdr(skb);
err = 0;
if (nlh->nlmsg_len < NLMSG_HDRLEN || skb->len < nlh->nlmsg_len)
return 0;
/*
* Generally speaking, the only requests are handled
* by the kernel, but RDMA_NL_LS is different, because it
* runs backward netlink scheme. Kernel initiates messages
* and waits for reply with data to keep pathrecord cache
* in sync.
*/
if (!(nlh->nlmsg_flags & NLM_F_REQUEST) &&
(RDMA_NL_GET_CLIENT(nlh->nlmsg_type) != RDMA_NL_LS))
goto ack;
/* Skip control messages */
if (nlh->nlmsg_type < NLMSG_MIN_TYPE)
goto ack;
err = cb(skb, nlh, &extack);
if (err == -EINTR)
goto skip;
ack:
if (nlh->nlmsg_flags & NLM_F_ACK || err)
netlink_ack(skb, nlh, err, &extack);
skip:
msglen = NLMSG_ALIGN(nlh->nlmsg_len);
if (msglen > skb->len)
msglen = skb->len;
skb_pull(skb, msglen);
}
return 0;
}
static void rdma_nl_rcv(struct sk_buff *skb)
{
mutex_lock(&rdma_nl_mutex);
rdma_nl_rcv_skb(skb, &rdma_nl_rcv_msg);
mutex_unlock(&rdma_nl_mutex);
}
int rdma_nl_unicast(struct sk_buff *skb, u32 pid)
{
int err;
err = netlink_unicast(nls, skb, pid, MSG_DONTWAIT);
return (err < 0) ? err : 0;
}
EXPORT_SYMBOL(rdma_nl_unicast);
int rdma_nl_unicast_wait(struct sk_buff *skb, __u32 pid)
{
int err;
err = netlink_unicast(nls, skb, pid, 0);
return (err < 0) ? err : 0;
}
EXPORT_SYMBOL(rdma_nl_unicast_wait);
int rdma_nl_multicast(struct sk_buff *skb, unsigned int group, gfp_t flags)
{
return nlmsg_multicast(nls, skb, 0, group, flags);
}
EXPORT_SYMBOL(rdma_nl_multicast);
int __init rdma_nl_init(void)
{
struct netlink_kernel_cfg cfg = {
.input = rdma_nl_rcv,
};
nls = netlink_kernel_create(&init_net, NETLINK_RDMA, &cfg);
if (!nls)
return -ENOMEM;
nls->sk_sndtimeo = 10 * HZ;
return 0;
}
void rdma_nl_exit(void)
{
int idx;
for (idx = 0; idx < RDMA_NL_NUM_CLIENTS; idx++)
rdma_nl_unregister(idx);
netlink_kernel_release(nls);
}
MODULE_ALIAS_NET_PF_PROTO(PF_NETLINK, NETLINK_RDMA);
File diff suppressed because it is too large. Load diff
+78
View File
@@ -0,0 +1,78 @@
/*
* Copyright (c) 2014 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
#ifndef __OPA_SMI_H_
#define __OPA_SMI_H_
#include <rdma/ib_smi.h>
#include <rdma/opa_smi.h>
#include "smi.h"
enum smi_action opa_smi_handle_dr_smp_recv(struct opa_smp *smp, bool is_switch,
int port_num, int phys_port_cnt);
int opa_smi_get_fwd_port(struct opa_smp *smp);
extern enum smi_forward_action opa_smi_check_forward_dr_smp(struct opa_smp *smp);
extern enum smi_action opa_smi_handle_dr_smp_send(struct opa_smp *smp,
bool is_switch, int port_num);
/*
* Return IB_SMI_HANDLE if the SMP should be handled by the local SMA/SM
* via process_mad
*/
static inline enum smi_action opa_smi_check_local_smp(struct opa_smp *smp,
struct ib_device *device)
{
/* C14-9:3 -- We're at the end of the DR segment of path */
/* C14-9:4 -- Hop Pointer = Hop Count + 1 -> give to SMA/SM */
return (device->process_mad &&
!opa_get_smp_direction(smp) &&
(smp->hop_ptr == smp->hop_cnt + 1)) ?
IB_SMI_HANDLE : IB_SMI_DISCARD;
}
/*
* Return IB_SMI_HANDLE if the SMP should be handled by the local SMA/SM
* via process_mad
*/
static inline enum smi_action opa_smi_check_local_returning_smp(struct opa_smp *smp,
struct ib_device *device)
{
/* C14-13:3 -- We're at the end of the DR segment of path */
/* C14-13:4 -- Hop Pointer == 0 -> give to SM */
return (device->process_mad &&
opa_get_smp_direction(smp) &&
!smp->hop_ptr) ? IB_SMI_HANDLE : IB_SMI_DISCARD;
}
#endif /* __OPA_SMI_H_ */
+201
View File
@@ -0,0 +1,201 @@
/*
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/export.h>
#include <linux/string.h>
#include <rdma/ib_pack.h>
static u64 value_read(int offset, int size, void *structure)
{
switch (size) {
case 1: return *(u8 *) (structure + offset);
case 2: return be16_to_cpup((__be16 *) (structure + offset));
case 4: return be32_to_cpup((__be32 *) (structure + offset));
case 8: return be64_to_cpup((__be64 *) (structure + offset));
default:
pr_warn("Field size %d bits not handled\n", size * 8);
return 0;
}
}
/**
* ib_pack - Pack a structure into a buffer
* @desc:Array of structure field descriptions
* @desc_len:Number of entries in @desc
* @structure:Structure to pack from
* @buf:Buffer to pack into
*
* ib_pack() packs a list of structure fields into a buffer,
* controlled by the array of fields in @desc.
*/
void ib_pack(const struct ib_field *desc,
int desc_len,
void *structure,
void *buf)
{
int i;
for (i = 0; i < desc_len; ++i) {
if (desc[i].size_bits <= 32) {
int shift;
u32 val;
__be32 mask;
__be32 *addr;
shift = 32 - desc[i].offset_bits - desc[i].size_bits;
if (desc[i].struct_size_bytes)
val = value_read(desc[i].struct_offset_bytes,
desc[i].struct_size_bytes,
structure) << shift;
else
val = 0;
mask = cpu_to_be32(((1ull << desc[i].size_bits) - 1) << shift);
addr = (__be32 *) buf + desc[i].offset_words;
*addr = (*addr & ~mask) | (cpu_to_be32(val) & mask);
} else if (desc[i].size_bits <= 64) {
int shift;
u64 val;
__be64 mask;
__be64 *addr;
shift = 64 - desc[i].offset_bits - desc[i].size_bits;
if (desc[i].struct_size_bytes)
val = value_read(desc[i].struct_offset_bytes,
desc[i].struct_size_bytes,
structure) << shift;
else
val = 0;
mask = cpu_to_be64((~0ull >> (64 - desc[i].size_bits)) << shift);
addr = (__be64 *) ((__be32 *) buf + desc[i].offset_words);
*addr = (*addr & ~mask) | (cpu_to_be64(val) & mask);
} else {
if (desc[i].offset_bits % 8 ||
desc[i].size_bits % 8) {
pr_warn("Structure field %s of size %d bits is not byte-aligned\n",
desc[i].field_name, desc[i].size_bits);
}
if (desc[i].struct_size_bytes)
memcpy(buf + desc[i].offset_words * 4 +
desc[i].offset_bits / 8,
structure + desc[i].struct_offset_bytes,
desc[i].size_bits / 8);
else
memset(buf + desc[i].offset_words * 4 +
desc[i].offset_bits / 8,
0,
desc[i].size_bits / 8);
}
}
}
EXPORT_SYMBOL(ib_pack);
static void value_write(int offset, int size, u64 val, void *structure)
{
switch (size * 8) {
case 8: *( u8 *) (structure + offset) = val; break;
case 16: *(__be16 *) (structure + offset) = cpu_to_be16(val); break;
case 32: *(__be32 *) (structure + offset) = cpu_to_be32(val); break;
case 64: *(__be64 *) (structure + offset) = cpu_to_be64(val); break;
default:
pr_warn("Field size %d bits not handled\n", size * 8);
}
}
/**
* ib_unpack - Unpack a buffer into a structure
* @desc:Array of structure field descriptions
* @desc_len:Number of entries in @desc
* @buf:Buffer to unpack from
* @structure:Structure to unpack into
*
* ib_pack() unpacks a list of structure fields from a buffer,
* controlled by the array of fields in @desc.
*/
void ib_unpack(const struct ib_field *desc,
int desc_len,
void *buf,
void *structure)
{
int i;
for (i = 0; i < desc_len; ++i) {
if (!desc[i].struct_size_bytes)
continue;
if (desc[i].size_bits <= 32) {
int shift;
u32 val;
u32 mask;
__be32 *addr;
shift = 32 - desc[i].offset_bits - desc[i].size_bits;
mask = ((1ull << desc[i].size_bits) - 1) << shift;
addr = (__be32 *) buf + desc[i].offset_words;
val = (be32_to_cpup(addr) & mask) >> shift;
value_write(desc[i].struct_offset_bytes,
desc[i].struct_size_bytes,
val,
structure);
} else if (desc[i].size_bits <= 64) {
int shift;
u64 val;
u64 mask;
__be64 *addr;
shift = 64 - desc[i].offset_bits - desc[i].size_bits;
mask = (~0ull >> (64 - desc[i].size_bits)) << shift;
addr = (__be64 *) buf + desc[i].offset_words;
val = (be64_to_cpup(addr) & mask) >> shift;
value_write(desc[i].struct_offset_bytes,
desc[i].struct_size_bytes,
val,
structure);
} else {
if (desc[i].offset_bits % 8 ||
desc[i].size_bits % 8) {
pr_warn("Structure field %s of size %d bits is not byte-aligned\n",
desc[i].field_name, desc[i].size_bits);
}
memcpy(structure + desc[i].struct_offset_bytes,
buf + desc[i].offset_words * 4 +
desc[i].offset_bits / 8,
desc[i].size_bits / 8);
}
}
}
EXPORT_SYMBOL(ib_unpack);
File diff suppressed because it is too large. Load diff
+164
View File
@@ -0,0 +1,164 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
* Copyright (c) 2005-2017 Mellanox Technologies. All rights reserved.
* Copyright (c) 2005 Voltaire, Inc. All rights reserved.
* Copyright (c) 2005 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef RDMA_CORE_H
#define RDMA_CORE_H
#include <linux/idr.h>
#include <rdma/uverbs_types.h>
#include <rdma/uverbs_ioctl.h>
#include <rdma/ib_verbs.h>
#include <linux/mutex.h>
struct ib_uverbs_device;
void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason);
int uobj_destroy(struct ib_uobject *uobj);
/*
* uverbs_uobject_get is called in order to increase the reference count on
* an uobject. This is useful when a handler wants to keep the uobject's memory
* alive, regardless if this uobject is still alive in the context's objects
* repository. Objects are put via uverbs_uobject_put.
*/
void uverbs_uobject_get(struct ib_uobject *uobject);
/*
* In order to indicate we no longer needs this uobject, uverbs_uobject_put
* is called. When the reference count is decreased, the uobject is freed.
* For example, this is used when attaching a completion channel to a CQ.
*/
void uverbs_uobject_put(struct ib_uobject *uobject);
/* Indicate this fd is no longer used by this consumer, but its memory isn't
* necessarily released yet. When the last reference is put, we release the
* memory. After this call is executed, calling uverbs_uobject_get isn't
* allowed.
* This must be called from the release file_operations of the file!
*/
void uverbs_close_fd(struct file *f);
/*
* Get an ib_uobject that corresponds to the given id from ufile, assuming
* the object is from the given type. Lock it to the required access when
* applicable.
* This function could create (access == NEW), destroy (access == DESTROY)
* or unlock (access == READ || access == WRITE) objects if required.
* The action will be finalized only when uverbs_finalize_object or
* uverbs_finalize_objects are called.
*/
struct ib_uobject *
uverbs_get_uobject_from_file(u16 object_id,
struct ib_uverbs_file *ufile,
enum uverbs_obj_access access, s64 id);
/*
* Note that certain finalize stages could return a status:
* (a) alloc_commit could return a failure if the object is committed at the
* same time when the context is destroyed.
* (b) remove_commit could fail if the object wasn't destroyed successfully.
* Since multiple objects could be finalized in one transaction, it is very NOT
* recommended to have several finalize actions which have side effects.
* For example, it's NOT recommended to have a certain action which has both
* a commit action and a destroy action or two destroy objects in the same
* action. The rule of thumb is to have one destroy or commit action with
* multiple lookups.
* The first non zero return value of finalize_object is returned from this
* function. For example, this could happen when we couldn't destroy an
* object.
*/
int uverbs_finalize_object(struct ib_uobject *uobj,
enum uverbs_obj_access access,
bool commit);
void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile);
void release_ufile_idr_uobject(struct ib_uverbs_file *ufile);
/*
* This is the runtime description of the uverbs API, used by the syscall
* machinery to validate and dispatch calls.
*/
/*
* Depending on ID the slot pointer in the radix tree points at one of these
* structs.
*/
struct uverbs_api_object {
const struct uverbs_obj_type *type_attrs;
const struct uverbs_obj_type_class *type_class;
};
struct uverbs_api_ioctl_method {
int (__rcu *handler)(struct ib_uverbs_file *ufile,
struct uverbs_attr_bundle *ctx);
DECLARE_BITMAP(attr_mandatory, UVERBS_API_ATTR_BKEY_LEN);
u16 bundle_size;
u8 use_stack:1;
u8 driver_method:1;
u8 key_bitmap_len;
u8 destroy_bkey;
};
struct uverbs_api_attr {
struct uverbs_attr_spec spec;
};
struct uverbs_api_object;
struct uverbs_api {
/* radix tree contains struct uverbs_api_* pointers */
struct radix_tree_root radix;
enum rdma_driver_id driver_id;
};
static inline const struct uverbs_api_object *
uapi_get_object(struct uverbs_api *uapi, u16 object_id)
{
return radix_tree_lookup(&uapi->radix, uapi_key_obj(object_id));
}
char *uapi_key_format(char *S, unsigned int key);
struct uverbs_api *uverbs_alloc_api(
const struct uverbs_object_tree_def *const *driver_specs,
enum rdma_driver_id driver_id);
void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev);
void uverbs_disassociate_api(struct uverbs_api *uapi);
void uverbs_destroy_api(struct uverbs_api *uapi);
void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
unsigned int num_attrs);
#endif /* RDMA_CORE_H */
+233
View File
@@ -0,0 +1,233 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved.
*/
#include <rdma/rdma_cm.h>
#include <rdma/ib_verbs.h>
#include <rdma/restrack.h>
#include <linux/mutex.h>
#include <linux/sched/task.h>
#include <linux/pid_namespace.h>
#include "cma_priv.h"
static int fill_res_noop(struct sk_buff *msg,
struct rdma_restrack_entry *entry)
{
return 0;
}
void rdma_restrack_init(struct rdma_restrack_root *res)
{
init_rwsem(&res->rwsem);
res->fill_res_entry = fill_res_noop;
}
static const char *type2str(enum rdma_restrack_type type)
{
static const char * const names[RDMA_RESTRACK_MAX] = {
[RDMA_RESTRACK_PD] = "PD",
[RDMA_RESTRACK_CQ] = "CQ",
[RDMA_RESTRACK_QP] = "QP",
[RDMA_RESTRACK_CM_ID] = "CM_ID",
[RDMA_RESTRACK_MR] = "MR",
};
return names[type];
};
void rdma_restrack_clean(struct rdma_restrack_root *res)
{
struct rdma_restrack_entry *e;
char buf[TASK_COMM_LEN];
struct ib_device *dev;
const char *owner;
int bkt;
if (hash_empty(res->hash))
return;
dev = container_of(res, struct ib_device, res);
pr_err("restrack: %s", CUT_HERE);
pr_err("restrack: BUG: RESTRACK detected leak of resources on %s\n",
dev->name);
hash_for_each(res->hash, bkt, e, node) {
if (rdma_is_kernel_res(e)) {
owner = e->kern_name;
} else {
/*
* There is no need to call get_task_struct here,
* because we can be here only if there are more
* get_task_struct() call than put_task_struct().
*/
get_task_comm(buf, e->task);
owner = buf;
}
pr_err("restrack: %s %s object allocated by %s is not freed\n",
rdma_is_kernel_res(e) ? "Kernel" : "User",
type2str(e->type), owner);
}
pr_err("restrack: %s", CUT_HERE);
}
int rdma_restrack_count(struct rdma_restrack_root *res,
enum rdma_restrack_type type,
struct pid_namespace *ns)
{
struct rdma_restrack_entry *e;
u32 cnt = 0;
down_read(&res->rwsem);
hash_for_each_possible(res->hash, e, node, type) {
if (ns == &init_pid_ns ||
(!rdma_is_kernel_res(e) &&
ns == task_active_pid_ns(e->task)))
cnt++;
}
up_read(&res->rwsem);
return cnt;
}
EXPORT_SYMBOL(rdma_restrack_count);
static void set_kern_name(struct rdma_restrack_entry *res)
{
struct ib_pd *pd;
switch (res->type) {
case RDMA_RESTRACK_QP:
pd = container_of(res, struct ib_qp, res)->pd;
if (!pd) {
WARN_ONCE(true, "XRC QPs are not supported\n");
/* Survive, despite the programmer's error */
res->kern_name = " ";
}
break;
case RDMA_RESTRACK_MR:
pd = container_of(res, struct ib_mr, res)->pd;
break;
default:
/* Other types set kern_name directly */
pd = NULL;
break;
}
if (pd)
res->kern_name = pd->res.kern_name;
}
static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
{
switch (res->type) {
case RDMA_RESTRACK_PD:
return container_of(res, struct ib_pd, res)->device;
case RDMA_RESTRACK_CQ:
return container_of(res, struct ib_cq, res)->device;
case RDMA_RESTRACK_QP:
return container_of(res, struct ib_qp, res)->device;
case RDMA_RESTRACK_CM_ID:
return container_of(res, struct rdma_id_private,
res)->id.device;
case RDMA_RESTRACK_MR:
return container_of(res, struct ib_mr, res)->device;
default:
WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type);
return NULL;
}
}
static bool res_is_user(struct rdma_restrack_entry *res)
{
switch (res->type) {
case RDMA_RESTRACK_PD:
return container_of(res, struct ib_pd, res)->uobject;
case RDMA_RESTRACK_CQ:
return container_of(res, struct ib_cq, res)->uobject;
case RDMA_RESTRACK_QP:
return container_of(res, struct ib_qp, res)->uobject;
case RDMA_RESTRACK_CM_ID:
return !res->kern_name;
case RDMA_RESTRACK_MR:
return container_of(res, struct ib_mr, res)->pd->uobject;
default:
WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type);
return false;
}
}
void rdma_restrack_add(struct rdma_restrack_entry *res)
{
struct ib_device *dev = res_to_dev(res);
if (!dev)
return;
if (res->type != RDMA_RESTRACK_CM_ID || !res_is_user(res))
res->task = NULL;
if (res_is_user(res)) {
if (!res->task)
rdma_restrack_set_task(res, current);
res->kern_name = NULL;
} else {
set_kern_name(res);
}
kref_init(&res->kref);
init_completion(&res->comp);
res->valid = true;
down_write(&dev->res.rwsem);
hash_add(dev->res.hash, &res->node, res->type);
up_write(&dev->res.rwsem);
}
EXPORT_SYMBOL(rdma_restrack_add);
int __must_check rdma_restrack_get(struct rdma_restrack_entry *res)
{
return kref_get_unless_zero(&res->kref);
}
EXPORT_SYMBOL(rdma_restrack_get);
static void restrack_release(struct kref *kref)
{
struct rdma_restrack_entry *res;
res = container_of(kref, struct rdma_restrack_entry, kref);
complete(&res->comp);
}
int rdma_restrack_put(struct rdma_restrack_entry *res)
{
return kref_put(&res->kref, restrack_release);
}
EXPORT_SYMBOL(rdma_restrack_put);
void rdma_restrack_del(struct rdma_restrack_entry *res)
{
struct ib_device *dev;
if (!res->valid)
goto out;
dev = res_to_dev(res);
if (!dev)
return;
rdma_restrack_put(res);
wait_for_completion(&res->comp);
down_write(&dev->res.rwsem);
hash_del(&res->node);
res->valid = false;
up_write(&dev->res.rwsem);
out:
if (res->task) {
put_task_struct(res->task);
res->task = NULL;
}
}
EXPORT_SYMBOL(rdma_restrack_del);
+924
View File
@@ -0,0 +1,924 @@
/*
* Copyright (c) 2015, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "core_priv.h"
#include <linux/in.h>
#include <linux/in6.h>
/* For in6_dev_get/in6_dev_put */
#include <net/addrconf.h>
#include <net/bonding.h>
#include <rdma/ib_cache.h>
#include <rdma/ib_addr.h>
static struct workqueue_struct *gid_cache_wq;
enum gid_op_type {
GID_DEL = 0,
GID_ADD
};
struct update_gid_event_work {
struct work_struct work;
union ib_gid gid;
struct ib_gid_attr gid_attr;
enum gid_op_type gid_op;
};
#define ROCE_NETDEV_CALLBACK_SZ 3
struct netdev_event_work_cmd {
roce_netdev_callback cb;
roce_netdev_filter filter;
struct net_device *ndev;
struct net_device *filter_ndev;
};
struct netdev_event_work {
struct work_struct work;
struct netdev_event_work_cmd cmds[ROCE_NETDEV_CALLBACK_SZ];
};
static const struct {
bool (*is_supported)(const struct ib_device *device, u8 port_num);
enum ib_gid_type gid_type;
} PORT_CAP_TO_GID_TYPE[] = {
{rdma_protocol_roce_eth_encap, IB_GID_TYPE_ROCE},
{rdma_protocol_roce_udp_encap, IB_GID_TYPE_ROCE_UDP_ENCAP},
};
#define CAP_TO_GID_TABLE_SIZE ARRAY_SIZE(PORT_CAP_TO_GID_TYPE)
unsigned long roce_gid_type_mask_support(struct ib_device *ib_dev, u8 port)
{
int i;
unsigned int ret_flags = 0;
if (!rdma_protocol_roce(ib_dev, port))
return 1UL << IB_GID_TYPE_IB;
for (i = 0; i < CAP_TO_GID_TABLE_SIZE; i++)
if (PORT_CAP_TO_GID_TYPE[i].is_supported(ib_dev, port))
ret_flags |= 1UL << PORT_CAP_TO_GID_TYPE[i].gid_type;
return ret_flags;
}
EXPORT_SYMBOL(roce_gid_type_mask_support);
static void update_gid(enum gid_op_type gid_op, struct ib_device *ib_dev,
u8 port, union ib_gid *gid,
struct ib_gid_attr *gid_attr)
{
int i;
unsigned long gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
for (i = 0; i < IB_GID_TYPE_SIZE; i++) {
if ((1UL << i) & gid_type_mask) {
gid_attr->gid_type = i;
switch (gid_op) {
case GID_ADD:
ib_cache_gid_add(ib_dev, port,
gid, gid_attr);
break;
case GID_DEL:
ib_cache_gid_del(ib_dev, port,
gid, gid_attr);
break;
}
}
}
}
enum bonding_slave_state {
BONDING_SLAVE_STATE_ACTIVE = 1UL << 0,
BONDING_SLAVE_STATE_INACTIVE = 1UL << 1,
/* No primary slave or the device isn't a slave in bonding */
BONDING_SLAVE_STATE_NA = 1UL << 2,
};
static enum bonding_slave_state is_eth_active_slave_of_bonding_rcu(struct net_device *dev,
struct net_device *upper)
{
if (upper && netif_is_bond_master(upper)) {
struct net_device *pdev =
bond_option_active_slave_get_rcu(netdev_priv(upper));
if (pdev)
return dev == pdev ? BONDING_SLAVE_STATE_ACTIVE :
BONDING_SLAVE_STATE_INACTIVE;
}
return BONDING_SLAVE_STATE_NA;
}
#define REQUIRED_BOND_STATES (BONDING_SLAVE_STATE_ACTIVE | \
BONDING_SLAVE_STATE_NA)
static bool
is_eth_port_of_netdev_filter(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
struct net_device *real_dev;
bool res;
if (!rdma_ndev)
return false;
rcu_read_lock();
real_dev = rdma_vlan_dev_real_dev(cookie);
if (!real_dev)
real_dev = cookie;
res = ((rdma_is_upper_dev_rcu(rdma_ndev, cookie) &&
(is_eth_active_slave_of_bonding_rcu(rdma_ndev, real_dev) &
REQUIRED_BOND_STATES)) ||
real_dev == rdma_ndev);
rcu_read_unlock();
return res;
}
static bool
is_eth_port_inactive_slave_filter(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
struct net_device *master_dev;
bool res;
if (!rdma_ndev)
return false;
rcu_read_lock();
master_dev = netdev_master_upper_dev_get_rcu(rdma_ndev);
res = is_eth_active_slave_of_bonding_rcu(rdma_ndev, master_dev) ==
BONDING_SLAVE_STATE_INACTIVE;
rcu_read_unlock();
return res;
}
/** is_ndev_for_default_gid_filter - Check if a given netdevice
* can be considered for default GIDs or not.
* @ib_dev: IB device to check
* @port: Port to consider for adding default GID
* @rdma_ndev: rdma netdevice pointer
* @cookie_ndev: Netdevice to consider to form a default GID
*
* is_ndev_for_default_gid_filter() returns true if a given netdevice can be
* considered for deriving default RoCE GID, returns false otherwise.
*/
static bool
is_ndev_for_default_gid_filter(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
struct net_device *cookie_ndev = cookie;
bool res;
if (!rdma_ndev)
return false;
rcu_read_lock();
/*
* When rdma netdevice is used in bonding, bonding master netdevice
* should be considered for default GIDs. Therefore, ignore slave rdma
* netdevices when bonding is considered.
* Additionally when event(cookie) netdevice is bond master device,
* make sure that it the upper netdevice of rdma netdevice.
*/
res = ((cookie_ndev == rdma_ndev && !netif_is_bond_slave(rdma_ndev)) ||
(netif_is_bond_master(cookie_ndev) &&
rdma_is_upper_dev_rcu(rdma_ndev, cookie_ndev)));
rcu_read_unlock();
return res;
}
static bool pass_all_filter(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
return true;
}
static bool upper_device_filter(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
bool res;
if (!rdma_ndev)
return false;
if (rdma_ndev == cookie)
return true;
rcu_read_lock();
res = rdma_is_upper_dev_rcu(rdma_ndev, cookie);
rcu_read_unlock();
return res;
}
/**
* is_upper_ndev_bond_master_filter - Check if a given netdevice
* is bond master device of netdevice of the the RDMA device of port.
* @ib_dev: IB device to check
* @port: Port to consider for adding default GID
* @rdma_ndev: Pointer to rdma netdevice
* @cookie: Netdevice to consider to form a default GID
*
* is_upper_ndev_bond_master_filter() returns true if a cookie_netdev
* is bond master device and rdma_ndev is its lower netdevice. It might
* not have been established as slave device yet.
*/
static bool
is_upper_ndev_bond_master_filter(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev,
void *cookie)
{
struct net_device *cookie_ndev = cookie;
bool match = false;
if (!rdma_ndev)
return false;
rcu_read_lock();
if (netif_is_bond_master(cookie_ndev) &&
rdma_is_upper_dev_rcu(rdma_ndev, cookie_ndev))
match = true;
rcu_read_unlock();
return match;
}
static void update_gid_ip(enum gid_op_type gid_op,
struct ib_device *ib_dev,
u8 port, struct net_device *ndev,
struct sockaddr *addr)
{
union ib_gid gid;
struct ib_gid_attr gid_attr;
rdma_ip2gid(addr, &gid);
memset(&gid_attr, 0, sizeof(gid_attr));
gid_attr.ndev = ndev;
update_gid(gid_op, ib_dev, port, &gid, &gid_attr);
}
static void bond_delete_netdev_default_gids(struct ib_device *ib_dev,
u8 port,
struct net_device *rdma_ndev,
struct net_device *event_ndev)
{
struct net_device *real_dev = rdma_vlan_dev_real_dev(event_ndev);
unsigned long gid_type_mask;
if (!rdma_ndev)
return;
if (!real_dev)
real_dev = event_ndev;
rcu_read_lock();
if (((rdma_ndev != event_ndev &&
!rdma_is_upper_dev_rcu(rdma_ndev, event_ndev)) ||
is_eth_active_slave_of_bonding_rcu(rdma_ndev, real_dev)
==
BONDING_SLAVE_STATE_INACTIVE)) {
rcu_read_unlock();
return;
}
rcu_read_unlock();
gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
ib_cache_gid_set_default_gid(ib_dev, port, rdma_ndev,
gid_type_mask,
IB_CACHE_GID_DEFAULT_MODE_DELETE);
}
static void enum_netdev_ipv4_ips(struct ib_device *ib_dev,
u8 port, struct net_device *ndev)
{
struct in_device *in_dev;
struct sin_list {
struct list_head list;
struct sockaddr_in ip;
};
struct sin_list *sin_iter;
struct sin_list *sin_temp;
LIST_HEAD(sin_list);
if (ndev->reg_state >= NETREG_UNREGISTERING)
return;
rcu_read_lock();
in_dev = __in_dev_get_rcu(ndev);
if (!in_dev) {
rcu_read_unlock();
return;
}
for_ifa(in_dev) {
struct sin_list *entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
if (!entry)
continue;
entry->ip.sin_family = AF_INET;
entry->ip.sin_addr.s_addr = ifa->ifa_address;
list_add_tail(&entry->list, &sin_list);
}
endfor_ifa(in_dev);
rcu_read_unlock();
list_for_each_entry_safe(sin_iter, sin_temp, &sin_list, list) {
update_gid_ip(GID_ADD, ib_dev, port, ndev,
(struct sockaddr *)&sin_iter->ip);
list_del(&sin_iter->list);
kfree(sin_iter);
}
}
static void enum_netdev_ipv6_ips(struct ib_device *ib_dev,
u8 port, struct net_device *ndev)
{
struct inet6_ifaddr *ifp;
struct inet6_dev *in6_dev;
struct sin6_list {
struct list_head list;
struct sockaddr_in6 sin6;
};
struct sin6_list *sin6_iter;
struct sin6_list *sin6_temp;
struct ib_gid_attr gid_attr = {.ndev = ndev};
LIST_HEAD(sin6_list);
if (ndev->reg_state >= NETREG_UNREGISTERING)
return;
in6_dev = in6_dev_get(ndev);
if (!in6_dev)
return;
read_lock_bh(&in6_dev->lock);
list_for_each_entry(ifp, &in6_dev->addr_list, if_list) {
struct sin6_list *entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
if (!entry)
continue;
entry->sin6.sin6_family = AF_INET6;
entry->sin6.sin6_addr = ifp->addr;
list_add_tail(&entry->list, &sin6_list);
}
read_unlock_bh(&in6_dev->lock);
in6_dev_put(in6_dev);
list_for_each_entry_safe(sin6_iter, sin6_temp, &sin6_list, list) {
union ib_gid gid;
rdma_ip2gid((struct sockaddr *)&sin6_iter->sin6, &gid);
update_gid(GID_ADD, ib_dev, port, &gid, &gid_attr);
list_del(&sin6_iter->list);
kfree(sin6_iter);
}
}
static void _add_netdev_ips(struct ib_device *ib_dev, u8 port,
struct net_device *ndev)
{
enum_netdev_ipv4_ips(ib_dev, port, ndev);
if (IS_ENABLED(CONFIG_IPV6))
enum_netdev_ipv6_ips(ib_dev, port, ndev);
}
static void add_netdev_ips(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
_add_netdev_ips(ib_dev, port, cookie);
}
static void del_netdev_ips(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
ib_cache_gid_del_all_netdev_gids(ib_dev, port, cookie);
}
/**
* del_default_gids - Delete default GIDs of the event/cookie netdevice
* @ib_dev: RDMA device pointer
* @port: Port of the RDMA device whose GID table to consider
* @rdma_ndev: Unused rdma netdevice
* @cookie: Pointer to event netdevice
*
* del_default_gids() deletes the default GIDs of the event/cookie netdevice.
*/
static void del_default_gids(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
struct net_device *cookie_ndev = cookie;
unsigned long gid_type_mask;
gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
ib_cache_gid_set_default_gid(ib_dev, port, cookie_ndev, gid_type_mask,
IB_CACHE_GID_DEFAULT_MODE_DELETE);
}
static void add_default_gids(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
struct net_device *event_ndev = cookie;
unsigned long gid_type_mask;
gid_type_mask = roce_gid_type_mask_support(ib_dev, port);
ib_cache_gid_set_default_gid(ib_dev, port, event_ndev, gid_type_mask,
IB_CACHE_GID_DEFAULT_MODE_SET);
}
static void enum_all_gids_of_dev_cb(struct ib_device *ib_dev,
u8 port,
struct net_device *rdma_ndev,
void *cookie)
{
struct net *net;
struct net_device *ndev;
/* Lock the rtnl to make sure the netdevs does not move under
* our feet
*/
rtnl_lock();
down_read(&net_rwsem);
for_each_net(net)
for_each_netdev(net, ndev) {
/*
* Filter and add default GIDs of the primary netdevice
* when not in bonding mode, or add default GIDs
* of bond master device, when in bonding mode.
*/
if (is_ndev_for_default_gid_filter(ib_dev, port,
rdma_ndev, ndev))
add_default_gids(ib_dev, port, rdma_ndev, ndev);
if (is_eth_port_of_netdev_filter(ib_dev, port,
rdma_ndev, ndev))
_add_netdev_ips(ib_dev, port, ndev);
}
up_read(&net_rwsem);
rtnl_unlock();
}
/**
* rdma_roce_rescan_device - Rescan all of the network devices in the system
* and add their gids, as needed, to the relevant RoCE devices.
*
* @device: the rdma device
*/
void rdma_roce_rescan_device(struct ib_device *ib_dev)
{
ib_enum_roce_netdev(ib_dev, pass_all_filter, NULL,
enum_all_gids_of_dev_cb, NULL);
}
EXPORT_SYMBOL(rdma_roce_rescan_device);
static void callback_for_addr_gid_device_scan(struct ib_device *device,
u8 port,
struct net_device *rdma_ndev,
void *cookie)
{
struct update_gid_event_work *parsed = cookie;
return update_gid(parsed->gid_op, device,
port, &parsed->gid,
&parsed->gid_attr);
}
struct upper_list {
struct list_head list;
struct net_device *upper;
};
static int netdev_upper_walk(struct net_device *upper, void *data)
{
struct upper_list *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
struct list_head *upper_list = data;
if (!entry)
return 0;
list_add_tail(&entry->list, upper_list);
dev_hold(upper);
entry->upper = upper;
return 0;
}
static void handle_netdev_upper(struct ib_device *ib_dev, u8 port,
void *cookie,
void (*handle_netdev)(struct ib_device *ib_dev,
u8 port,
struct net_device *ndev))
{
struct net_device *ndev = cookie;
struct upper_list *upper_iter;
struct upper_list *upper_temp;
LIST_HEAD(upper_list);
rcu_read_lock();
netdev_walk_all_upper_dev_rcu(ndev, netdev_upper_walk, &upper_list);
rcu_read_unlock();
handle_netdev(ib_dev, port, ndev);
list_for_each_entry_safe(upper_iter, upper_temp, &upper_list,
list) {
handle_netdev(ib_dev, port, upper_iter->upper);
dev_put(upper_iter->upper);
list_del(&upper_iter->list);
kfree(upper_iter);
}
}
static void _roce_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,
struct net_device *event_ndev)
{
ib_cache_gid_del_all_netdev_gids(ib_dev, port, event_ndev);
}
static void del_netdev_upper_ips(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
handle_netdev_upper(ib_dev, port, cookie, _roce_del_all_netdev_gids);
}
static void add_netdev_upper_ips(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev, void *cookie)
{
handle_netdev_upper(ib_dev, port, cookie, _add_netdev_ips);
}
static void del_netdev_default_ips_join(struct ib_device *ib_dev, u8 port,
struct net_device *rdma_ndev,
void *cookie)
{
struct net_device *master_ndev;
rcu_read_lock();
master_ndev = netdev_master_upper_dev_get_rcu(rdma_ndev);
if (master_ndev)
dev_hold(master_ndev);
rcu_read_unlock();
if (master_ndev) {
bond_delete_netdev_default_gids(ib_dev, port, rdma_ndev,
master_ndev);
dev_put(master_ndev);
}
}
/* The following functions operate on all IB devices. netdevice_event and
* addr_event execute ib_enum_all_roce_netdevs through a work.
* ib_enum_all_roce_netdevs iterates through all IB devices.
*/
static void netdevice_event_work_handler(struct work_struct *_work)
{
struct netdev_event_work *work =
container_of(_work, struct netdev_event_work, work);
unsigned int i;
for (i = 0; i < ARRAY_SIZE(work->cmds) && work->cmds[i].cb; i++) {
ib_enum_all_roce_netdevs(work->cmds[i].filter,
work->cmds[i].filter_ndev,
work->cmds[i].cb,
work->cmds[i].ndev);
dev_put(work->cmds[i].ndev);
dev_put(work->cmds[i].filter_ndev);
}
kfree(work);
}
static int netdevice_queue_work(struct netdev_event_work_cmd *cmds,
struct net_device *ndev)
{
unsigned int i;
struct netdev_event_work *ndev_work =
kmalloc(sizeof(*ndev_work), GFP_KERNEL);
if (!ndev_work)
return NOTIFY_DONE;
memcpy(ndev_work->cmds, cmds, sizeof(ndev_work->cmds));
for (i = 0; i < ARRAY_SIZE(ndev_work->cmds) && ndev_work->cmds[i].cb; i++) {
if (!ndev_work->cmds[i].ndev)
ndev_work->cmds[i].ndev = ndev;
if (!ndev_work->cmds[i].filter_ndev)
ndev_work->cmds[i].filter_ndev = ndev;
dev_hold(ndev_work->cmds[i].ndev);
dev_hold(ndev_work->cmds[i].filter_ndev);
}
INIT_WORK(&ndev_work->work, netdevice_event_work_handler);
queue_work(gid_cache_wq, &ndev_work->work);
return NOTIFY_DONE;
}
static const struct netdev_event_work_cmd add_cmd = {
.cb = add_netdev_ips,
.filter = is_eth_port_of_netdev_filter
};
static const struct netdev_event_work_cmd add_cmd_upper_ips = {
.cb = add_netdev_upper_ips,
.filter = is_eth_port_of_netdev_filter
};
static void
ndev_event_unlink(struct netdev_notifier_changeupper_info *changeupper_info,
struct netdev_event_work_cmd *cmds)
{
static const struct netdev_event_work_cmd
upper_ips_del_cmd = {
.cb = del_netdev_upper_ips,
.filter = upper_device_filter
};
cmds[0] = upper_ips_del_cmd;
cmds[0].ndev = changeupper_info->upper_dev;
cmds[1] = add_cmd;
}
static const struct netdev_event_work_cmd bonding_default_add_cmd = {
.cb = add_default_gids,
.filter = is_upper_ndev_bond_master_filter
};
static void
ndev_event_link(struct net_device *event_ndev,
struct netdev_notifier_changeupper_info *changeupper_info,
struct netdev_event_work_cmd *cmds)
{
static const struct netdev_event_work_cmd
bonding_default_del_cmd = {
.cb = del_default_gids,
.filter = is_upper_ndev_bond_master_filter
};
/*
* When a lower netdev is linked to its upper bonding
* netdev, delete lower slave netdev's default GIDs.
*/
cmds[0] = bonding_default_del_cmd;
cmds[0].ndev = event_ndev;
cmds[0].filter_ndev = changeupper_info->upper_dev;
/* Now add bonding upper device default GIDs */
cmds[1] = bonding_default_add_cmd;
cmds[1].ndev = changeupper_info->upper_dev;
cmds[1].filter_ndev = changeupper_info->upper_dev;
/* Now add bonding upper device IP based GIDs */
cmds[2] = add_cmd_upper_ips;
cmds[2].ndev = changeupper_info->upper_dev;
cmds[2].filter_ndev = changeupper_info->upper_dev;
}
static void netdevice_event_changeupper(struct net_device *event_ndev,
struct netdev_notifier_changeupper_info *changeupper_info,
struct netdev_event_work_cmd *cmds)
{
if (changeupper_info->linking)
ndev_event_link(event_ndev, changeupper_info, cmds);
else
ndev_event_unlink(changeupper_info, cmds);
}
static const struct netdev_event_work_cmd add_default_gid_cmd = {
.cb = add_default_gids,
.filter = is_ndev_for_default_gid_filter,
};
static int netdevice_event(struct notifier_block *this, unsigned long event,
void *ptr)
{
static const struct netdev_event_work_cmd del_cmd = {
.cb = del_netdev_ips, .filter = pass_all_filter};
static const struct netdev_event_work_cmd
bonding_default_del_cmd_join = {
.cb = del_netdev_default_ips_join,
.filter = is_eth_port_inactive_slave_filter
};
static const struct netdev_event_work_cmd
netdev_del_cmd = {
.cb = del_netdev_ips,
.filter = is_eth_port_of_netdev_filter
};
static const struct netdev_event_work_cmd bonding_event_ips_del_cmd = {
.cb = del_netdev_upper_ips, .filter = upper_device_filter};
struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
struct netdev_event_work_cmd cmds[ROCE_NETDEV_CALLBACK_SZ] = { {NULL} };
if (ndev->type != ARPHRD_ETHER)
return NOTIFY_DONE;
switch (event) {
case NETDEV_REGISTER:
case NETDEV_UP:
cmds[0] = bonding_default_del_cmd_join;
cmds[1] = add_default_gid_cmd;
cmds[2] = add_cmd;
break;
case NETDEV_UNREGISTER:
if (ndev->reg_state < NETREG_UNREGISTERED)
cmds[0] = del_cmd;
else
return NOTIFY_DONE;
break;
case NETDEV_CHANGEADDR:
cmds[0] = netdev_del_cmd;
if (ndev->reg_state == NETREG_REGISTERED) {
cmds[1] = add_default_gid_cmd;
cmds[2] = add_cmd;
}
break;
case NETDEV_CHANGEUPPER:
netdevice_event_changeupper(ndev,
container_of(ptr, struct netdev_notifier_changeupper_info, info),
cmds);
break;
case NETDEV_BONDING_FAILOVER:
cmds[0] = bonding_event_ips_del_cmd;
/* Add default GIDs of the bond device */
cmds[1] = bonding_default_add_cmd;
/* Add IP based GIDs of the bond device */
cmds[2] = add_cmd_upper_ips;
break;
default:
return NOTIFY_DONE;
}
return netdevice_queue_work(cmds, ndev);
}
static void update_gid_event_work_handler(struct work_struct *_work)
{
struct update_gid_event_work *work =
container_of(_work, struct update_gid_event_work, work);
ib_enum_all_roce_netdevs(is_eth_port_of_netdev_filter,
work->gid_attr.ndev,
callback_for_addr_gid_device_scan, work);
dev_put(work->gid_attr.ndev);
kfree(work);
}
static int addr_event(struct notifier_block *this, unsigned long event,
struct sockaddr *sa, struct net_device *ndev)
{
struct update_gid_event_work *work;
enum gid_op_type gid_op;
if (ndev->type != ARPHRD_ETHER)
return NOTIFY_DONE;
switch (event) {
case NETDEV_UP:
gid_op = GID_ADD;
break;
case NETDEV_DOWN:
gid_op = GID_DEL;
break;
default:
return NOTIFY_DONE;
}
work = kmalloc(sizeof(*work), GFP_ATOMIC);
if (!work)
return NOTIFY_DONE;
INIT_WORK(&work->work, update_gid_event_work_handler);
rdma_ip2gid(sa, &work->gid);
work->gid_op = gid_op;
memset(&work->gid_attr, 0, sizeof(work->gid_attr));
dev_hold(ndev);
work->gid_attr.ndev = ndev;
queue_work(gid_cache_wq, &work->work);
return NOTIFY_DONE;
}
static int inetaddr_event(struct notifier_block *this, unsigned long event,
void *ptr)
{
struct sockaddr_in in;
struct net_device *ndev;
struct in_ifaddr *ifa = ptr;
in.sin_family = AF_INET;
in.sin_addr.s_addr = ifa->ifa_address;
ndev = ifa->ifa_dev->dev;
return addr_event(this, event, (struct sockaddr *)&in, ndev);
}
static int inet6addr_event(struct notifier_block *this, unsigned long event,
void *ptr)
{
struct sockaddr_in6 in6;
struct net_device *ndev;
struct inet6_ifaddr *ifa6 = ptr;
in6.sin6_family = AF_INET6;
in6.sin6_addr = ifa6->addr;
ndev = ifa6->idev->dev;
return addr_event(this, event, (struct sockaddr *)&in6, ndev);
}
static struct notifier_block nb_netdevice = {
.notifier_call = netdevice_event
};
static struct notifier_block nb_inetaddr = {
.notifier_call = inetaddr_event
};
static struct notifier_block nb_inet6addr = {
.notifier_call = inet6addr_event
};
int __init roce_gid_mgmt_init(void)
{
gid_cache_wq = alloc_ordered_workqueue("gid-cache-wq", 0);
if (!gid_cache_wq)
return -ENOMEM;
register_inetaddr_notifier(&nb_inetaddr);
if (IS_ENABLED(CONFIG_IPV6))
register_inet6addr_notifier(&nb_inet6addr);
/* We relay on the netdevice notifier to enumerate all
* existing devices in the system. Register to this notifier
* last to make sure we will not miss any IP add/del
* callbacks.
*/
register_netdevice_notifier(&nb_netdevice);
return 0;
}
void __exit roce_gid_mgmt_cleanup(void)
{
if (IS_ENABLED(CONFIG_IPV6))
unregister_inet6addr_notifier(&nb_inet6addr);
unregister_inetaddr_notifier(&nb_inetaddr);
unregister_netdevice_notifier(&nb_netdevice);
/* Ensure all gid deletion tasks complete before we go down,
* to avoid any reference to free'd memory. By the time
* ib-core is removed, all physical devices have been removed,
* so no issue with remaining hardware contexts.
*/
destroy_workqueue(gid_cache_wq);
}
+745
View File
@@ -0,0 +1,745 @@
/*
* Copyright (c) 2016 HGST, a Western Digital Company.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*/
#include <linux/moduleparam.h>
#include <linux/slab.h>
#include <rdma/mr_pool.h>
#include <rdma/rw.h>
enum {
RDMA_RW_SINGLE_WR,
RDMA_RW_MULTI_WR,
RDMA_RW_MR,
RDMA_RW_SIG_MR,
};
static bool rdma_rw_force_mr;
module_param_named(force_mr, rdma_rw_force_mr, bool, 0);
MODULE_PARM_DESC(force_mr, "Force usage of MRs for RDMA READ/WRITE operations");
/*
* Check if the device might use memory registration. This is currently only
* true for iWarp devices. In the future we can hopefully fine tune this based
* on HCA driver input.
*/
static inline bool rdma_rw_can_use_mr(struct ib_device *dev, u8 port_num)
{
if (rdma_protocol_iwarp(dev, port_num))
return true;
if (unlikely(rdma_rw_force_mr))
return true;
return false;
}
/*
* Check if the device will use memory registration for this RW operation.
* We currently always use memory registrations for iWarp RDMA READs, and
* have a debug option to force usage of MRs.
*
* XXX: In the future we can hopefully fine tune this based on HCA driver
* input.
*/
static inline bool rdma_rw_io_needs_mr(struct ib_device *dev, u8 port_num,
enum dma_data_direction dir, int dma_nents)
{
if (rdma_protocol_iwarp(dev, port_num) && dir == DMA_FROM_DEVICE)
return true;
if (unlikely(rdma_rw_force_mr))
return true;
return false;
}
static inline u32 rdma_rw_fr_page_list_len(struct ib_device *dev)
{
/* arbitrary limit to avoid allocating gigantic resources */
return min_t(u32, dev->attrs.max_fast_reg_page_list_len, 256);
}
/* Caller must have zero-initialized *reg. */
static int rdma_rw_init_one_mr(struct ib_qp *qp, u8 port_num,
struct rdma_rw_reg_ctx *reg, struct scatterlist *sg,
u32 sg_cnt, u32 offset)
{
u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device);
u32 nents = min(sg_cnt, pages_per_mr);
int count = 0, ret;
reg->mr = ib_mr_pool_get(qp, &qp->rdma_mrs);
if (!reg->mr)
return -EAGAIN;
if (reg->mr->need_inval) {
reg->inv_wr.opcode = IB_WR_LOCAL_INV;
reg->inv_wr.ex.invalidate_rkey = reg->mr->lkey;
reg->inv_wr.next = &reg->reg_wr.wr;
count++;
} else {
reg->inv_wr.next = NULL;
}
ret = ib_map_mr_sg(reg->mr, sg, nents, &offset, PAGE_SIZE);
if (ret < 0 || ret < nents) {
ib_mr_pool_put(qp, &qp->rdma_mrs, reg->mr);
return -EINVAL;
}
reg->reg_wr.wr.opcode = IB_WR_REG_MR;
reg->reg_wr.mr = reg->mr;
reg->reg_wr.access = IB_ACCESS_LOCAL_WRITE;
if (rdma_protocol_iwarp(qp->device, port_num))
reg->reg_wr.access |= IB_ACCESS_REMOTE_WRITE;
count++;
reg->sge.addr = reg->mr->iova;
reg->sge.length = reg->mr->length;
return count;
}
static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
u8 port_num, struct scatterlist *sg, u32 sg_cnt, u32 offset,
u64 remote_addr, u32 rkey, enum dma_data_direction dir)
{
struct rdma_rw_reg_ctx *prev = NULL;
u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device);
int i, j, ret = 0, count = 0;
ctx->nr_ops = (sg_cnt + pages_per_mr - 1) / pages_per_mr;
ctx->reg = kcalloc(ctx->nr_ops, sizeof(*ctx->reg), GFP_KERNEL);
if (!ctx->reg) {
ret = -ENOMEM;
goto out;
}
for (i = 0; i < ctx->nr_ops; i++) {
struct rdma_rw_reg_ctx *reg = &ctx->reg[i];
u32 nents = min(sg_cnt, pages_per_mr);
ret = rdma_rw_init_one_mr(qp, port_num, reg, sg, sg_cnt,
offset);
if (ret < 0)
goto out_free;
count += ret;
if (prev) {
if (reg->mr->need_inval)
prev->wr.wr.next = &reg->inv_wr;
else
prev->wr.wr.next = &reg->reg_wr.wr;
}
reg->reg_wr.wr.next = &reg->wr.wr;
reg->wr.wr.sg_list = &reg->sge;
reg->wr.wr.num_sge = 1;
reg->wr.remote_addr = remote_addr;
reg->wr.rkey = rkey;
if (dir == DMA_TO_DEVICE) {
reg->wr.wr.opcode = IB_WR_RDMA_WRITE;
} else if (!rdma_cap_read_inv(qp->device, port_num)) {
reg->wr.wr.opcode = IB_WR_RDMA_READ;
} else {
reg->wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV;
reg->wr.wr.ex.invalidate_rkey = reg->mr->lkey;
}
count++;
remote_addr += reg->sge.length;
sg_cnt -= nents;
for (j = 0; j < nents; j++)
sg = sg_next(sg);
prev = reg;
offset = 0;
}
if (prev)
prev->wr.wr.next = NULL;
ctx->type = RDMA_RW_MR;
return count;
out_free:
while (--i >= 0)
ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg[i].mr);
kfree(ctx->reg);
out:
return ret;
}
static int rdma_rw_init_map_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
struct scatterlist *sg, u32 sg_cnt, u32 offset,
u64 remote_addr, u32 rkey, enum dma_data_direction dir)
{
struct ib_device *dev = qp->pd->device;
u32 max_sge = dir == DMA_TO_DEVICE ? qp->max_write_sge :
qp->max_read_sge;
struct ib_sge *sge;
u32 total_len = 0, i, j;
ctx->nr_ops = DIV_ROUND_UP(sg_cnt, max_sge);
ctx->map.sges = sge = kcalloc(sg_cnt, sizeof(*sge), GFP_KERNEL);
if (!ctx->map.sges)
goto out;
ctx->map.wrs = kcalloc(ctx->nr_ops, sizeof(*ctx->map.wrs), GFP_KERNEL);
if (!ctx->map.wrs)
goto out_free_sges;
for (i = 0; i < ctx->nr_ops; i++) {
struct ib_rdma_wr *rdma_wr = &ctx->map.wrs[i];
u32 nr_sge = min(sg_cnt, max_sge);
if (dir == DMA_TO_DEVICE)
rdma_wr->wr.opcode = IB_WR_RDMA_WRITE;
else
rdma_wr->wr.opcode = IB_WR_RDMA_READ;
rdma_wr->remote_addr = remote_addr + total_len;
rdma_wr->rkey = rkey;
rdma_wr->wr.num_sge = nr_sge;
rdma_wr->wr.sg_list = sge;
for (j = 0; j < nr_sge; j++, sg = sg_next(sg)) {
sge->addr = ib_sg_dma_address(dev, sg) + offset;
sge->length = ib_sg_dma_len(dev, sg) - offset;
sge->lkey = qp->pd->local_dma_lkey;
total_len += sge->length;
sge++;
sg_cnt--;
offset = 0;
}
rdma_wr->wr.next = i + 1 < ctx->nr_ops ?
&ctx->map.wrs[i + 1].wr : NULL;
}
ctx->type = RDMA_RW_MULTI_WR;
return ctx->nr_ops;
out_free_sges:
kfree(ctx->map.sges);
out:
return -ENOMEM;
}
static int rdma_rw_init_single_wr(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
struct scatterlist *sg, u32 offset, u64 remote_addr, u32 rkey,
enum dma_data_direction dir)
{
struct ib_device *dev = qp->pd->device;
struct ib_rdma_wr *rdma_wr = &ctx->single.wr;
ctx->nr_ops = 1;
ctx->single.sge.lkey = qp->pd->local_dma_lkey;
ctx->single.sge.addr = ib_sg_dma_address(dev, sg) + offset;
ctx->single.sge.length = ib_sg_dma_len(dev, sg) - offset;
memset(rdma_wr, 0, sizeof(*rdma_wr));
if (dir == DMA_TO_DEVICE)
rdma_wr->wr.opcode = IB_WR_RDMA_WRITE;
else
rdma_wr->wr.opcode = IB_WR_RDMA_READ;
rdma_wr->wr.sg_list = &ctx->single.sge;
rdma_wr->wr.num_sge = 1;
rdma_wr->remote_addr = remote_addr;
rdma_wr->rkey = rkey;
ctx->type = RDMA_RW_SINGLE_WR;
return 1;
}
/**
* rdma_rw_ctx_init - initialize a RDMA READ/WRITE context
* @ctx: context to initialize
* @qp: queue pair to operate on
* @port_num: port num to which the connection is bound
* @sg: scatterlist to READ/WRITE from/to
* @sg_cnt: number of entries in @sg
* @sg_offset: current byte offset into @sg
* @remote_addr:remote address to read/write (relative to @rkey)
* @rkey: remote key to operate on
* @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ
*
* Returns the number of WQEs that will be needed on the workqueue if
* successful, or a negative error code.
*/
int rdma_rw_ctx_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num,
struct scatterlist *sg, u32 sg_cnt, u32 sg_offset,
u64 remote_addr, u32 rkey, enum dma_data_direction dir)
{
struct ib_device *dev = qp->pd->device;
int ret;
ret = ib_dma_map_sg(dev, sg, sg_cnt, dir);
if (!ret)
return -ENOMEM;
sg_cnt = ret;
/*
* Skip to the S/G entry that sg_offset falls into:
*/
for (;;) {
u32 len = ib_sg_dma_len(dev, sg);
if (sg_offset < len)
break;
sg = sg_next(sg);
sg_offset -= len;
sg_cnt--;
}
ret = -EIO;
if (WARN_ON_ONCE(sg_cnt == 0))
goto out_unmap_sg;
if (rdma_rw_io_needs_mr(qp->device, port_num, dir, sg_cnt)) {
ret = rdma_rw_init_mr_wrs(ctx, qp, port_num, sg, sg_cnt,
sg_offset, remote_addr, rkey, dir);
} else if (sg_cnt > 1) {
ret = rdma_rw_init_map_wrs(ctx, qp, sg, sg_cnt, sg_offset,
remote_addr, rkey, dir);
} else {
ret = rdma_rw_init_single_wr(ctx, qp, sg, sg_offset,
remote_addr, rkey, dir);
}
if (ret < 0)
goto out_unmap_sg;
return ret;
out_unmap_sg:
ib_dma_unmap_sg(dev, sg, sg_cnt, dir);
return ret;
}
EXPORT_SYMBOL(rdma_rw_ctx_init);
/**
* rdma_rw_ctx_signature_init - initialize a RW context with signature offload
* @ctx: context to initialize
* @qp: queue pair to operate on
* @port_num: port num to which the connection is bound
* @sg: scatterlist to READ/WRITE from/to
* @sg_cnt: number of entries in @sg
* @prot_sg: scatterlist to READ/WRITE protection information from/to
* @prot_sg_cnt: number of entries in @prot_sg
* @sig_attrs: signature offloading algorithms
* @remote_addr:remote address to read/write (relative to @rkey)
* @rkey: remote key to operate on
* @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ
*
* Returns the number of WQEs that will be needed on the workqueue if
* successful, or a negative error code.
*/
int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
u8 port_num, struct scatterlist *sg, u32 sg_cnt,
struct scatterlist *prot_sg, u32 prot_sg_cnt,
struct ib_sig_attrs *sig_attrs,
u64 remote_addr, u32 rkey, enum dma_data_direction dir)
{
struct ib_device *dev = qp->pd->device;
u32 pages_per_mr = rdma_rw_fr_page_list_len(qp->pd->device);
struct ib_rdma_wr *rdma_wr;
struct ib_send_wr *prev_wr = NULL;
int count = 0, ret;
if (sg_cnt > pages_per_mr || prot_sg_cnt > pages_per_mr) {
pr_err("SG count too large\n");
return -EINVAL;
}
ret = ib_dma_map_sg(dev, sg, sg_cnt, dir);
if (!ret)
return -ENOMEM;
sg_cnt = ret;
ret = ib_dma_map_sg(dev, prot_sg, prot_sg_cnt, dir);
if (!ret) {
ret = -ENOMEM;
goto out_unmap_sg;
}
prot_sg_cnt = ret;
ctx->type = RDMA_RW_SIG_MR;
ctx->nr_ops = 1;
ctx->sig = kcalloc(1, sizeof(*ctx->sig), GFP_KERNEL);
if (!ctx->sig) {
ret = -ENOMEM;
goto out_unmap_prot_sg;
}
ret = rdma_rw_init_one_mr(qp, port_num, &ctx->sig->data, sg, sg_cnt, 0);
if (ret < 0)
goto out_free_ctx;
count += ret;
prev_wr = &ctx->sig->data.reg_wr.wr;
ret = rdma_rw_init_one_mr(qp, port_num, &ctx->sig->prot,
prot_sg, prot_sg_cnt, 0);
if (ret < 0)
goto out_destroy_data_mr;
count += ret;
if (ctx->sig->prot.inv_wr.next)
prev_wr->next = &ctx->sig->prot.inv_wr;
else
prev_wr->next = &ctx->sig->prot.reg_wr.wr;
prev_wr = &ctx->sig->prot.reg_wr.wr;
ctx->sig->sig_mr = ib_mr_pool_get(qp, &qp->sig_mrs);
if (!ctx->sig->sig_mr) {
ret = -EAGAIN;
goto out_destroy_prot_mr;
}
if (ctx->sig->sig_mr->need_inval) {
memset(&ctx->sig->sig_inv_wr, 0, sizeof(ctx->sig->sig_inv_wr));
ctx->sig->sig_inv_wr.opcode = IB_WR_LOCAL_INV;
ctx->sig->sig_inv_wr.ex.invalidate_rkey = ctx->sig->sig_mr->rkey;
prev_wr->next = &ctx->sig->sig_inv_wr;
prev_wr = &ctx->sig->sig_inv_wr;
}
ctx->sig->sig_wr.wr.opcode = IB_WR_REG_SIG_MR;
ctx->sig->sig_wr.wr.wr_cqe = NULL;
ctx->sig->sig_wr.wr.sg_list = &ctx->sig->data.sge;
ctx->sig->sig_wr.wr.num_sge = 1;
ctx->sig->sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE;
ctx->sig->sig_wr.sig_attrs = sig_attrs;
ctx->sig->sig_wr.sig_mr = ctx->sig->sig_mr;
if (prot_sg_cnt)
ctx->sig->sig_wr.prot = &ctx->sig->prot.sge;
prev_wr->next = &ctx->sig->sig_wr.wr;
prev_wr = &ctx->sig->sig_wr.wr;
count++;
ctx->sig->sig_sge.addr = 0;
ctx->sig->sig_sge.length = ctx->sig->data.sge.length;
if (sig_attrs->wire.sig_type != IB_SIG_TYPE_NONE)
ctx->sig->sig_sge.length += ctx->sig->prot.sge.length;
rdma_wr = &ctx->sig->data.wr;
rdma_wr->wr.sg_list = &ctx->sig->sig_sge;
rdma_wr->wr.num_sge = 1;
rdma_wr->remote_addr = remote_addr;
rdma_wr->rkey = rkey;
if (dir == DMA_TO_DEVICE)
rdma_wr->wr.opcode = IB_WR_RDMA_WRITE;
else
rdma_wr->wr.opcode = IB_WR_RDMA_READ;
prev_wr->next = &rdma_wr->wr;
prev_wr = &rdma_wr->wr;
count++;
return count;
out_destroy_prot_mr:
if (prot_sg_cnt)
ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->prot.mr);
out_destroy_data_mr:
ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->data.mr);
out_free_ctx:
kfree(ctx->sig);
out_unmap_prot_sg:
ib_dma_unmap_sg(dev, prot_sg, prot_sg_cnt, dir);
out_unmap_sg:
ib_dma_unmap_sg(dev, sg, sg_cnt, dir);
return ret;
}
EXPORT_SYMBOL(rdma_rw_ctx_signature_init);
/*
* Now that we are going to post the WRs we can update the lkey and need_inval
* state on the MRs. If we were doing this at init time, we would get double
* or missing invalidations if a context was initialized but not actually
* posted.
*/
static void rdma_rw_update_lkey(struct rdma_rw_reg_ctx *reg, bool need_inval)
{
reg->mr->need_inval = need_inval;
ib_update_fast_reg_key(reg->mr, ib_inc_rkey(reg->mr->lkey));
reg->reg_wr.key = reg->mr->lkey;
reg->sge.lkey = reg->mr->lkey;
}
/**
* rdma_rw_ctx_wrs - return chain of WRs for a RDMA READ or WRITE operation
* @ctx: context to operate on
* @qp: queue pair to operate on
* @port_num: port num to which the connection is bound
* @cqe: completion queue entry for the last WR
* @chain_wr: WR to append to the posted chain
*
* Return the WR chain for the set of RDMA READ/WRITE operations described by
* @ctx, as well as any memory registration operations needed. If @chain_wr
* is non-NULL the WR it points to will be appended to the chain of WRs posted.
* If @chain_wr is not set @cqe must be set so that the caller gets a
* completion notification.
*/
struct ib_send_wr *rdma_rw_ctx_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
u8 port_num, struct ib_cqe *cqe, struct ib_send_wr *chain_wr)
{
struct ib_send_wr *first_wr, *last_wr;
int i;
switch (ctx->type) {
case RDMA_RW_SIG_MR:
rdma_rw_update_lkey(&ctx->sig->data, true);
if (ctx->sig->prot.mr)
rdma_rw_update_lkey(&ctx->sig->prot, true);
ctx->sig->sig_mr->need_inval = true;
ib_update_fast_reg_key(ctx->sig->sig_mr,
ib_inc_rkey(ctx->sig->sig_mr->lkey));
ctx->sig->sig_sge.lkey = ctx->sig->sig_mr->lkey;
if (ctx->sig->data.inv_wr.next)
first_wr = &ctx->sig->data.inv_wr;
else
first_wr = &ctx->sig->data.reg_wr.wr;
last_wr = &ctx->sig->data.wr.wr;
break;
case RDMA_RW_MR:
for (i = 0; i < ctx->nr_ops; i++) {
rdma_rw_update_lkey(&ctx->reg[i],
ctx->reg[i].wr.wr.opcode !=
IB_WR_RDMA_READ_WITH_INV);
}
if (ctx->reg[0].inv_wr.next)
first_wr = &ctx->reg[0].inv_wr;
else
first_wr = &ctx->reg[0].reg_wr.wr;
last_wr = &ctx->reg[ctx->nr_ops - 1].wr.wr;
break;
case RDMA_RW_MULTI_WR:
first_wr = &ctx->map.wrs[0].wr;
last_wr = &ctx->map.wrs[ctx->nr_ops - 1].wr;
break;
case RDMA_RW_SINGLE_WR:
first_wr = &ctx->single.wr.wr;
last_wr = &ctx->single.wr.wr;
break;
default:
BUG();
}
if (chain_wr) {
last_wr->next = chain_wr;
} else {
last_wr->wr_cqe = cqe;
last_wr->send_flags |= IB_SEND_SIGNALED;
}
return first_wr;
}
EXPORT_SYMBOL(rdma_rw_ctx_wrs);
/**
* rdma_rw_ctx_post - post a RDMA READ or RDMA WRITE operation
* @ctx: context to operate on
* @qp: queue pair to operate on
* @port_num: port num to which the connection is bound
* @cqe: completion queue entry for the last WR
* @chain_wr: WR to append to the posted chain
*
* Post the set of RDMA READ/WRITE operations described by @ctx, as well as
* any memory registration operations needed. If @chain_wr is non-NULL the
* WR it points to will be appended to the chain of WRs posted. If @chain_wr
* is not set @cqe must be set so that the caller gets a completion
* notification.
*/
int rdma_rw_ctx_post(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num,
struct ib_cqe *cqe, struct ib_send_wr *chain_wr)
{
struct ib_send_wr *first_wr;
first_wr = rdma_rw_ctx_wrs(ctx, qp, port_num, cqe, chain_wr);
return ib_post_send(qp, first_wr, NULL);
}
EXPORT_SYMBOL(rdma_rw_ctx_post);
/**
* rdma_rw_ctx_destroy - release all resources allocated by rdma_rw_ctx_init
* @ctx: context to release
* @qp: queue pair to operate on
* @port_num: port num to which the connection is bound
* @sg: scatterlist that was used for the READ/WRITE
* @sg_cnt: number of entries in @sg
* @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ
*/
void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp, u8 port_num,
struct scatterlist *sg, u32 sg_cnt, enum dma_data_direction dir)
{
int i;
switch (ctx->type) {
case RDMA_RW_MR:
for (i = 0; i < ctx->nr_ops; i++)
ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg[i].mr);
kfree(ctx->reg);
break;
case RDMA_RW_MULTI_WR:
kfree(ctx->map.wrs);
kfree(ctx->map.sges);
break;
case RDMA_RW_SINGLE_WR:
break;
default:
BUG();
break;
}
ib_dma_unmap_sg(qp->pd->device, sg, sg_cnt, dir);
}
EXPORT_SYMBOL(rdma_rw_ctx_destroy);
/**
* rdma_rw_ctx_destroy_signature - release all resources allocated by
* rdma_rw_ctx_init_signature
* @ctx: context to release
* @qp: queue pair to operate on
* @port_num: port num to which the connection is bound
* @sg: scatterlist that was used for the READ/WRITE
* @sg_cnt: number of entries in @sg
* @prot_sg: scatterlist that was used for the READ/WRITE of the PI
* @prot_sg_cnt: number of entries in @prot_sg
* @dir: %DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ
*/
void rdma_rw_ctx_destroy_signature(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
u8 port_num, struct scatterlist *sg, u32 sg_cnt,
struct scatterlist *prot_sg, u32 prot_sg_cnt,
enum dma_data_direction dir)
{
if (WARN_ON_ONCE(ctx->type != RDMA_RW_SIG_MR))
return;
ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->data.mr);
ib_dma_unmap_sg(qp->pd->device, sg, sg_cnt, dir);
if (ctx->sig->prot.mr) {
ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->sig->prot.mr);
ib_dma_unmap_sg(qp->pd->device, prot_sg, prot_sg_cnt, dir);
}
ib_mr_pool_put(qp, &qp->sig_mrs, ctx->sig->sig_mr);
kfree(ctx->sig);
}
EXPORT_SYMBOL(rdma_rw_ctx_destroy_signature);
/**
* rdma_rw_mr_factor - return number of MRs required for a payload
* @device: device handling the connection
* @port_num: port num to which the connection is bound
* @maxpages: maximum payload pages per rdma_rw_ctx
*
* Returns the number of MRs the device requires to move @maxpayload
* bytes. The returned value is used during transport creation to
* compute max_rdma_ctxts and the size of the transport's Send and
* Send Completion Queues.
*/
unsigned int rdma_rw_mr_factor(struct ib_device *device, u8 port_num,
unsigned int maxpages)
{
unsigned int mr_pages;
if (rdma_rw_can_use_mr(device, port_num))
mr_pages = rdma_rw_fr_page_list_len(device);
else
mr_pages = device->attrs.max_sge_rd;
return DIV_ROUND_UP(maxpages, mr_pages);
}
EXPORT_SYMBOL(rdma_rw_mr_factor);
void rdma_rw_init_qp(struct ib_device *dev, struct ib_qp_init_attr *attr)
{
u32 factor;
WARN_ON_ONCE(attr->port_num == 0);
/*
* Each context needs at least one RDMA READ or WRITE WR.
*
* For some hardware we might need more, eventually we should ask the
* HCA driver for a multiplier here.
*/
factor = 1;
/*
* If the devices needs MRs to perform RDMA READ or WRITE operations,
* we'll need two additional MRs for the registrations and the
* invalidation.
*/
if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN)
factor += 6; /* (inv + reg) * (data + prot + sig) */
else if (rdma_rw_can_use_mr(dev, attr->port_num))
factor += 2; /* inv + reg */
attr->cap.max_send_wr += factor * attr->cap.max_rdma_ctxs;
/*
* But maybe we were just too high in the sky and the device doesn't
* even support all we need, and we'll have to live with what we get..
*/
attr->cap.max_send_wr =
min_t(u32, attr->cap.max_send_wr, dev->attrs.max_qp_wr);
}
int rdma_rw_init_mrs(struct ib_qp *qp, struct ib_qp_init_attr *attr)
{
struct ib_device *dev = qp->pd->device;
u32 nr_mrs = 0, nr_sig_mrs = 0;
int ret = 0;
if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN) {
nr_sig_mrs = attr->cap.max_rdma_ctxs;
nr_mrs = attr->cap.max_rdma_ctxs * 2;
} else if (rdma_rw_can_use_mr(dev, attr->port_num)) {
nr_mrs = attr->cap.max_rdma_ctxs;
}
if (nr_mrs) {
ret = ib_mr_pool_init(qp, &qp->rdma_mrs, nr_mrs,
IB_MR_TYPE_MEM_REG,
rdma_rw_fr_page_list_len(dev));
if (ret) {
pr_err("%s: failed to allocated %d MRs\n",
__func__, nr_mrs);
return ret;
}
}
if (nr_sig_mrs) {
ret = ib_mr_pool_init(qp, &qp->sig_mrs, nr_sig_mrs,
IB_MR_TYPE_SIGNATURE, 2);
if (ret) {
pr_err("%s: failed to allocated %d SIG MRs\n",
__func__, nr_mrs);
goto out_free_rdma_mrs;
}
}
return 0;
out_free_rdma_mrs:
ib_mr_pool_destroy(qp, &qp->rdma_mrs);
return ret;
}
void rdma_rw_cleanup_mrs(struct ib_qp *qp)
{
ib_mr_pool_destroy(qp, &qp->sig_mrs);
ib_mr_pool_destroy(qp, &qp->rdma_mrs);
}
+66
View File
@@ -0,0 +1,66 @@
/*
* Copyright (c) 2004 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Voltaire, Inc. All rights reserved.
* Copyright (c) 2006 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef SA_H
#define SA_H
#include <rdma/ib_sa.h>
static inline void ib_sa_client_get(struct ib_sa_client *client)
{
atomic_inc(&client->users);
}
static inline void ib_sa_client_put(struct ib_sa_client *client)
{
if (atomic_dec_and_test(&client->users))
complete(&client->comp);
}
int ib_sa_mcmember_rec_query(struct ib_sa_client *client,
struct ib_device *device, u8 port_num,
u8 method,
struct ib_sa_mcmember_rec *rec,
ib_sa_comp_mask comp_mask,
int timeout_ms, gfp_t gfp_mask,
void (*callback)(int status,
struct ib_sa_mcmember_rec *resp,
void *context),
void *context,
struct ib_sa_query **sa_query);
int mcast_init(void);
void mcast_cleanup(void);
#endif /* SA_H */
File diff suppressed because it is too large. Load diff
+749
View File
@@ -0,0 +1,749 @@
/*
* Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/security.h>
#include <linux/completion.h>
#include <linux/list.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_cache.h>
#include "core_priv.h"
#include "mad_priv.h"
static struct pkey_index_qp_list *get_pkey_idx_qp_list(struct ib_port_pkey *pp)
{
struct pkey_index_qp_list *pkey = NULL;
struct pkey_index_qp_list *tmp_pkey;
struct ib_device *dev = pp->sec->dev;
spin_lock(&dev->port_pkey_list[pp->port_num].list_lock);
list_for_each_entry(tmp_pkey,
&dev->port_pkey_list[pp->port_num].pkey_list,
pkey_index_list) {
if (tmp_pkey->pkey_index == pp->pkey_index) {
pkey = tmp_pkey;
break;
}
}
spin_unlock(&dev->port_pkey_list[pp->port_num].list_lock);
return pkey;
}
static int get_pkey_and_subnet_prefix(struct ib_port_pkey *pp,
u16 *pkey,
u64 *subnet_prefix)
{
struct ib_device *dev = pp->sec->dev;
int ret;
ret = ib_get_cached_pkey(dev, pp->port_num, pp->pkey_index, pkey);
if (ret)
return ret;
ret = ib_get_cached_subnet_prefix(dev, pp->port_num, subnet_prefix);
return ret;
}
static int enforce_qp_pkey_security(u16 pkey,
u64 subnet_prefix,
struct ib_qp_security *qp_sec)
{
struct ib_qp_security *shared_qp_sec;
int ret;
ret = security_ib_pkey_access(qp_sec->security, subnet_prefix, pkey);
if (ret)
return ret;
list_for_each_entry(shared_qp_sec,
&qp_sec->shared_qp_list,
shared_qp_list) {
ret = security_ib_pkey_access(shared_qp_sec->security,
subnet_prefix,
pkey);
if (ret)
return ret;
}
return 0;
}
/* The caller of this function must hold the QP security
* mutex of the QP of the security structure in *pps.
*
* It takes separate ports_pkeys and security structure
* because in some cases the pps will be for a new settings
* or the pps will be for the real QP and security structure
* will be for a shared QP.
*/
static int check_qp_port_pkey_settings(struct ib_ports_pkeys *pps,
struct ib_qp_security *sec)
{
u64 subnet_prefix;
u16 pkey;
int ret = 0;
if (!pps)
return 0;
if (pps->main.state != IB_PORT_PKEY_NOT_VALID) {
ret = get_pkey_and_subnet_prefix(&pps->main,
&pkey,
&subnet_prefix);
if (ret)
return ret;
ret = enforce_qp_pkey_security(pkey,
subnet_prefix,
sec);
if (ret)
return ret;
}
if (pps->alt.state != IB_PORT_PKEY_NOT_VALID) {
ret = get_pkey_and_subnet_prefix(&pps->alt,
&pkey,
&subnet_prefix);
if (ret)
return ret;
ret = enforce_qp_pkey_security(pkey,
subnet_prefix,
sec);
}
return ret;
}
/* The caller of this function must hold the QP security
* mutex.
*/
static void qp_to_error(struct ib_qp_security *sec)
{
struct ib_qp_security *shared_qp_sec;
struct ib_qp_attr attr = {
.qp_state = IB_QPS_ERR
};
struct ib_event event = {
.event = IB_EVENT_QP_FATAL
};
/* If the QP is in the process of being destroyed
* the qp pointer in the security structure is
* undefined. It cannot be modified now.
*/
if (sec->destroying)
return;
ib_modify_qp(sec->qp,
&attr,
IB_QP_STATE);
if (sec->qp->event_handler && sec->qp->qp_context) {
event.element.qp = sec->qp;
sec->qp->event_handler(&event,
sec->qp->qp_context);
}
list_for_each_entry(shared_qp_sec,
&sec->shared_qp_list,
shared_qp_list) {
struct ib_qp *qp = shared_qp_sec->qp;
if (qp->event_handler && qp->qp_context) {
event.element.qp = qp;
event.device = qp->device;
qp->event_handler(&event,
qp->qp_context);
}
}
}
static inline void check_pkey_qps(struct pkey_index_qp_list *pkey,
struct ib_device *device,
u8 port_num,
u64 subnet_prefix)
{
struct ib_port_pkey *pp, *tmp_pp;
bool comp;
LIST_HEAD(to_error_list);
u16 pkey_val;
if (!ib_get_cached_pkey(device,
port_num,
pkey->pkey_index,
&pkey_val)) {
spin_lock(&pkey->qp_list_lock);
list_for_each_entry(pp, &pkey->qp_list, qp_list) {
if (atomic_read(&pp->sec->error_list_count))
continue;
if (enforce_qp_pkey_security(pkey_val,
subnet_prefix,
pp->sec)) {
atomic_inc(&pp->sec->error_list_count);
list_add(&pp->to_error_list,
&to_error_list);
}
}
spin_unlock(&pkey->qp_list_lock);
}
list_for_each_entry_safe(pp,
tmp_pp,
&to_error_list,
to_error_list) {
mutex_lock(&pp->sec->mutex);
qp_to_error(pp->sec);
list_del(&pp->to_error_list);
atomic_dec(&pp->sec->error_list_count);
comp = pp->sec->destroying;
mutex_unlock(&pp->sec->mutex);
if (comp)
complete(&pp->sec->error_complete);
}
}
/* The caller of this function must hold the QP security
* mutex.
*/
static int port_pkey_list_insert(struct ib_port_pkey *pp)
{
struct pkey_index_qp_list *tmp_pkey;
struct pkey_index_qp_list *pkey;
struct ib_device *dev;
u8 port_num = pp->port_num;
int ret = 0;
if (pp->state != IB_PORT_PKEY_VALID)
return 0;
dev = pp->sec->dev;
pkey = get_pkey_idx_qp_list(pp);
if (!pkey) {
bool found = false;
pkey = kzalloc(sizeof(*pkey), GFP_KERNEL);
if (!pkey)
return -ENOMEM;
spin_lock(&dev->port_pkey_list[port_num].list_lock);
/* Check for the PKey again. A racing process may
* have created it.
*/
list_for_each_entry(tmp_pkey,
&dev->port_pkey_list[port_num].pkey_list,
pkey_index_list) {
if (tmp_pkey->pkey_index == pp->pkey_index) {
kfree(pkey);
pkey = tmp_pkey;
found = true;
break;
}
}
if (!found) {
pkey->pkey_index = pp->pkey_index;
spin_lock_init(&pkey->qp_list_lock);
INIT_LIST_HEAD(&pkey->qp_list);
list_add(&pkey->pkey_index_list,
&dev->port_pkey_list[port_num].pkey_list);
}
spin_unlock(&dev->port_pkey_list[port_num].list_lock);
}
spin_lock(&pkey->qp_list_lock);
list_add(&pp->qp_list, &pkey->qp_list);
spin_unlock(&pkey->qp_list_lock);
pp->state = IB_PORT_PKEY_LISTED;
return ret;
}
/* The caller of this function must hold the QP security
* mutex.
*/
static void port_pkey_list_remove(struct ib_port_pkey *pp)
{
struct pkey_index_qp_list *pkey;
if (pp->state != IB_PORT_PKEY_LISTED)
return;
pkey = get_pkey_idx_qp_list(pp);
spin_lock(&pkey->qp_list_lock);
list_del(&pp->qp_list);
spin_unlock(&pkey->qp_list_lock);
/* The setting may still be valid, i.e. after
* a destroy has failed for example.
*/
pp->state = IB_PORT_PKEY_VALID;
}
static void destroy_qp_security(struct ib_qp_security *sec)
{
security_ib_free_security(sec->security);
kfree(sec->ports_pkeys);
kfree(sec);
}
/* The caller of this function must hold the QP security
* mutex.
*/
static struct ib_ports_pkeys *get_new_pps(const struct ib_qp *qp,
const struct ib_qp_attr *qp_attr,
int qp_attr_mask)
{
struct ib_ports_pkeys *new_pps;
struct ib_ports_pkeys *qp_pps = qp->qp_sec->ports_pkeys;
new_pps = kzalloc(sizeof(*new_pps), GFP_KERNEL);
if (!new_pps)
return NULL;
if (qp_attr_mask & IB_QP_PORT)
new_pps->main.port_num = qp_attr->port_num;
else if (qp_pps)
new_pps->main.port_num = qp_pps->main.port_num;
if (qp_attr_mask & IB_QP_PKEY_INDEX)
new_pps->main.pkey_index = qp_attr->pkey_index;
else if (qp_pps)
new_pps->main.pkey_index = qp_pps->main.pkey_index;
if (((qp_attr_mask & IB_QP_PKEY_INDEX) &&
(qp_attr_mask & IB_QP_PORT)) ||
(qp_pps && qp_pps->main.state != IB_PORT_PKEY_NOT_VALID))
new_pps->main.state = IB_PORT_PKEY_VALID;
if (qp_attr_mask & IB_QP_ALT_PATH) {
new_pps->alt.port_num = qp_attr->alt_port_num;
new_pps->alt.pkey_index = qp_attr->alt_pkey_index;
new_pps->alt.state = IB_PORT_PKEY_VALID;
} else if (qp_pps) {
new_pps->alt.port_num = qp_pps->alt.port_num;
new_pps->alt.pkey_index = qp_pps->alt.pkey_index;
if (qp_pps->alt.state != IB_PORT_PKEY_NOT_VALID)
new_pps->alt.state = IB_PORT_PKEY_VALID;
}
new_pps->main.sec = qp->qp_sec;
new_pps->alt.sec = qp->qp_sec;
return new_pps;
}
int ib_open_shared_qp_security(struct ib_qp *qp, struct ib_device *dev)
{
struct ib_qp *real_qp = qp->real_qp;
int ret;
ret = ib_create_qp_security(qp, dev);
if (ret)
return ret;
if (!qp->qp_sec)
return 0;
mutex_lock(&real_qp->qp_sec->mutex);
ret = check_qp_port_pkey_settings(real_qp->qp_sec->ports_pkeys,
qp->qp_sec);
if (ret)
goto ret;
if (qp != real_qp)
list_add(&qp->qp_sec->shared_qp_list,
&real_qp->qp_sec->shared_qp_list);
ret:
mutex_unlock(&real_qp->qp_sec->mutex);
if (ret)
destroy_qp_security(qp->qp_sec);
return ret;
}
void ib_close_shared_qp_security(struct ib_qp_security *sec)
{
struct ib_qp *real_qp = sec->qp->real_qp;
mutex_lock(&real_qp->qp_sec->mutex);
list_del(&sec->shared_qp_list);
mutex_unlock(&real_qp->qp_sec->mutex);
destroy_qp_security(sec);
}
int ib_create_qp_security(struct ib_qp *qp, struct ib_device *dev)
{
u8 i = rdma_start_port(dev);
bool is_ib = false;
int ret;
while (i <= rdma_end_port(dev) && !is_ib)
is_ib = rdma_protocol_ib(dev, i++);
/* If this isn't an IB device don't create the security context */
if (!is_ib)
return 0;
qp->qp_sec = kzalloc(sizeof(*qp->qp_sec), GFP_KERNEL);
if (!qp->qp_sec)
return -ENOMEM;
qp->qp_sec->qp = qp;
qp->qp_sec->dev = dev;
mutex_init(&qp->qp_sec->mutex);
INIT_LIST_HEAD(&qp->qp_sec->shared_qp_list);
atomic_set(&qp->qp_sec->error_list_count, 0);
init_completion(&qp->qp_sec->error_complete);
ret = security_ib_alloc_security(&qp->qp_sec->security);
if (ret) {
kfree(qp->qp_sec);
qp->qp_sec = NULL;
}
return ret;
}
EXPORT_SYMBOL(ib_create_qp_security);
void ib_destroy_qp_security_begin(struct ib_qp_security *sec)
{
/* Return if not IB */
if (!sec)
return;
mutex_lock(&sec->mutex);
/* Remove the QP from the lists so it won't get added to
* a to_error_list during the destroy process.
*/
if (sec->ports_pkeys) {
port_pkey_list_remove(&sec->ports_pkeys->main);
port_pkey_list_remove(&sec->ports_pkeys->alt);
}
/* If the QP is already in one or more of those lists
* the destroying flag will ensure the to error flow
* doesn't operate on an undefined QP.
*/
sec->destroying = true;
/* Record the error list count to know how many completions
* to wait for.
*/
sec->error_comps_pending = atomic_read(&sec->error_list_count);
mutex_unlock(&sec->mutex);
}
void ib_destroy_qp_security_abort(struct ib_qp_security *sec)
{
int ret;
int i;
/* Return if not IB */
if (!sec)
return;
/* If a concurrent cache update is in progress this
* QP security could be marked for an error state
* transition. Wait for this to complete.
*/
for (i = 0; i < sec->error_comps_pending; i++)
wait_for_completion(&sec->error_complete);
mutex_lock(&sec->mutex);
sec->destroying = false;
/* Restore the position in the lists and verify
* access is still allowed in case a cache update
* occurred while attempting to destroy.
*
* Because these setting were listed already
* and removed during ib_destroy_qp_security_begin
* we know the pkey_index_qp_list for the PKey
* already exists so port_pkey_list_insert won't fail.
*/
if (sec->ports_pkeys) {
port_pkey_list_insert(&sec->ports_pkeys->main);
port_pkey_list_insert(&sec->ports_pkeys->alt);
}
ret = check_qp_port_pkey_settings(sec->ports_pkeys, sec);
if (ret)
qp_to_error(sec);
mutex_unlock(&sec->mutex);
}
void ib_destroy_qp_security_end(struct ib_qp_security *sec)
{
int i;
/* Return if not IB */
if (!sec)
return;
/* If a concurrent cache update is occurring we must
* wait until this QP security structure is processed
* in the QP to error flow before destroying it because
* the to_error_list is in use.
*/
for (i = 0; i < sec->error_comps_pending; i++)
wait_for_completion(&sec->error_complete);
destroy_qp_security(sec);
}
void ib_security_cache_change(struct ib_device *device,
u8 port_num,
u64 subnet_prefix)
{
struct pkey_index_qp_list *pkey;
list_for_each_entry(pkey,
&device->port_pkey_list[port_num].pkey_list,
pkey_index_list) {
check_pkey_qps(pkey,
device,
port_num,
subnet_prefix);
}
}
void ib_security_destroy_port_pkey_list(struct ib_device *device)
{
struct pkey_index_qp_list *pkey, *tmp_pkey;
int i;
for (i = rdma_start_port(device); i <= rdma_end_port(device); i++) {
spin_lock(&device->port_pkey_list[i].list_lock);
list_for_each_entry_safe(pkey,
tmp_pkey,
&device->port_pkey_list[i].pkey_list,
pkey_index_list) {
list_del(&pkey->pkey_index_list);
kfree(pkey);
}
spin_unlock(&device->port_pkey_list[i].list_lock);
}
}
int ib_security_modify_qp(struct ib_qp *qp,
struct ib_qp_attr *qp_attr,
int qp_attr_mask,
struct ib_udata *udata)
{
int ret = 0;
struct ib_ports_pkeys *tmp_pps;
struct ib_ports_pkeys *new_pps = NULL;
struct ib_qp *real_qp = qp->real_qp;
bool special_qp = (real_qp->qp_type == IB_QPT_SMI ||
real_qp->qp_type == IB_QPT_GSI ||
real_qp->qp_type >= IB_QPT_RESERVED1);
bool pps_change = ((qp_attr_mask & (IB_QP_PKEY_INDEX | IB_QP_PORT)) ||
(qp_attr_mask & IB_QP_ALT_PATH));
WARN_ONCE((qp_attr_mask & IB_QP_PORT &&
rdma_protocol_ib(real_qp->device, qp_attr->port_num) &&
!real_qp->qp_sec),
"%s: QP security is not initialized for IB QP: %d\n",
__func__, real_qp->qp_num);
/* The port/pkey settings are maintained only for the real QP. Open
* handles on the real QP will be in the shared_qp_list. When
* enforcing security on the real QP all the shared QPs will be
* checked as well.
*/
if (pps_change && !special_qp && real_qp->qp_sec) {
mutex_lock(&real_qp->qp_sec->mutex);
new_pps = get_new_pps(real_qp,
qp_attr,
qp_attr_mask);
if (!new_pps) {
mutex_unlock(&real_qp->qp_sec->mutex);
return -ENOMEM;
}
/* Add this QP to the lists for the new port
* and pkey settings before checking for permission
* in case there is a concurrent cache update
* occurring. Walking the list for a cache change
* doesn't acquire the security mutex unless it's
* sending the QP to error.
*/
ret = port_pkey_list_insert(&new_pps->main);
if (!ret)
ret = port_pkey_list_insert(&new_pps->alt);
if (!ret)
ret = check_qp_port_pkey_settings(new_pps,
real_qp->qp_sec);
}
if (!ret)
ret = real_qp->device->modify_qp(real_qp,
qp_attr,
qp_attr_mask,
udata);
if (new_pps) {
/* Clean up the lists and free the appropriate
* ports_pkeys structure.
*/
if (ret) {
tmp_pps = new_pps;
} else {
tmp_pps = real_qp->qp_sec->ports_pkeys;
real_qp->qp_sec->ports_pkeys = new_pps;
}
if (tmp_pps) {
port_pkey_list_remove(&tmp_pps->main);
port_pkey_list_remove(&tmp_pps->alt);
}
kfree(tmp_pps);
mutex_unlock(&real_qp->qp_sec->mutex);
}
return ret;
}
static int ib_security_pkey_access(struct ib_device *dev,
u8 port_num,
u16 pkey_index,
void *sec)
{
u64 subnet_prefix;
u16 pkey;
int ret;
if (!rdma_protocol_ib(dev, port_num))
return 0;
ret = ib_get_cached_pkey(dev, port_num, pkey_index, &pkey);
if (ret)
return ret;
ret = ib_get_cached_subnet_prefix(dev, port_num, &subnet_prefix);
if (ret)
return ret;
return security_ib_pkey_access(sec, subnet_prefix, pkey);
}
static int ib_mad_agent_security_change(struct notifier_block *nb,
unsigned long event,
void *data)
{
struct ib_mad_agent *ag = container_of(nb, struct ib_mad_agent, lsm_nb);
if (event != LSM_POLICY_CHANGE)
return NOTIFY_DONE;
ag->smp_allowed = !security_ib_endport_manage_subnet(ag->security,
ag->device->name,
ag->port_num);
return NOTIFY_OK;
}
int ib_mad_agent_security_setup(struct ib_mad_agent *agent,
enum ib_qp_type qp_type)
{
int ret;
if (!rdma_protocol_ib(agent->device, agent->port_num))
return 0;
ret = security_ib_alloc_security(&agent->security);
if (ret)
return ret;
if (qp_type != IB_QPT_SMI)
return 0;
ret = security_ib_endport_manage_subnet(agent->security,
agent->device->name,
agent->port_num);
if (ret)
goto free_security;
agent->lsm_nb.notifier_call = ib_mad_agent_security_change;
ret = register_lsm_notifier(&agent->lsm_nb);
if (ret)
goto free_security;
agent->smp_allowed = true;
agent->lsm_nb_reg = true;
return 0;
free_security:
security_ib_free_security(agent->security);
return ret;
}
void ib_mad_agent_security_cleanup(struct ib_mad_agent *agent)
{
if (!rdma_protocol_ib(agent->device, agent->port_num))
return;
if (agent->lsm_nb_reg)
unregister_lsm_notifier(&agent->lsm_nb);
security_ib_free_security(agent->security);
}
int ib_mad_enforce_security(struct ib_mad_agent_private *map, u16 pkey_index)
{
if (!rdma_protocol_ib(map->agent.device, map->agent.port_num))
return 0;
if (map->agent.qp->qp_type == IB_QPT_SMI) {
if (!map->agent.smp_allowed)
return -EACCES;
return 0;
}
return ib_security_pkey_access(map->agent.device,
map->agent.port_num,
pkey_index,
map->agent.security);
}
+338
View File
@@ -0,0 +1,338 @@
/*
* Copyright (c) 2004, 2005 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2004, 2005 Infinicon Corporation. All rights reserved.
* Copyright (c) 2004, 2005 Intel Corporation. All rights reserved.
* Copyright (c) 2004, 2005 Topspin Corporation. All rights reserved.
* Copyright (c) 2004-2007 Voltaire Corporation. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2014 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
#include <rdma/ib_smi.h>
#include "smi.h"
#include "opa_smi.h"
static enum smi_action __smi_handle_dr_smp_send(bool is_switch, int port_num,
u8 *hop_ptr, u8 hop_cnt,
const u8 *initial_path,
const u8 *return_path,
u8 direction,
bool dr_dlid_is_permissive,
bool dr_slid_is_permissive)
{
/* See section 14.2.2.2, Vol 1 IB spec */
/* C14-6 -- valid hop_cnt values are from 0 to 63 */
if (hop_cnt >= IB_SMP_MAX_PATH_HOPS)
return IB_SMI_DISCARD;
if (!direction) {
/* C14-9:1 */
if (hop_cnt && *hop_ptr == 0) {
(*hop_ptr)++;
return (initial_path[*hop_ptr] ==
port_num ? IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-9:2 */
if (*hop_ptr && *hop_ptr < hop_cnt) {
if (!is_switch)
return IB_SMI_DISCARD;
/* return_path set when received */
(*hop_ptr)++;
return (initial_path[*hop_ptr] ==
port_num ? IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-9:3 -- We're at the end of the DR segment of path */
if (*hop_ptr == hop_cnt) {
/* return_path set when received */
(*hop_ptr)++;
return (is_switch ||
dr_dlid_is_permissive ?
IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-9:4 -- hop_ptr = hop_cnt + 1 -> give to SMA/SM */
/* C14-9:5 -- Fail unreasonable hop pointer */
return (*hop_ptr == hop_cnt + 1 ? IB_SMI_HANDLE : IB_SMI_DISCARD);
} else {
/* C14-13:1 */
if (hop_cnt && *hop_ptr == hop_cnt + 1) {
(*hop_ptr)--;
return (return_path[*hop_ptr] ==
port_num ? IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-13:2 */
if (2 <= *hop_ptr && *hop_ptr <= hop_cnt) {
if (!is_switch)
return IB_SMI_DISCARD;
(*hop_ptr)--;
return (return_path[*hop_ptr] ==
port_num ? IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-13:3 -- at the end of the DR segment of path */
if (*hop_ptr == 1) {
(*hop_ptr)--;
/* C14-13:3 -- SMPs destined for SM shouldn't be here */
return (is_switch ||
dr_slid_is_permissive ?
IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-13:4 -- hop_ptr = 0 -> should have gone to SM */
if (*hop_ptr == 0)
return IB_SMI_HANDLE;
/* C14-13:5 -- Check for unreasonable hop pointer */
return IB_SMI_DISCARD;
}
}
/*
* Fixup a directed route SMP for sending
* Return IB_SMI_DISCARD if the SMP should be discarded
*/
enum smi_action smi_handle_dr_smp_send(struct ib_smp *smp,
bool is_switch, int port_num)
{
return __smi_handle_dr_smp_send(is_switch, port_num,
&smp->hop_ptr, smp->hop_cnt,
smp->initial_path,
smp->return_path,
ib_get_smp_direction(smp),
smp->dr_dlid == IB_LID_PERMISSIVE,
smp->dr_slid == IB_LID_PERMISSIVE);
}
enum smi_action opa_smi_handle_dr_smp_send(struct opa_smp *smp,
bool is_switch, int port_num)
{
return __smi_handle_dr_smp_send(is_switch, port_num,
&smp->hop_ptr, smp->hop_cnt,
smp->route.dr.initial_path,
smp->route.dr.return_path,
opa_get_smp_direction(smp),
smp->route.dr.dr_dlid ==
OPA_LID_PERMISSIVE,
smp->route.dr.dr_slid ==
OPA_LID_PERMISSIVE);
}
static enum smi_action __smi_handle_dr_smp_recv(bool is_switch, int port_num,
int phys_port_cnt,
u8 *hop_ptr, u8 hop_cnt,
const u8 *initial_path,
u8 *return_path,
u8 direction,
bool dr_dlid_is_permissive,
bool dr_slid_is_permissive)
{
/* See section 14.2.2.2, Vol 1 IB spec */
/* C14-6 -- valid hop_cnt values are from 0 to 63 */
if (hop_cnt >= IB_SMP_MAX_PATH_HOPS)
return IB_SMI_DISCARD;
if (!direction) {
/* C14-9:1 -- sender should have incremented hop_ptr */
if (hop_cnt && *hop_ptr == 0)
return IB_SMI_DISCARD;
/* C14-9:2 -- intermediate hop */
if (*hop_ptr && *hop_ptr < hop_cnt) {
if (!is_switch)
return IB_SMI_DISCARD;
return_path[*hop_ptr] = port_num;
/* hop_ptr updated when sending */
return (initial_path[*hop_ptr+1] <= phys_port_cnt ?
IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-9:3 -- We're at the end of the DR segment of path */
if (*hop_ptr == hop_cnt) {
if (hop_cnt)
return_path[*hop_ptr] = port_num;
/* hop_ptr updated when sending */
return (is_switch ||
dr_dlid_is_permissive ?
IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-9:4 -- hop_ptr = hop_cnt + 1 -> give to SMA/SM */
/* C14-9:5 -- fail unreasonable hop pointer */
return (*hop_ptr == hop_cnt + 1 ? IB_SMI_HANDLE : IB_SMI_DISCARD);
} else {
/* C14-13:1 */
if (hop_cnt && *hop_ptr == hop_cnt + 1) {
(*hop_ptr)--;
return (return_path[*hop_ptr] ==
port_num ? IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-13:2 */
if (2 <= *hop_ptr && *hop_ptr <= hop_cnt) {
if (!is_switch)
return IB_SMI_DISCARD;
/* hop_ptr updated when sending */
return (return_path[*hop_ptr-1] <= phys_port_cnt ?
IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-13:3 -- We're at the end of the DR segment of path */
if (*hop_ptr == 1) {
if (dr_slid_is_permissive) {
/* giving SMP to SM - update hop_ptr */
(*hop_ptr)--;
return IB_SMI_HANDLE;
}
/* hop_ptr updated when sending */
return (is_switch ? IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/* C14-13:4 -- hop_ptr = 0 -> give to SM */
/* C14-13:5 -- Check for unreasonable hop pointer */
return (*hop_ptr == 0 ? IB_SMI_HANDLE : IB_SMI_DISCARD);
}
}
/*
* Adjust information for a received SMP
* Return IB_SMI_DISCARD if the SMP should be dropped
*/
enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, bool is_switch,
int port_num, int phys_port_cnt)
{
return __smi_handle_dr_smp_recv(is_switch, port_num, phys_port_cnt,
&smp->hop_ptr, smp->hop_cnt,
smp->initial_path,
smp->return_path,
ib_get_smp_direction(smp),
smp->dr_dlid == IB_LID_PERMISSIVE,
smp->dr_slid == IB_LID_PERMISSIVE);
}
/*
* Adjust information for a received SMP
* Return IB_SMI_DISCARD if the SMP should be dropped
*/
enum smi_action opa_smi_handle_dr_smp_recv(struct opa_smp *smp, bool is_switch,
int port_num, int phys_port_cnt)
{
return __smi_handle_dr_smp_recv(is_switch, port_num, phys_port_cnt,
&smp->hop_ptr, smp->hop_cnt,
smp->route.dr.initial_path,
smp->route.dr.return_path,
opa_get_smp_direction(smp),
smp->route.dr.dr_dlid ==
OPA_LID_PERMISSIVE,
smp->route.dr.dr_slid ==
OPA_LID_PERMISSIVE);
}
static enum smi_forward_action __smi_check_forward_dr_smp(u8 hop_ptr, u8 hop_cnt,
u8 direction,
bool dr_dlid_is_permissive,
bool dr_slid_is_permissive)
{
if (!direction) {
/* C14-9:2 -- intermediate hop */
if (hop_ptr && hop_ptr < hop_cnt)
return IB_SMI_FORWARD;
/* C14-9:3 -- at the end of the DR segment of path */
if (hop_ptr == hop_cnt)
return (dr_dlid_is_permissive ?
IB_SMI_SEND : IB_SMI_LOCAL);
/* C14-9:4 -- hop_ptr = hop_cnt + 1 -> give to SMA/SM */
if (hop_ptr == hop_cnt + 1)
return IB_SMI_SEND;
} else {
/* C14-13:2 -- intermediate hop */
if (2 <= hop_ptr && hop_ptr <= hop_cnt)
return IB_SMI_FORWARD;
/* C14-13:3 -- at the end of the DR segment of path */
if (hop_ptr == 1)
return (!dr_slid_is_permissive ?
IB_SMI_SEND : IB_SMI_LOCAL);
}
return IB_SMI_LOCAL;
}
enum smi_forward_action smi_check_forward_dr_smp(struct ib_smp *smp)
{
return __smi_check_forward_dr_smp(smp->hop_ptr, smp->hop_cnt,
ib_get_smp_direction(smp),
smp->dr_dlid == IB_LID_PERMISSIVE,
smp->dr_slid == IB_LID_PERMISSIVE);
}
enum smi_forward_action opa_smi_check_forward_dr_smp(struct opa_smp *smp)
{
return __smi_check_forward_dr_smp(smp->hop_ptr, smp->hop_cnt,
opa_get_smp_direction(smp),
smp->route.dr.dr_dlid ==
OPA_LID_PERMISSIVE,
smp->route.dr.dr_slid ==
OPA_LID_PERMISSIVE);
}
/*
* Return the forwarding port number from initial_path for outgoing SMP and
* from return_path for returning SMP
*/
int smi_get_fwd_port(struct ib_smp *smp)
{
return (!ib_get_smp_direction(smp) ? smp->initial_path[smp->hop_ptr+1] :
smp->return_path[smp->hop_ptr-1]);
}
/*
* Return the forwarding port number from initial_path for outgoing SMP and
* from return_path for returning SMP
*/
int opa_smi_get_fwd_port(struct opa_smp *smp)
{
return !opa_get_smp_direction(smp) ? smp->route.dr.initial_path[smp->hop_ptr+1] :
smp->route.dr.return_path[smp->hop_ptr-1];
}
+90
View File
@@ -0,0 +1,90 @@
/*
* Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved.
* Copyright (c) 2004 Infinicon Corporation. All rights reserved.
* Copyright (c) 2004 Intel Corporation. All rights reserved.
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
* Copyright (c) 2004-2007 Voltaire Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
#ifndef __SMI_H_
#define __SMI_H_
#include <rdma/ib_smi.h>
enum smi_action {
IB_SMI_DISCARD,
IB_SMI_HANDLE
};
enum smi_forward_action {
IB_SMI_LOCAL, /* SMP should be completed up the stack */
IB_SMI_SEND, /* received DR SMP should be forwarded to the send queue */
IB_SMI_FORWARD /* SMP should be forwarded (for switches only) */
};
enum smi_action smi_handle_dr_smp_recv(struct ib_smp *smp, bool is_switch,
int port_num, int phys_port_cnt);
int smi_get_fwd_port(struct ib_smp *smp);
extern enum smi_forward_action smi_check_forward_dr_smp(struct ib_smp *smp);
extern enum smi_action smi_handle_dr_smp_send(struct ib_smp *smp,
bool is_switch, int port_num);
/*
* Return IB_SMI_HANDLE if the SMP should be handled by the local SMA/SM
* via process_mad
*/
static inline enum smi_action smi_check_local_smp(struct ib_smp *smp,
struct ib_device *device)
{
/* C14-9:3 -- We're at the end of the DR segment of path */
/* C14-9:4 -- Hop Pointer = Hop Count + 1 -> give to SMA/SM */
return ((device->process_mad &&
!ib_get_smp_direction(smp) &&
(smp->hop_ptr == smp->hop_cnt + 1)) ?
IB_SMI_HANDLE : IB_SMI_DISCARD);
}
/*
* Return IB_SMI_HANDLE if the SMP should be handled by the local SMA/SM
* via process_mad
*/
static inline enum smi_action smi_check_local_returning_smp(struct ib_smp *smp,
struct ib_device *device)
{
/* C14-13:3 -- We're at the end of the DR segment of path */
/* C14-13:4 -- Hop Pointer == 0 -> give to SM */
return ((device->process_mad &&
ib_get_smp_direction(smp) &&
!smp->hop_ptr) ? IB_SMI_HANDLE : IB_SMI_DISCARD);
}
#endif /* __SMI_H_ */
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+547
View File
@@ -0,0 +1,547 @@
/*
* Copyright (c) 2004 Topspin Corporation. All rights reserved.
* Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/export.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <rdma/ib_pack.h>
#define STRUCT_FIELD(header, field) \
.struct_offset_bytes = offsetof(struct ib_unpacked_ ## header, field), \
.struct_size_bytes = sizeof ((struct ib_unpacked_ ## header *) 0)->field, \
.field_name = #header ":" #field
static const struct ib_field lrh_table[] = {
{ STRUCT_FIELD(lrh, virtual_lane),
.offset_words = 0,
.offset_bits = 0,
.size_bits = 4 },
{ STRUCT_FIELD(lrh, link_version),
.offset_words = 0,
.offset_bits = 4,
.size_bits = 4 },
{ STRUCT_FIELD(lrh, service_level),
.offset_words = 0,
.offset_bits = 8,
.size_bits = 4 },
{ RESERVED,
.offset_words = 0,
.offset_bits = 12,
.size_bits = 2 },
{ STRUCT_FIELD(lrh, link_next_header),
.offset_words = 0,
.offset_bits = 14,
.size_bits = 2 },
{ STRUCT_FIELD(lrh, destination_lid),
.offset_words = 0,
.offset_bits = 16,
.size_bits = 16 },
{ RESERVED,
.offset_words = 1,
.offset_bits = 0,
.size_bits = 5 },
{ STRUCT_FIELD(lrh, packet_length),
.offset_words = 1,
.offset_bits = 5,
.size_bits = 11 },
{ STRUCT_FIELD(lrh, source_lid),
.offset_words = 1,
.offset_bits = 16,
.size_bits = 16 }
};
static const struct ib_field eth_table[] = {
{ STRUCT_FIELD(eth, dmac_h),
.offset_words = 0,
.offset_bits = 0,
.size_bits = 32 },
{ STRUCT_FIELD(eth, dmac_l),
.offset_words = 1,
.offset_bits = 0,
.size_bits = 16 },
{ STRUCT_FIELD(eth, smac_h),
.offset_words = 1,
.offset_bits = 16,
.size_bits = 16 },
{ STRUCT_FIELD(eth, smac_l),
.offset_words = 2,
.offset_bits = 0,
.size_bits = 32 },
{ STRUCT_FIELD(eth, type),
.offset_words = 3,
.offset_bits = 0,
.size_bits = 16 }
};
static const struct ib_field vlan_table[] = {
{ STRUCT_FIELD(vlan, tag),
.offset_words = 0,
.offset_bits = 0,
.size_bits = 16 },
{ STRUCT_FIELD(vlan, type),
.offset_words = 0,
.offset_bits = 16,
.size_bits = 16 }
};
static const struct ib_field ip4_table[] = {
{ STRUCT_FIELD(ip4, ver),
.offset_words = 0,
.offset_bits = 0,
.size_bits = 4 },
{ STRUCT_FIELD(ip4, hdr_len),
.offset_words = 0,
.offset_bits = 4,
.size_bits = 4 },
{ STRUCT_FIELD(ip4, tos),
.offset_words = 0,
.offset_bits = 8,
.size_bits = 8 },
{ STRUCT_FIELD(ip4, tot_len),
.offset_words = 0,
.offset_bits = 16,
.size_bits = 16 },
{ STRUCT_FIELD(ip4, id),
.offset_words = 1,
.offset_bits = 0,
.size_bits = 16 },
{ STRUCT_FIELD(ip4, frag_off),
.offset_words = 1,
.offset_bits = 16,
.size_bits = 16 },
{ STRUCT_FIELD(ip4, ttl),
.offset_words = 2,
.offset_bits = 0,
.size_bits = 8 },
{ STRUCT_FIELD(ip4, protocol),
.offset_words = 2,
.offset_bits = 8,
.size_bits = 8 },
{ STRUCT_FIELD(ip4, check),
.offset_words = 2,
.offset_bits = 16,
.size_bits = 16 },
{ STRUCT_FIELD(ip4, saddr),
.offset_words = 3,
.offset_bits = 0,
.size_bits = 32 },
{ STRUCT_FIELD(ip4, daddr),
.offset_words = 4,
.offset_bits = 0,
.size_bits = 32 }
};
static const struct ib_field udp_table[] = {
{ STRUCT_FIELD(udp, sport),
.offset_words = 0,
.offset_bits = 0,
.size_bits = 16 },
{ STRUCT_FIELD(udp, dport),
.offset_words = 0,
.offset_bits = 16,
.size_bits = 16 },
{ STRUCT_FIELD(udp, length),
.offset_words = 1,
.offset_bits = 0,
.size_bits = 16 },
{ STRUCT_FIELD(udp, csum),
.offset_words = 1,
.offset_bits = 16,
.size_bits = 16 }
};
static const struct ib_field grh_table[] = {
{ STRUCT_FIELD(grh, ip_version),
.offset_words = 0,
.offset_bits = 0,
.size_bits = 4 },
{ STRUCT_FIELD(grh, traffic_class),
.offset_words = 0,
.offset_bits = 4,
.size_bits = 8 },
{ STRUCT_FIELD(grh, flow_label),
.offset_words = 0,
.offset_bits = 12,
.size_bits = 20 },
{ STRUCT_FIELD(grh, payload_length),
.offset_words = 1,
.offset_bits = 0,
.size_bits = 16 },
{ STRUCT_FIELD(grh, next_header),
.offset_words = 1,
.offset_bits = 16,
.size_bits = 8 },
{ STRUCT_FIELD(grh, hop_limit),
.offset_words = 1,
.offset_bits = 24,
.size_bits = 8 },
{ STRUCT_FIELD(grh, source_gid),
.offset_words = 2,
.offset_bits = 0,
.size_bits = 128 },
{ STRUCT_FIELD(grh, destination_gid),
.offset_words = 6,
.offset_bits = 0,
.size_bits = 128 }
};
static const struct ib_field bth_table[] = {
{ STRUCT_FIELD(bth, opcode),
.offset_words = 0,
.offset_bits = 0,
.size_bits = 8 },
{ STRUCT_FIELD(bth, solicited_event),
.offset_words = 0,
.offset_bits = 8,
.size_bits = 1 },
{ STRUCT_FIELD(bth, mig_req),
.offset_words = 0,
.offset_bits = 9,
.size_bits = 1 },
{ STRUCT_FIELD(bth, pad_count),
.offset_words = 0,
.offset_bits = 10,
.size_bits = 2 },
{ STRUCT_FIELD(bth, transport_header_version),
.offset_words = 0,
.offset_bits = 12,
.size_bits = 4 },
{ STRUCT_FIELD(bth, pkey),
.offset_words = 0,
.offset_bits = 16,
.size_bits = 16 },
{ RESERVED,
.offset_words = 1,
.offset_bits = 0,
.size_bits = 8 },
{ STRUCT_FIELD(bth, destination_qpn),
.offset_words = 1,
.offset_bits = 8,
.size_bits = 24 },
{ STRUCT_FIELD(bth, ack_req),
.offset_words = 2,
.offset_bits = 0,
.size_bits = 1 },
{ RESERVED,
.offset_words = 2,
.offset_bits = 1,
.size_bits = 7 },
{ STRUCT_FIELD(bth, psn),
.offset_words = 2,
.offset_bits = 8,
.size_bits = 24 }
};
static const struct ib_field deth_table[] = {
{ STRUCT_FIELD(deth, qkey),
.offset_words = 0,
.offset_bits = 0,
.size_bits = 32 },
{ RESERVED,
.offset_words = 1,
.offset_bits = 0,
.size_bits = 8 },
{ STRUCT_FIELD(deth, source_qpn),
.offset_words = 1,
.offset_bits = 8,
.size_bits = 24 }
};
__sum16 ib_ud_ip4_csum(struct ib_ud_header *header)
{
struct iphdr iph;
iph.ihl = 5;
iph.version = 4;
iph.tos = header->ip4.tos;
iph.tot_len = header->ip4.tot_len;
iph.id = header->ip4.id;
iph.frag_off = header->ip4.frag_off;
iph.ttl = header->ip4.ttl;
iph.protocol = header->ip4.protocol;
iph.check = 0;
iph.saddr = header->ip4.saddr;
iph.daddr = header->ip4.daddr;
return ip_fast_csum((u8 *)&iph, iph.ihl);
}
EXPORT_SYMBOL(ib_ud_ip4_csum);
/**
* ib_ud_header_init - Initialize UD header structure
* @payload_bytes:Length of packet payload
* @lrh_present: specify if LRH is present
* @eth_present: specify if Eth header is present
* @vlan_present: packet is tagged vlan
* @grh_present: GRH flag (if non-zero, GRH will be included)
* @ip_version: if non-zero, IP header, V4 or V6, will be included
* @udp_present :if non-zero, UDP header will be included
* @immediate_present: specify if immediate data is present
* @header:Structure to initialize
*/
int ib_ud_header_init(int payload_bytes,
int lrh_present,
int eth_present,
int vlan_present,
int grh_present,
int ip_version,
int udp_present,
int immediate_present,
struct ib_ud_header *header)
{
size_t udp_bytes = udp_present ? IB_UDP_BYTES : 0;
grh_present = grh_present && !ip_version;
memset(header, 0, sizeof *header);
/*
* UDP header without IP header doesn't make sense
*/
if (udp_present && ip_version != 4 && ip_version != 6)
return -EINVAL;
if (lrh_present) {
u16 packet_length;
header->lrh.link_version = 0;
header->lrh.link_next_header =
grh_present ? IB_LNH_IBA_GLOBAL : IB_LNH_IBA_LOCAL;
packet_length = (IB_LRH_BYTES +
IB_BTH_BYTES +
IB_DETH_BYTES +
(grh_present ? IB_GRH_BYTES : 0) +
payload_bytes +
4 + /* ICRC */
3) / 4; /* round up */
header->lrh.packet_length = cpu_to_be16(packet_length);
}
if (vlan_present)
header->eth.type = cpu_to_be16(ETH_P_8021Q);
if (ip_version == 6 || grh_present) {
header->grh.ip_version = 6;
header->grh.payload_length =
cpu_to_be16((udp_bytes +
IB_BTH_BYTES +
IB_DETH_BYTES +
payload_bytes +
4 + /* ICRC */
3) & ~3); /* round up */
header->grh.next_header = udp_present ? IPPROTO_UDP : 0x1b;
}
if (ip_version == 4) {
header->ip4.ver = 4; /* version 4 */
header->ip4.hdr_len = 5; /* 5 words */
header->ip4.tot_len =
cpu_to_be16(IB_IP4_BYTES +
udp_bytes +
IB_BTH_BYTES +
IB_DETH_BYTES +
payload_bytes +
4); /* ICRC */
header->ip4.protocol = IPPROTO_UDP;
}
if (udp_present && ip_version)
header->udp.length =
cpu_to_be16(IB_UDP_BYTES +
IB_BTH_BYTES +
IB_DETH_BYTES +
payload_bytes +
4); /* ICRC */
if (immediate_present)
header->bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
else
header->bth.opcode = IB_OPCODE_UD_SEND_ONLY;
header->bth.pad_count = (4 - payload_bytes) & 3;
header->bth.transport_header_version = 0;
header->lrh_present = lrh_present;
header->eth_present = eth_present;
header->vlan_present = vlan_present;
header->grh_present = grh_present || (ip_version == 6);
header->ipv4_present = ip_version == 4;
header->udp_present = udp_present;
header->immediate_present = immediate_present;
return 0;
}
EXPORT_SYMBOL(ib_ud_header_init);
/**
* ib_ud_header_pack - Pack UD header struct into wire format
* @header:UD header struct
* @buf:Buffer to pack into
*
* ib_ud_header_pack() packs the UD header structure @header into wire
* format in the buffer @buf.
*/
int ib_ud_header_pack(struct ib_ud_header *header,
void *buf)
{
int len = 0;
if (header->lrh_present) {
ib_pack(lrh_table, ARRAY_SIZE(lrh_table),
&header->lrh, buf + len);
len += IB_LRH_BYTES;
}
if (header->eth_present) {
ib_pack(eth_table, ARRAY_SIZE(eth_table),
&header->eth, buf + len);
len += IB_ETH_BYTES;
}
if (header->vlan_present) {
ib_pack(vlan_table, ARRAY_SIZE(vlan_table),
&header->vlan, buf + len);
len += IB_VLAN_BYTES;
}
if (header->grh_present) {
ib_pack(grh_table, ARRAY_SIZE(grh_table),
&header->grh, buf + len);
len += IB_GRH_BYTES;
}
if (header->ipv4_present) {
ib_pack(ip4_table, ARRAY_SIZE(ip4_table),
&header->ip4, buf + len);
len += IB_IP4_BYTES;
}
if (header->udp_present) {
ib_pack(udp_table, ARRAY_SIZE(udp_table),
&header->udp, buf + len);
len += IB_UDP_BYTES;
}
ib_pack(bth_table, ARRAY_SIZE(bth_table),
&header->bth, buf + len);
len += IB_BTH_BYTES;
ib_pack(deth_table, ARRAY_SIZE(deth_table),
&header->deth, buf + len);
len += IB_DETH_BYTES;
if (header->immediate_present) {
memcpy(buf + len, &header->immediate_data, sizeof header->immediate_data);
len += sizeof header->immediate_data;
}
return len;
}
EXPORT_SYMBOL(ib_ud_header_pack);
/**
* ib_ud_header_unpack - Unpack UD header struct from wire format
* @header:UD header struct
* @buf:Buffer to pack into
*
* ib_ud_header_pack() unpacks the UD header structure @header from wire
* format in the buffer @buf.
*/
int ib_ud_header_unpack(void *buf,
struct ib_ud_header *header)
{
ib_unpack(lrh_table, ARRAY_SIZE(lrh_table),
buf, &header->lrh);
buf += IB_LRH_BYTES;
if (header->lrh.link_version != 0) {
pr_warn("Invalid LRH.link_version %d\n",
header->lrh.link_version);
return -EINVAL;
}
switch (header->lrh.link_next_header) {
case IB_LNH_IBA_LOCAL:
header->grh_present = 0;
break;
case IB_LNH_IBA_GLOBAL:
header->grh_present = 1;
ib_unpack(grh_table, ARRAY_SIZE(grh_table),
buf, &header->grh);
buf += IB_GRH_BYTES;
if (header->grh.ip_version != 6) {
pr_warn("Invalid GRH.ip_version %d\n",
header->grh.ip_version);
return -EINVAL;
}
if (header->grh.next_header != 0x1b) {
pr_warn("Invalid GRH.next_header 0x%02x\n",
header->grh.next_header);
return -EINVAL;
}
break;
default:
pr_warn("Invalid LRH.link_next_header %d\n",
header->lrh.link_next_header);
return -EINVAL;
}
ib_unpack(bth_table, ARRAY_SIZE(bth_table),
buf, &header->bth);
buf += IB_BTH_BYTES;
switch (header->bth.opcode) {
case IB_OPCODE_UD_SEND_ONLY:
header->immediate_present = 0;
break;
case IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE:
header->immediate_present = 1;
break;
default:
pr_warn("Invalid BTH.opcode 0x%02x\n", header->bth.opcode);
return -EINVAL;
}
if (header->bth.transport_header_version != 0) {
pr_warn("Invalid BTH.transport_header_version %d\n",
header->bth.transport_header_version);
return -EINVAL;
}
ib_unpack(deth_table, ARRAY_SIZE(deth_table),
buf, &header->deth);
buf += IB_DETH_BYTES;
if (header->immediate_present)
memcpy(&header->immediate_data, buf, sizeof header->immediate_data);
return 0;
}
EXPORT_SYMBOL(ib_ud_header_unpack);
+350
View File
@@ -0,0 +1,350 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2005 Cisco Systems. All rights reserved.
* Copyright (c) 2005 Mellanox Technologies. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/mm.h>
#include <linux/dma-mapping.h>
#include <linux/sched/signal.h>
#include <linux/sched/mm.h>
#include <linux/export.h>
#include <linux/hugetlb.h>
#include <linux/slab.h>
#include <rdma/ib_umem_odp.h>
#include "uverbs.h"
static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty)
{
struct scatterlist *sg;
struct page *page;
int i;
if (umem->nmap > 0)
ib_dma_unmap_sg(dev, umem->sg_head.sgl,
umem->npages,
DMA_BIDIRECTIONAL);
for_each_sg(umem->sg_head.sgl, sg, umem->npages, i) {
page = sg_page(sg);
if (!PageDirty(page) && umem->writable && dirty)
set_page_dirty_lock(page);
put_page(page);
}
sg_free_table(&umem->sg_head);
}
/**
* ib_umem_get - Pin and DMA map userspace memory.
*
* If access flags indicate ODP memory, avoid pinning. Instead, stores
* the mm for future page fault handling in conjunction with MMU notifiers.
*
* @context: userspace context to pin memory for
* @addr: userspace virtual address to start at
* @size: length of region to pin
* @access: IB_ACCESS_xxx flags for memory being pinned
* @dmasync: flush in-flight DMA when the memory region is written
*/
struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
size_t size, int access, int dmasync)
{
struct ib_umem *umem;
struct page **page_list;
struct vm_area_struct **vma_list;
unsigned long lock_limit;
unsigned long cur_base;
unsigned long npages;
int ret;
int i;
unsigned long dma_attrs = 0;
struct scatterlist *sg, *sg_list_start;
unsigned int gup_flags = FOLL_WRITE;
if (dmasync)
dma_attrs |= DMA_ATTR_WRITE_BARRIER;
/*
* If the combination of the addr and size requested for this memory
* region causes an integer overflow, return error.
*/
if (((addr + size) < addr) ||
PAGE_ALIGN(addr + size) < (addr + size))
return ERR_PTR(-EINVAL);
if (!can_do_mlock())
return ERR_PTR(-EPERM);
umem = kzalloc(sizeof *umem, GFP_KERNEL);
if (!umem)
return ERR_PTR(-ENOMEM);
umem->context = context;
umem->length = size;
umem->address = addr;
umem->page_shift = PAGE_SHIFT;
umem->writable = ib_access_writable(access);
if (access & IB_ACCESS_ON_DEMAND) {
ret = ib_umem_odp_get(context, umem, access);
if (ret)
goto umem_kfree;
return umem;
}
umem->odp_data = NULL;
/* We assume the memory is from hugetlb until proved otherwise */
umem->hugetlb = 1;
page_list = (struct page **) __get_free_page(GFP_KERNEL);
if (!page_list) {
ret = -ENOMEM;
goto umem_kfree;
}
/*
* if we can't alloc the vma_list, it's not so bad;
* just assume the memory is not hugetlb memory
*/
vma_list = (struct vm_area_struct **) __get_free_page(GFP_KERNEL);
if (!vma_list)
umem->hugetlb = 0;
npages = ib_umem_num_pages(umem);
lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
down_write(&current->mm->mmap_sem);
current->mm->pinned_vm += npages;
if ((current->mm->pinned_vm > lock_limit) && !capable(CAP_IPC_LOCK)) {
up_write(&current->mm->mmap_sem);
ret = -ENOMEM;
goto vma;
}
up_write(&current->mm->mmap_sem);
cur_base = addr & PAGE_MASK;
if (npages == 0 || npages > UINT_MAX) {
ret = -EINVAL;
goto vma;
}
ret = sg_alloc_table(&umem->sg_head, npages, GFP_KERNEL);
if (ret)
goto vma;
if (!umem->writable)
gup_flags |= FOLL_FORCE;
sg_list_start = umem->sg_head.sgl;
down_read(&current->mm->mmap_sem);
while (npages) {
ret = get_user_pages(cur_base,
min_t(unsigned long, npages,
PAGE_SIZE / sizeof (struct page *)),
gup_flags | FOLL_LONGTERM,
page_list, vma_list);
if (ret < 0) {
up_read(&current->mm->mmap_sem);
goto umem_release;
}
umem->npages += ret;
cur_base += ret * PAGE_SIZE;
npages -= ret;
for_each_sg(sg_list_start, sg, ret, i) {
if (vma_list && !is_vm_hugetlb_page(vma_list[i]))
umem->hugetlb = 0;
sg_set_page(sg, page_list[i], PAGE_SIZE, 0);
}
/* preparing for next loop */
sg_list_start = sg;
}
up_read(&current->mm->mmap_sem);
umem->nmap = ib_dma_map_sg_attrs(context->device,
umem->sg_head.sgl,
umem->npages,
DMA_BIDIRECTIONAL,
dma_attrs);
if (!umem->nmap) {
ret = -ENOMEM;
goto umem_release;
}
ret = 0;
goto out;
umem_release:
__ib_umem_release(context->device, umem, 0);
vma:
down_write(&current->mm->mmap_sem);
current->mm->pinned_vm -= ib_umem_num_pages(umem);
up_write(&current->mm->mmap_sem);
out:
if (vma_list)
free_page((unsigned long) vma_list);
free_page((unsigned long) page_list);
umem_kfree:
if (ret)
kfree(umem);
return ret ? ERR_PTR(ret) : umem;
}
EXPORT_SYMBOL(ib_umem_get);
static void ib_umem_account(struct work_struct *work)
{
struct ib_umem *umem = container_of(work, struct ib_umem, work);
down_write(&umem->mm->mmap_sem);
umem->mm->pinned_vm -= umem->diff;
up_write(&umem->mm->mmap_sem);
mmput(umem->mm);
kfree(umem);
}
/**
* ib_umem_release - release memory pinned with ib_umem_get
* @umem: umem struct to release
*/
void ib_umem_release(struct ib_umem *umem)
{
struct ib_ucontext *context = umem->context;
struct mm_struct *mm;
struct task_struct *task;
unsigned long diff;
if (umem->odp_data) {
ib_umem_odp_release(umem);
return;
}
__ib_umem_release(umem->context->device, umem, 1);
task = get_pid_task(umem->context->tgid, PIDTYPE_PID);
if (!task)
goto out;
mm = get_task_mm(task);
put_task_struct(task);
if (!mm)
goto out;
diff = ib_umem_num_pages(umem);
/*
* We may be called with the mm's mmap_sem already held. This
* can happen when a userspace munmap() is the call that drops
* the last reference to our file and calls our release
* method. If there are memory regions to destroy, we'll end
* up here and not be able to take the mmap_sem. In that case
* we defer the vm_locked accounting to the system workqueue.
*/
if (context->closing) {
if (!down_write_trylock(&mm->mmap_sem)) {
INIT_WORK(&umem->work, ib_umem_account);
umem->mm = mm;
umem->diff = diff;
queue_work(ib_wq, &umem->work);
return;
}
} else
down_write(&mm->mmap_sem);
mm->pinned_vm -= diff;
up_write(&mm->mmap_sem);
mmput(mm);
out:
kfree(umem);
}
EXPORT_SYMBOL(ib_umem_release);
int ib_umem_page_count(struct ib_umem *umem)
{
int i;
int n;
struct scatterlist *sg;
if (umem->odp_data)
return ib_umem_num_pages(umem);
n = 0;
for_each_sg(umem->sg_head.sgl, sg, umem->nmap, i)
n += sg_dma_len(sg) >> umem->page_shift;
return n;
}
EXPORT_SYMBOL(ib_umem_page_count);
/*
* Copy from the given ib_umem's pages to the given buffer.
*
* umem - the umem to copy from
* offset - offset to start copying from
* dst - destination buffer
* length - buffer length
*
* Returns 0 on success, or an error code.
*/
int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
size_t length)
{
size_t end = offset + length;
int ret;
if (offset > umem->length || length > umem->length - offset) {
pr_err("ib_umem_copy_from not in range. offset: %zd umem length: %zd end: %zd\n",
offset, umem->length, end);
return -EINVAL;
}
ret = sg_pcopy_to_buffer(umem->sg_head.sgl, umem->npages, dst, length,
offset + ib_umem_offset(umem));
if (ret < 0)
return ret;
else if (ret != length)
return -EINVAL;
else
return 0;
}
EXPORT_SYMBOL(ib_umem_copy_from);
+852
View File
@@ -0,0 +1,852 @@
/*
* Copyright (c) 2014 Mellanox Technologies. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/types.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/sched/task.h>
#include <linux/pid.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/vmalloc.h>
#include <linux/hugetlb.h>
#include <linux/interval_tree_generic.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_umem_odp.h>
/*
* The ib_umem list keeps track of memory regions for which the HW
* device request to receive notification when the related memory
* mapping is changed.
*
* ib_umem_lock protects the list.
*/
static u64 node_start(struct umem_odp_node *n)
{
struct ib_umem_odp *umem_odp =
container_of(n, struct ib_umem_odp, interval_tree);
return ib_umem_start(umem_odp->umem);
}
/* Note that the representation of the intervals in the interval tree
* considers the ending point as contained in the interval, while the
* function ib_umem_end returns the first address which is not contained
* in the umem.
*/
static u64 node_last(struct umem_odp_node *n)
{
struct ib_umem_odp *umem_odp =
container_of(n, struct ib_umem_odp, interval_tree);
return ib_umem_end(umem_odp->umem) - 1;
}
INTERVAL_TREE_DEFINE(struct umem_odp_node, rb, u64, __subtree_last,
node_start, node_last, static, rbt_ib_umem)
static void ib_umem_notifier_start_account(struct ib_umem *item)
{
mutex_lock(&item->odp_data->umem_mutex);
/* Only update private counters for this umem if it has them.
* Otherwise skip it. All page faults will be delayed for this umem. */
if (item->odp_data->mn_counters_active) {
int notifiers_count = item->odp_data->notifiers_count++;
if (notifiers_count == 0)
/* Initialize the completion object for waiting on
* notifiers. Since notifier_count is zero, no one
* should be waiting right now. */
reinit_completion(&item->odp_data->notifier_completion);
}
mutex_unlock(&item->odp_data->umem_mutex);
}
static void ib_umem_notifier_end_account(struct ib_umem *item)
{
mutex_lock(&item->odp_data->umem_mutex);
/* Only update private counters for this umem if it has them.
* Otherwise skip it. All page faults will be delayed for this umem. */
if (item->odp_data->mn_counters_active) {
/*
* This sequence increase will notify the QP page fault that
* the page that is going to be mapped in the spte could have
* been freed.
*/
++item->odp_data->notifiers_seq;
if (--item->odp_data->notifiers_count == 0)
complete_all(&item->odp_data->notifier_completion);
}
mutex_unlock(&item->odp_data->umem_mutex);
}
/* Account for a new mmu notifier in an ib_ucontext. */
static void ib_ucontext_notifier_start_account(struct ib_ucontext *context)
{
atomic_inc(&context->notifier_count);
}
/* Account for a terminating mmu notifier in an ib_ucontext.
*
* Must be called with the ib_ucontext->umem_rwsem semaphore unlocked, since
* the function takes the semaphore itself. */
static void ib_ucontext_notifier_end_account(struct ib_ucontext *context)
{
int zero_notifiers = atomic_dec_and_test(&context->notifier_count);
if (zero_notifiers &&
!list_empty(&context->no_private_counters)) {
/* No currently running mmu notifiers. Now is the chance to
* add private accounting to all previously added umems. */
struct ib_umem_odp *odp_data, *next;
/* Prevent concurrent mmu notifiers from working on the
* no_private_counters list. */
down_write(&context->umem_rwsem);
/* Read the notifier_count again, with the umem_rwsem
* semaphore taken for write. */
if (!atomic_read(&context->notifier_count)) {
list_for_each_entry_safe(odp_data, next,
&context->no_private_counters,
no_private_counters) {
mutex_lock(&odp_data->umem_mutex);
odp_data->mn_counters_active = true;
list_del(&odp_data->no_private_counters);
complete_all(&odp_data->notifier_completion);
mutex_unlock(&odp_data->umem_mutex);
}
}
up_write(&context->umem_rwsem);
}
}
static int ib_umem_notifier_release_trampoline(struct ib_umem *item, u64 start,
u64 end, void *cookie) {
/*
* Increase the number of notifiers running, to
* prevent any further fault handling on this MR.
*/
ib_umem_notifier_start_account(item);
item->odp_data->dying = 1;
/* Make sure that the fact the umem is dying is out before we release
* all pending page faults. */
smp_wmb();
complete_all(&item->odp_data->notifier_completion);
item->context->invalidate_range(item, ib_umem_start(item),
ib_umem_end(item));
return 0;
}
static void ib_umem_notifier_release(struct mmu_notifier *mn,
struct mm_struct *mm)
{
struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
if (!context->invalidate_range)
return;
ib_ucontext_notifier_start_account(context);
down_read(&context->umem_rwsem);
rbt_ib_umem_for_each_in_range(&context->umem_tree, 0,
ULLONG_MAX,
ib_umem_notifier_release_trampoline,
true,
NULL);
up_read(&context->umem_rwsem);
}
static int invalidate_page_trampoline(struct ib_umem *item, u64 start,
u64 end, void *cookie)
{
ib_umem_notifier_start_account(item);
item->context->invalidate_range(item, start, start + PAGE_SIZE);
ib_umem_notifier_end_account(item);
return 0;
}
static int invalidate_range_start_trampoline(struct ib_umem *item, u64 start,
u64 end, void *cookie)
{
ib_umem_notifier_start_account(item);
item->context->invalidate_range(item, start, end);
return 0;
}
static int ib_umem_notifier_invalidate_range_start(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start,
unsigned long end,
bool blockable)
{
struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
int ret;
if (!context->invalidate_range)
return 0;
if (blockable)
down_read(&context->umem_rwsem);
else if (!down_read_trylock(&context->umem_rwsem))
return -EAGAIN;
ib_ucontext_notifier_start_account(context);
ret = rbt_ib_umem_for_each_in_range(&context->umem_tree, start,
end,
invalidate_range_start_trampoline,
blockable, NULL);
up_read(&context->umem_rwsem);
return ret;
}
static int invalidate_range_end_trampoline(struct ib_umem *item, u64 start,
u64 end, void *cookie)
{
ib_umem_notifier_end_account(item);
return 0;
}
static void ib_umem_notifier_invalidate_range_end(struct mmu_notifier *mn,
struct mm_struct *mm,
unsigned long start,
unsigned long end)
{
struct ib_ucontext *context = container_of(mn, struct ib_ucontext, mn);
if (!context->invalidate_range)
return;
/*
* TODO: we currently bail out if there is any sleepable work to be done
* in ib_umem_notifier_invalidate_range_start so we shouldn't really block
* here. But this is ugly and fragile.
*/
down_read(&context->umem_rwsem);
rbt_ib_umem_for_each_in_range(&context->umem_tree, start,
end,
invalidate_range_end_trampoline, true, NULL);
up_read(&context->umem_rwsem);
ib_ucontext_notifier_end_account(context);
}
static const struct mmu_notifier_ops ib_umem_notifiers = {
.release = ib_umem_notifier_release,
.invalidate_range_start = ib_umem_notifier_invalidate_range_start,
.invalidate_range_end = ib_umem_notifier_invalidate_range_end,
};
struct ib_umem *ib_alloc_odp_umem(struct ib_ucontext *context,
unsigned long addr,
size_t size)
{
struct ib_umem *umem;
struct ib_umem_odp *odp_data;
int pages = size >> PAGE_SHIFT;
int ret;
umem = kzalloc(sizeof(*umem), GFP_KERNEL);
if (!umem)
return ERR_PTR(-ENOMEM);
umem->context = context;
umem->length = size;
umem->address = addr;
umem->page_shift = PAGE_SHIFT;
umem->writable = 1;
odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL);
if (!odp_data) {
ret = -ENOMEM;
goto out_umem;
}
odp_data->umem = umem;
mutex_init(&odp_data->umem_mutex);
init_completion(&odp_data->notifier_completion);
odp_data->page_list =
vzalloc(array_size(pages, sizeof(*odp_data->page_list)));
if (!odp_data->page_list) {
ret = -ENOMEM;
goto out_odp_data;
}
odp_data->dma_list =
vzalloc(array_size(pages, sizeof(*odp_data->dma_list)));
if (!odp_data->dma_list) {
ret = -ENOMEM;
goto out_page_list;
}
down_write(&context->umem_rwsem);
context->odp_mrs_count++;
rbt_ib_umem_insert(&odp_data->interval_tree, &context->umem_tree);
if (likely(!atomic_read(&context->notifier_count)))
odp_data->mn_counters_active = true;
else
list_add(&odp_data->no_private_counters,
&context->no_private_counters);
up_write(&context->umem_rwsem);
umem->odp_data = odp_data;
return umem;
out_page_list:
vfree(odp_data->page_list);
out_odp_data:
kfree(odp_data);
out_umem:
kfree(umem);
return ERR_PTR(ret);
}
EXPORT_SYMBOL(ib_alloc_odp_umem);
int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
int access)
{
int ret_val;
struct pid *our_pid;
struct mm_struct *mm = get_task_mm(current);
if (!mm)
return -EINVAL;
if (access & IB_ACCESS_HUGETLB) {
struct vm_area_struct *vma;
struct hstate *h;
down_read(&mm->mmap_sem);
vma = find_vma(mm, ib_umem_start(umem));
if (!vma || !is_vm_hugetlb_page(vma)) {
up_read(&mm->mmap_sem);
ret_val = -EINVAL;
goto out_mm;
}
h = hstate_vma(vma);
umem->page_shift = huge_page_shift(h);
up_read(&mm->mmap_sem);
umem->hugetlb = 1;
} else {
umem->hugetlb = 0;
}
/* Prevent creating ODP MRs in child processes */
rcu_read_lock();
our_pid = get_task_pid(current->group_leader, PIDTYPE_PID);
rcu_read_unlock();
put_pid(our_pid);
if (context->tgid != our_pid) {
ret_val = -EINVAL;
goto out_mm;
}
umem->odp_data = kzalloc(sizeof(*umem->odp_data), GFP_KERNEL);
if (!umem->odp_data) {
ret_val = -ENOMEM;
goto out_mm;
}
umem->odp_data->umem = umem;
mutex_init(&umem->odp_data->umem_mutex);
init_completion(&umem->odp_data->notifier_completion);
if (ib_umem_num_pages(umem)) {
umem->odp_data->page_list =
vzalloc(array_size(sizeof(*umem->odp_data->page_list),
ib_umem_num_pages(umem)));
if (!umem->odp_data->page_list) {
ret_val = -ENOMEM;
goto out_odp_data;
}
umem->odp_data->dma_list =
vzalloc(array_size(sizeof(*umem->odp_data->dma_list),
ib_umem_num_pages(umem)));
if (!umem->odp_data->dma_list) {
ret_val = -ENOMEM;
goto out_page_list;
}
}
/*
* When using MMU notifiers, we will get a
* notification before the "current" task (and MM) is
* destroyed. We use the umem_rwsem semaphore to synchronize.
*/
down_write(&context->umem_rwsem);
context->odp_mrs_count++;
if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
rbt_ib_umem_insert(&umem->odp_data->interval_tree,
&context->umem_tree);
if (likely(!atomic_read(&context->notifier_count)) ||
context->odp_mrs_count == 1)
umem->odp_data->mn_counters_active = true;
else
list_add(&umem->odp_data->no_private_counters,
&context->no_private_counters);
downgrade_write(&context->umem_rwsem);
if (context->odp_mrs_count == 1) {
/*
* Note that at this point, no MMU notifier is running
* for this context!
*/
atomic_set(&context->notifier_count, 0);
INIT_HLIST_NODE(&context->mn.hlist);
context->mn.ops = &ib_umem_notifiers;
/*
* Lock-dep detects a false positive for mmap_sem vs.
* umem_rwsem, due to not grasping downgrade_write correctly.
*/
lockdep_off();
ret_val = mmu_notifier_register(&context->mn, mm);
lockdep_on();
if (ret_val) {
pr_err("Failed to register mmu_notifier %d\n", ret_val);
ret_val = -EBUSY;
goto out_mutex;
}
}
up_read(&context->umem_rwsem);
/*
* Note that doing an mmput can cause a notifier for the relevant mm.
* If the notifier is called while we hold the umem_rwsem, this will
* cause a deadlock. Therefore, we release the reference only after we
* released the semaphore.
*/
mmput(mm);
return 0;
out_mutex:
up_read(&context->umem_rwsem);
vfree(umem->odp_data->dma_list);
out_page_list:
vfree(umem->odp_data->page_list);
out_odp_data:
kfree(umem->odp_data);
out_mm:
mmput(mm);
return ret_val;
}
void ib_umem_odp_release(struct ib_umem *umem)
{
struct ib_ucontext *context = umem->context;
/*
* Ensure that no more pages are mapped in the umem.
*
* It is the driver's responsibility to ensure, before calling us,
* that the hardware will not attempt to access the MR any more.
*/
ib_umem_odp_unmap_dma_pages(umem, ib_umem_start(umem),
ib_umem_end(umem));
down_write(&context->umem_rwsem);
if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
rbt_ib_umem_remove(&umem->odp_data->interval_tree,
&context->umem_tree);
context->odp_mrs_count--;
if (!umem->odp_data->mn_counters_active) {
list_del(&umem->odp_data->no_private_counters);
complete_all(&umem->odp_data->notifier_completion);
}
/*
* Downgrade the lock to a read lock. This ensures that the notifiers
* (who lock the mutex for reading) will be able to finish, and we
* will be able to enventually obtain the mmu notifiers SRCU. Note
* that since we are doing it atomically, no other user could register
* and unregister while we do the check.
*/
downgrade_write(&context->umem_rwsem);
if (!context->odp_mrs_count) {
struct task_struct *owning_process = NULL;
struct mm_struct *owning_mm = NULL;
owning_process = get_pid_task(context->tgid,
PIDTYPE_PID);
if (owning_process == NULL)
/*
* The process is already dead, notifier were removed
* already.
*/
goto out;
owning_mm = get_task_mm(owning_process);
if (owning_mm == NULL)
/*
* The process' mm is already dead, notifier were
* removed already.
*/
goto out_put_task;
mmu_notifier_unregister(&context->mn, owning_mm);
mmput(owning_mm);
out_put_task:
put_task_struct(owning_process);
}
out:
up_read(&context->umem_rwsem);
vfree(umem->odp_data->dma_list);
vfree(umem->odp_data->page_list);
kfree(umem->odp_data);
kfree(umem);
}
/*
* Map for DMA and insert a single page into the on-demand paging page tables.
*
* @umem: the umem to insert the page to.
* @page_index: index in the umem to add the page to.
* @page: the page struct to map and add.
* @access_mask: access permissions needed for this page.
* @current_seq: sequence number for synchronization with invalidations.
* the sequence number is taken from
* umem->odp_data->notifiers_seq.
*
* The function returns -EFAULT if the DMA mapping operation fails. It returns
* -EAGAIN if a concurrent invalidation prevents us from updating the page.
*
* The page is released via put_page even if the operation failed. For
* on-demand pinning, the page is released whenever it isn't stored in the
* umem.
*/
static int ib_umem_odp_map_dma_single_page(
struct ib_umem *umem,
int page_index,
struct page *page,
u64 access_mask,
unsigned long current_seq)
{
struct ib_device *dev = umem->context->device;
dma_addr_t dma_addr;
int stored_page = 0;
int remove_existing_mapping = 0;
int ret = 0;
/*
* Note: we avoid writing if seq is different from the initial seq, to
* handle case of a racing notifier. This check also allows us to bail
* early if we have a notifier running in parallel with us.
*/
if (ib_umem_mmu_notifier_retry(umem, current_seq)) {
ret = -EAGAIN;
goto out;
}
if (!(umem->odp_data->dma_list[page_index])) {
dma_addr = ib_dma_map_page(dev,
page,
0, BIT(umem->page_shift),
DMA_BIDIRECTIONAL);
if (ib_dma_mapping_error(dev, dma_addr)) {
ret = -EFAULT;
goto out;
}
umem->odp_data->dma_list[page_index] = dma_addr | access_mask;
umem->odp_data->page_list[page_index] = page;
umem->npages++;
stored_page = 1;
} else if (umem->odp_data->page_list[page_index] == page) {
umem->odp_data->dma_list[page_index] |= access_mask;
} else {
pr_err("error: got different pages in IB device and from get_user_pages. IB device page: %p, gup page: %p\n",
umem->odp_data->page_list[page_index], page);
/* Better remove the mapping now, to prevent any further
* damage. */
remove_existing_mapping = 1;
}
out:
/* On Demand Paging - avoid pinning the page */
if (umem->context->invalidate_range || !stored_page)
put_page(page);
if (remove_existing_mapping && umem->context->invalidate_range) {
invalidate_page_trampoline(
umem,
ib_umem_start(umem) + (page_index >> umem->page_shift),
ib_umem_start(umem) + ((page_index + 1) >>
umem->page_shift),
NULL);
ret = -EAGAIN;
}
return ret;
}
/**
* ib_umem_odp_map_dma_pages - Pin and DMA map userspace memory in an ODP MR.
*
* Pins the range of pages passed in the argument, and maps them to
* DMA addresses. The DMA addresses of the mapped pages is updated in
* umem->odp_data->dma_list.
*
* Returns the number of pages mapped in success, negative error code
* for failure.
* An -EAGAIN error code is returned when a concurrent mmu notifier prevents
* the function from completing its task.
* An -ENOENT error code indicates that userspace process is being terminated
* and mm was already destroyed.
* @umem: the umem to map and pin
* @user_virt: the address from which we need to map.
* @bcnt: the minimal number of bytes to pin and map. The mapping might be
* bigger due to alignment, and may also be smaller in case of an error
* pinning or mapping a page. The actual pages mapped is returned in
* the return value.
* @access_mask: bit mask of the requested access permissions for the given
* range.
* @current_seq: the MMU notifiers sequance value for synchronization with
* invalidations. the sequance number is read from
* umem->odp_data->notifiers_seq before calling this function
*/
int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
u64 access_mask, unsigned long current_seq)
{
struct task_struct *owning_process = NULL;
struct mm_struct *owning_mm = NULL;
struct page **local_page_list = NULL;
u64 page_mask, off;
int j, k, ret = 0, start_idx, npages = 0, page_shift;
unsigned int flags = 0;
phys_addr_t p = 0;
if (access_mask == 0)
return -EINVAL;
if (user_virt < ib_umem_start(umem) ||
user_virt + bcnt > ib_umem_end(umem))
return -EFAULT;
local_page_list = (struct page **)__get_free_page(GFP_KERNEL);
if (!local_page_list)
return -ENOMEM;
page_shift = umem->page_shift;
page_mask = ~(BIT(page_shift) - 1);
off = user_virt & (~page_mask);
user_virt = user_virt & page_mask;
bcnt += off; /* Charge for the first page offset as well. */
owning_process = get_pid_task(umem->context->tgid, PIDTYPE_PID);
if (owning_process == NULL) {
ret = -EINVAL;
goto out_no_task;
}
owning_mm = get_task_mm(owning_process);
if (owning_mm == NULL) {
ret = -ENOENT;
goto out_put_task;
}
if (access_mask & ODP_WRITE_ALLOWED_BIT)
flags |= FOLL_WRITE;
start_idx = (user_virt - ib_umem_start(umem)) >> page_shift;
k = start_idx;
while (bcnt > 0) {
const size_t gup_num_pages = min_t(size_t,
ALIGN(bcnt, PAGE_SIZE) / PAGE_SIZE,
PAGE_SIZE / sizeof(struct page *));
down_read(&owning_mm->mmap_sem);
/*
* Note: this might result in redundent page getting. We can
* avoid this by checking dma_list to be 0 before calling
* get_user_pages. However, this make the code much more
* complex (and doesn't gain us much performance in most use
* cases).
*/
npages = get_user_pages_remote(owning_process, owning_mm,
user_virt, gup_num_pages,
flags, local_page_list, NULL, NULL);
up_read(&owning_mm->mmap_sem);
if (npages < 0)
break;
bcnt -= min_t(size_t, npages << PAGE_SHIFT, bcnt);
mutex_lock(&umem->odp_data->umem_mutex);
for (j = 0; j < npages; j++, user_virt += PAGE_SIZE) {
if (user_virt & ~page_mask) {
p += PAGE_SIZE;
if (page_to_phys(local_page_list[j]) != p) {
ret = -EFAULT;
break;
}
put_page(local_page_list[j]);
continue;
}
ret = ib_umem_odp_map_dma_single_page(
umem, k, local_page_list[j],
access_mask, current_seq);
if (ret < 0)
break;
p = page_to_phys(local_page_list[j]);
k++;
}
mutex_unlock(&umem->odp_data->umem_mutex);
if (ret < 0) {
/* Release left over pages when handling errors. */
for (++j; j < npages; ++j)
put_page(local_page_list[j]);
break;
}
}
if (ret >= 0) {
if (npages < 0 && k == start_idx)
ret = npages;
else
ret = k - start_idx;
}
mmput(owning_mm);
out_put_task:
put_task_struct(owning_process);
out_no_task:
free_page((unsigned long)local_page_list);
return ret;
}
EXPORT_SYMBOL(ib_umem_odp_map_dma_pages);
void ib_umem_odp_unmap_dma_pages(struct ib_umem *umem, u64 virt,
u64 bound)
{
int idx;
u64 addr;
struct ib_device *dev = umem->context->device;
virt = max_t(u64, virt, ib_umem_start(umem));
bound = min_t(u64, bound, ib_umem_end(umem));
/* Note that during the run of this function, the
* notifiers_count of the MR is > 0, preventing any racing
* faults from completion. We might be racing with other
* invalidations, so we must make sure we free each page only
* once. */
mutex_lock(&umem->odp_data->umem_mutex);
for (addr = virt; addr < bound; addr += BIT(umem->page_shift)) {
idx = (addr - ib_umem_start(umem)) >> umem->page_shift;
if (umem->odp_data->page_list[idx]) {
struct page *page = umem->odp_data->page_list[idx];
dma_addr_t dma = umem->odp_data->dma_list[idx];
dma_addr_t dma_addr = dma & ODP_DMA_ADDR_MASK;
WARN_ON(!dma_addr);
ib_dma_unmap_page(dev, dma_addr, PAGE_SIZE,
DMA_BIDIRECTIONAL);
if (dma & ODP_WRITE_ALLOWED_BIT) {
struct page *head_page = compound_head(page);
/*
* set_page_dirty prefers being called with
* the page lock. However, MMU notifiers are
* called sometimes with and sometimes without
* the lock. We rely on the umem_mutex instead
* to prevent other mmu notifiers from
* continuing and allowing the page mapping to
* be removed.
*/
set_page_dirty(head_page);
}
/* on demand pinning support */
if (!umem->context->invalidate_range)
put_page(page);
umem->odp_data->page_list[idx] = NULL;
umem->odp_data->dma_list[idx] = 0;
umem->npages--;
}
}
mutex_unlock(&umem->odp_data->umem_mutex);
}
EXPORT_SYMBOL(ib_umem_odp_unmap_dma_pages);
/* @last is not a part of the interval. See comment for function
* node_last.
*/
int rbt_ib_umem_for_each_in_range(struct rb_root_cached *root,
u64 start, u64 last,
umem_call_back cb,
bool blockable,
void *cookie)
{
int ret_val = 0;
struct umem_odp_node *node, *next;
struct ib_umem_odp *umem;
if (unlikely(start == last))
return ret_val;
for (node = rbt_ib_umem_iter_first(root, start, last - 1);
node; node = next) {
/* TODO move the blockable decision up to the callback */
if (!blockable)
return -EAGAIN;
next = rbt_ib_umem_iter_next(node, start, last - 1);
umem = container_of(node, struct ib_umem_odp, interval_tree);
ret_val = cb(umem->umem, start, last, cookie) || ret_val;
}
return ret_val;
}
EXPORT_SYMBOL(rbt_ib_umem_for_each_in_range);
struct ib_umem_odp *rbt_ib_umem_lookup(struct rb_root_cached *root,
u64 addr, u64 length)
{
struct umem_odp_node *node;
node = rbt_ib_umem_iter_first(root, addr, addr + length - 1);
if (node)
return container_of(node, struct ib_umem_odp, interval_tree);
return NULL;
}
EXPORT_SYMBOL(rbt_ib_umem_lookup);
File diff suppressed because it is too large. Load diff
+362
View File
@@ -0,0 +1,362 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
* Copyright (c) 2005 Mellanox Technologies. All rights reserved.
* Copyright (c) 2005 Voltaire, Inc. All rights reserved.
* Copyright (c) 2005 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef UVERBS_H
#define UVERBS_H
#include <linux/kref.h>
#include <linux/idr.h>
#include <linux/mutex.h>
#include <linux/completion.h>
#include <linux/cdev.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/uverbs_std_types.h>
#define UVERBS_MODULE_NAME ib_uverbs
#include <rdma/uverbs_named_ioctl.h>
static inline void
ib_uverbs_init_udata(struct ib_udata *udata,
const void __user *ibuf,
void __user *obuf,
size_t ilen, size_t olen)
{
udata->inbuf = ibuf;
udata->outbuf = obuf;
udata->inlen = ilen;
udata->outlen = olen;
}
static inline void
ib_uverbs_init_udata_buf_or_null(struct ib_udata *udata,
const void __user *ibuf,
void __user *obuf,
size_t ilen, size_t olen)
{
ib_uverbs_init_udata(udata,
ilen ? ibuf : NULL, olen ? obuf : NULL,
ilen, olen);
}
/*
* Our lifetime rules for these structs are the following:
*
* struct ib_uverbs_device: One reference is held by the module and
* released in ib_uverbs_remove_one(). Another reference is taken by
* ib_uverbs_open() each time the character special file is opened,
* and released in ib_uverbs_release_file() when the file is released.
*
* struct ib_uverbs_file: One reference is held by the VFS and
* released when the file is closed. Another reference is taken when
* an asynchronous event queue file is created and released when the
* event file is closed.
*
* struct ib_uverbs_event_queue: Base structure for
* struct ib_uverbs_async_event_file and struct ib_uverbs_completion_event_file.
* One reference is held by the VFS and released when the file is closed.
* For asynchronous event files, another reference is held by the corresponding
* main context file and released when that file is closed. For completion
* event files, a reference is taken when a CQ is created that uses the file,
* and released when the CQ is destroyed.
*/
struct ib_uverbs_device {
atomic_t refcount;
u32 num_comp_vectors;
struct completion comp;
struct device *dev;
struct ib_device __rcu *ib_dev;
int devnum;
struct cdev cdev;
struct rb_root xrcd_tree;
struct mutex xrcd_tree_mutex;
struct kobject kobj;
struct srcu_struct disassociate_srcu;
struct mutex lists_mutex; /* protect lists */
struct list_head uverbs_file_list;
struct list_head uverbs_events_file_list;
struct uverbs_api *uapi;
};
struct ib_uverbs_event_queue {
spinlock_t lock;
int is_closed;
wait_queue_head_t poll_wait;
struct fasync_struct *async_queue;
struct list_head event_list;
};
struct ib_uverbs_async_event_file {
struct ib_uverbs_event_queue ev_queue;
struct ib_uverbs_file *uverbs_file;
struct kref ref;
struct list_head list;
};
struct ib_uverbs_completion_event_file {
struct ib_uobject uobj;
struct ib_uverbs_event_queue ev_queue;
};
struct ib_uverbs_file {
struct kref ref;
struct ib_uverbs_device *device;
struct mutex ucontext_lock;
/*
* ucontext must be accessed via ib_uverbs_get_ucontext() or with
* ucontext_lock held
*/
struct ib_ucontext *ucontext;
struct ib_event_handler event_handler;
struct ib_uverbs_async_event_file *async_file;
struct list_head list;
int is_closed;
/*
* To access the uobjects list hw_destroy_rwsem must be held for write
* OR hw_destroy_rwsem held for read AND uobjects_lock held.
* hw_destroy_rwsem should be called across any destruction of the HW
* object of an associated uobject.
*/
struct rw_semaphore hw_destroy_rwsem;
spinlock_t uobjects_lock;
struct list_head uobjects;
u64 uverbs_cmd_mask;
u64 uverbs_ex_cmd_mask;
struct idr idr;
/* spinlock protects write access to idr */
spinlock_t idr_lock;
};
struct ib_uverbs_event {
union {
struct ib_uverbs_async_event_desc async;
struct ib_uverbs_comp_event_desc comp;
} desc;
struct list_head list;
struct list_head obj_list;
u32 *counter;
};
struct ib_uverbs_mcast_entry {
struct list_head list;
union ib_gid gid;
u16 lid;
};
struct ib_uevent_object {
struct ib_uobject uobject;
struct list_head event_list;
u32 events_reported;
};
struct ib_uxrcd_object {
struct ib_uobject uobject;
atomic_t refcnt;
};
struct ib_usrq_object {
struct ib_uevent_object uevent;
struct ib_uxrcd_object *uxrcd;
};
struct ib_uqp_object {
struct ib_uevent_object uevent;
/* lock for mcast list */
struct mutex mcast_lock;
struct list_head mcast_list;
struct ib_uxrcd_object *uxrcd;
};
struct ib_uwq_object {
struct ib_uevent_object uevent;
};
struct ib_ucq_object {
struct ib_uobject uobject;
struct list_head comp_list;
struct list_head async_list;
u32 comp_events_reported;
u32 async_events_reported;
};
struct ib_uflow_resources;
struct ib_uflow_object {
struct ib_uobject uobject;
struct ib_uflow_resources *resources;
};
extern const struct file_operations uverbs_event_fops;
void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue);
struct file *ib_uverbs_alloc_async_event_file(struct ib_uverbs_file *uverbs_file,
struct ib_device *ib_dev);
void ib_uverbs_free_async_event_file(struct ib_uverbs_file *uverbs_file);
void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res);
void ib_uverbs_release_ucq(struct ib_uverbs_file *file,
struct ib_uverbs_completion_event_file *ev_file,
struct ib_ucq_object *uobj);
void ib_uverbs_release_uevent(struct ib_uverbs_file *file,
struct ib_uevent_object *uobj);
void ib_uverbs_release_file(struct kref *ref);
void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context);
void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr);
void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr);
void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr);
void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr);
void ib_uverbs_event_handler(struct ib_event_handler *handler,
struct ib_event *event);
int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,
enum rdma_remove_reason why);
int uverbs_dealloc_mw(struct ib_mw *mw);
void ib_uverbs_detach_umcast(struct ib_qp *qp,
struct ib_uqp_object *uobj);
void create_udata(struct uverbs_attr_bundle *ctx, struct ib_udata *udata);
long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
struct ib_uverbs_flow_spec {
union {
union {
struct ib_uverbs_flow_spec_hdr hdr;
struct {
__u32 type;
__u16 size;
__u16 reserved;
};
};
struct ib_uverbs_flow_spec_eth eth;
struct ib_uverbs_flow_spec_ipv4 ipv4;
struct ib_uverbs_flow_spec_esp esp;
struct ib_uverbs_flow_spec_tcp_udp tcp_udp;
struct ib_uverbs_flow_spec_ipv6 ipv6;
struct ib_uverbs_flow_spec_action_tag flow_tag;
struct ib_uverbs_flow_spec_action_drop drop;
struct ib_uverbs_flow_spec_action_handle action;
struct ib_uverbs_flow_spec_action_count flow_count;
};
};
int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,
const void *kern_spec_mask,
const void *kern_spec_val,
size_t kern_filter_sz,
union ib_flow_spec *ib_spec);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_DEVICE);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_PD);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_MR);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_COMP_CHANNEL);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_CQ);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_QP);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_AH);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_MW);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_SRQ);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_FLOW);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_WQ);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_RWQ_IND_TBL);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_XRCD);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_FLOW_ACTION);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_DM);
extern const struct uverbs_object_def UVERBS_OBJECT(UVERBS_OBJECT_COUNTERS);
#define IB_UVERBS_DECLARE_CMD(name) \
ssize_t ib_uverbs_##name(struct ib_uverbs_file *file, \
const char __user *buf, int in_len, \
int out_len)
IB_UVERBS_DECLARE_CMD(get_context);
IB_UVERBS_DECLARE_CMD(query_device);
IB_UVERBS_DECLARE_CMD(query_port);
IB_UVERBS_DECLARE_CMD(alloc_pd);
IB_UVERBS_DECLARE_CMD(dealloc_pd);
IB_UVERBS_DECLARE_CMD(reg_mr);
IB_UVERBS_DECLARE_CMD(rereg_mr);
IB_UVERBS_DECLARE_CMD(dereg_mr);
IB_UVERBS_DECLARE_CMD(alloc_mw);
IB_UVERBS_DECLARE_CMD(dealloc_mw);
IB_UVERBS_DECLARE_CMD(create_comp_channel);
IB_UVERBS_DECLARE_CMD(create_cq);
IB_UVERBS_DECLARE_CMD(resize_cq);
IB_UVERBS_DECLARE_CMD(poll_cq);
IB_UVERBS_DECLARE_CMD(req_notify_cq);
IB_UVERBS_DECLARE_CMD(destroy_cq);
IB_UVERBS_DECLARE_CMD(create_qp);
IB_UVERBS_DECLARE_CMD(open_qp);
IB_UVERBS_DECLARE_CMD(query_qp);
IB_UVERBS_DECLARE_CMD(modify_qp);
IB_UVERBS_DECLARE_CMD(destroy_qp);
IB_UVERBS_DECLARE_CMD(post_send);
IB_UVERBS_DECLARE_CMD(post_recv);
IB_UVERBS_DECLARE_CMD(post_srq_recv);
IB_UVERBS_DECLARE_CMD(create_ah);
IB_UVERBS_DECLARE_CMD(destroy_ah);
IB_UVERBS_DECLARE_CMD(attach_mcast);
IB_UVERBS_DECLARE_CMD(detach_mcast);
IB_UVERBS_DECLARE_CMD(create_srq);
IB_UVERBS_DECLARE_CMD(modify_srq);
IB_UVERBS_DECLARE_CMD(query_srq);
IB_UVERBS_DECLARE_CMD(destroy_srq);
IB_UVERBS_DECLARE_CMD(create_xsrq);
IB_UVERBS_DECLARE_CMD(open_xrcd);
IB_UVERBS_DECLARE_CMD(close_xrcd);
#define IB_UVERBS_DECLARE_EX_CMD(name) \
int ib_uverbs_ex_##name(struct ib_uverbs_file *file, \
struct ib_udata *ucore, \
struct ib_udata *uhw)
IB_UVERBS_DECLARE_EX_CMD(create_flow);
IB_UVERBS_DECLARE_EX_CMD(destroy_flow);
IB_UVERBS_DECLARE_EX_CMD(query_device);
IB_UVERBS_DECLARE_EX_CMD(create_cq);
IB_UVERBS_DECLARE_EX_CMD(create_qp);
IB_UVERBS_DECLARE_EX_CMD(create_wq);
IB_UVERBS_DECLARE_EX_CMD(modify_wq);
IB_UVERBS_DECLARE_EX_CMD(destroy_wq);
IB_UVERBS_DECLARE_EX_CMD(create_rwq_ind_table);
IB_UVERBS_DECLARE_EX_CMD(destroy_rwq_ind_table);
IB_UVERBS_DECLARE_EX_CMD(modify_qp);
IB_UVERBS_DECLARE_EX_CMD(modify_cq);
#endif /* UVERBS_H */
File diff suppressed because it is too large. Load diff
+613
View File
@@ -0,0 +1,613 @@
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <rdma/rdma_user_ioctl.h>
#include <rdma/uverbs_ioctl.h>
#include "rdma_core.h"
#include "uverbs.h"
struct bundle_alloc_head {
struct bundle_alloc_head *next;
u8 data[];
};
struct bundle_priv {
/* Must be first */
struct bundle_alloc_head alloc_head;
struct bundle_alloc_head *allocated_mem;
size_t internal_avail;
size_t internal_used;
struct radix_tree_root *radix;
const struct uverbs_api_ioctl_method *method_elm;
void __rcu **radix_slots;
unsigned long radix_slots_len;
u32 method_key;
struct ib_uverbs_attr __user *user_attrs;
struct ib_uverbs_attr *uattrs;
DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
/*
* Must be last. bundle ends in a flex array which overlaps
* internal_buffer.
*/
struct uverbs_attr_bundle bundle;
u64 internal_buffer[32];
};
/*
* Each method has an absolute minimum amount of memory it needs to allocate,
* precompute that amount and determine if the onstack memory can be used or
* if allocation is need.
*/
void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
unsigned int num_attrs)
{
struct bundle_priv *pbundle;
size_t bundle_size =
offsetof(struct bundle_priv, internal_buffer) +
sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len +
sizeof(*pbundle->uattrs) * num_attrs;
method_elm->use_stack = bundle_size <= sizeof(*pbundle);
method_elm->bundle_size =
ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
/* Do not want order-2 allocations for this. */
WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
}
/**
* uverbs_alloc() - Quickly allocate memory for use with a bundle
* @bundle: The bundle
* @size: Number of bytes to allocate
* @flags: Allocator flags
*
* The bundle allocator is intended for allocations that are connected with
* processing the system call related to the bundle. The allocated memory is
* always freed once the system call completes, and cannot be freed any other
* way.
*
* This tries to use a small pool of pre-allocated memory for performance.
*/
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
gfp_t flags)
{
struct bundle_priv *pbundle =
container_of(bundle, struct bundle_priv, bundle);
size_t new_used;
void *res;
if (check_add_overflow(size, pbundle->internal_used, &new_used))
return ERR_PTR(-EOVERFLOW);
if (new_used > pbundle->internal_avail) {
struct bundle_alloc_head *buf;
buf = kvmalloc(struct_size(buf, data, size), flags);
if (!buf)
return ERR_PTR(-ENOMEM);
buf->next = pbundle->allocated_mem;
pbundle->allocated_mem = buf;
return buf->data;
}
res = (void *)pbundle->internal_buffer + pbundle->internal_used;
pbundle->internal_used =
ALIGN(new_used, sizeof(*pbundle->internal_buffer));
if (want_init_on_alloc(flags))
memset(res, 0, size);
return res;
}
EXPORT_SYMBOL(_uverbs_alloc);
static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
u16 len)
{
if (uattr->len > sizeof(((struct ib_uverbs_attr *)0)->data))
return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
uattr->len - len);
return !memchr_inv((const void *)&uattr->data + len,
0, uattr->len - len);
}
static int uverbs_process_attr(struct bundle_priv *pbundle,
const struct uverbs_api_attr *attr_uapi,
struct ib_uverbs_attr *uattr, u32 attr_bkey)
{
const struct uverbs_attr_spec *spec = &attr_uapi->spec;
struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey];
const struct uverbs_attr_spec *val_spec = spec;
struct uverbs_obj_attr *o_attr;
switch (spec->type) {
case UVERBS_ATTR_TYPE_ENUM_IN:
if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
return -EOPNOTSUPP;
if (uattr->attr_data.enum_data.reserved)
return -EINVAL;
val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
/* Currently we only support PTR_IN based enums */
if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
return -EOPNOTSUPP;
e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
/* fall through */
case UVERBS_ATTR_TYPE_PTR_IN:
/* Ensure that any data provided by userspace beyond the known
* struct is zero. Userspace that knows how to use some future
* longer struct will fail here if used with an old kernel and
* non-zero content, making ABI compat/discovery simpler.
*/
if (uattr->len > val_spec->u.ptr.len &&
val_spec->zero_trailing &&
!uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
return -EOPNOTSUPP;
/* fall through */
case UVERBS_ATTR_TYPE_PTR_OUT:
if (uattr->len < val_spec->u.ptr.min_len ||
(!val_spec->zero_trailing &&
uattr->len > val_spec->u.ptr.len))
return -EINVAL;
if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
uattr->attr_data.reserved)
return -EINVAL;
e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
e->ptr_attr.len = uattr->len;
if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
void *p;
p = uverbs_alloc(&pbundle->bundle, uattr->len);
if (IS_ERR(p))
return PTR_ERR(p);
e->ptr_attr.ptr = p;
if (copy_from_user(p, u64_to_user_ptr(uattr->data),
uattr->len))
return -EFAULT;
} else {
e->ptr_attr.data = uattr->data;
}
break;
case UVERBS_ATTR_TYPE_IDR:
case UVERBS_ATTR_TYPE_FD:
if (uattr->attr_data.reserved)
return -EINVAL;
if (uattr->len != 0)
return -EINVAL;
o_attr = &e->obj_attr;
o_attr->attr_elm = attr_uapi;
/*
* The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
* s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
* here without caring about truncation as we know that the
* IDR implementation today rejects negative IDs
*/
o_attr->uobject = uverbs_get_uobject_from_file(
spec->u.obj.obj_type,
pbundle->bundle.ufile,
spec->u.obj.access,
uattr->data_s64);
if (IS_ERR(o_attr->uobject))
return PTR_ERR(o_attr->uobject);
__set_bit(attr_bkey, pbundle->uobj_finalize);
if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
unsigned int uattr_idx = uattr - pbundle->uattrs;
s64 id = o_attr->uobject->id;
/* Copy the allocated id to the user-space */
if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
return -EFAULT;
}
break;
default:
return -EOPNOTSUPP;
}
return 0;
}
/*
* We search the radix tree with the method prefix and now we want to fast
* search the suffix bits to get a particular attribute pointer. It is not
* totally clear to me if this breaks the radix tree encasulation or not, but
* it uses the iter data to determine if the method iter points at the same
* chunk that will store the attribute, if so it just derefs it directly. By
* construction in most kernel configs the method and attrs will all fit in a
* single radix chunk, so in most cases this will have no search. Other cases
* this falls back to a full search.
*/
static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
u32 attr_key)
{
void __rcu **slot;
if (likely(attr_key < pbundle->radix_slots_len)) {
void *entry;
slot = pbundle->radix_slots + attr_key;
entry = rcu_dereference_raw(*slot);
if (likely(!radix_tree_is_internal_node(entry) && entry))
return slot;
}
return radix_tree_lookup_slot(pbundle->radix,
pbundle->method_key | attr_key);
}
static int uverbs_set_attr(struct bundle_priv *pbundle,
struct ib_uverbs_attr *uattr)
{
u32 attr_key = uapi_key_attr(uattr->attr_id);
u32 attr_bkey = uapi_bkey_attr(attr_key);
const struct uverbs_api_attr *attr;
void __rcu **slot;
int ret;
slot = uapi_get_attr_for_method(pbundle, attr_key);
if (!slot) {
/*
* Kernel does not support the attribute but user-space says it
* is mandatory
*/
if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
return -EPROTONOSUPPORT;
return 0;
}
attr = srcu_dereference(
*slot, &pbundle->bundle.ufile->device->disassociate_srcu);
/* Reject duplicate attributes from user-space */
if (test_bit(attr_bkey, pbundle->bundle.attr_present))
return -EINVAL;
ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
if (ret)
return ret;
__set_bit(attr_bkey, pbundle->bundle.attr_present);
return 0;
}
static int ib_uverbs_run_method(struct bundle_priv *pbundle,
unsigned int num_attrs)
{
int (*handler)(struct ib_uverbs_file *ufile,
struct uverbs_attr_bundle *ctx);
size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
unsigned int i;
int ret;
/* See uverbs_disassociate_api() */
handler = srcu_dereference(
pbundle->method_elm->handler,
&pbundle->bundle.ufile->device->disassociate_srcu);
if (!handler)
return -EIO;
pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size);
if (IS_ERR(pbundle->uattrs))
return PTR_ERR(pbundle->uattrs);
if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
return -EFAULT;
for (i = 0; i != num_attrs; i++) {
ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
if (unlikely(ret))
return ret;
}
/* User space did not provide all the mandatory attributes */
if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
pbundle->bundle.attr_present,
pbundle->method_elm->key_bitmap_len)))
return -EINVAL;
if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
struct uverbs_obj_attr *destroy_attr =
&pbundle->bundle.attrs[destroy_bkey].obj_attr;
ret = uobj_destroy(destroy_attr->uobject);
if (ret)
return ret;
__clear_bit(destroy_bkey, pbundle->uobj_finalize);
ret = handler(pbundle->bundle.ufile, &pbundle->bundle);
uobj_put_destroy(destroy_attr->uobject);
} else {
ret = handler(pbundle->bundle.ufile, &pbundle->bundle);
}
/*
* EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
* not invoke the method because the request is not supported. No
* other cases should return this code.
*/
if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
return -EINVAL;
return ret;
}
static int bundle_destroy(struct bundle_priv *pbundle, bool commit)
{
unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
struct bundle_alloc_head *memblock;
unsigned int i;
int ret = 0;
i = -1;
while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
i + 1)) < key_bitmap_len) {
struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
int current_ret;
current_ret = uverbs_finalize_object(
attr->obj_attr.uobject,
attr->obj_attr.attr_elm->spec.u.obj.access, commit);
if (!ret)
ret = current_ret;
}
for (memblock = pbundle->allocated_mem; memblock;) {
struct bundle_alloc_head *tmp = memblock;
memblock = memblock->next;
kvfree(tmp);
}
return ret;
}
static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
struct ib_uverbs_ioctl_hdr *hdr,
struct ib_uverbs_attr __user *user_attrs)
{
const struct uverbs_api_ioctl_method *method_elm;
struct uverbs_api *uapi = ufile->device->uapi;
struct radix_tree_iter attrs_iter;
struct bundle_priv *pbundle;
struct bundle_priv onstack;
void __rcu **slot;
int destroy_ret;
int ret;
if (unlikely(hdr->driver_id != uapi->driver_id))
return -EINVAL;
slot = radix_tree_iter_lookup(
&uapi->radix, &attrs_iter,
uapi_key_obj(hdr->object_id) |
uapi_key_ioctl_method(hdr->method_id));
if (unlikely(!slot))
return -EPROTONOSUPPORT;
method_elm = srcu_dereference(*slot, &ufile->device->disassociate_srcu);
if (!method_elm->use_stack) {
pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
if (!pbundle)
return -ENOMEM;
pbundle->internal_avail =
method_elm->bundle_size -
offsetof(struct bundle_priv, internal_buffer);
pbundle->alloc_head.next = NULL;
pbundle->allocated_mem = &pbundle->alloc_head;
} else {
pbundle = &onstack;
pbundle->internal_avail = sizeof(pbundle->internal_buffer);
pbundle->allocated_mem = NULL;
}
/* Space for the pbundle->bundle.attrs flex array */
pbundle->method_elm = method_elm;
pbundle->method_key = attrs_iter.index;
pbundle->bundle.ufile = ufile;
pbundle->radix = &uapi->radix;
pbundle->radix_slots = slot;
pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
pbundle->user_attrs = user_attrs;
pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
sizeof(*pbundle->bundle.attrs),
sizeof(*pbundle->internal_buffer));
memset(pbundle->bundle.attr_present, 0,
sizeof(pbundle->bundle.attr_present));
memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
destroy_ret = bundle_destroy(pbundle, ret == 0);
if (unlikely(destroy_ret && !ret))
return destroy_ret;
return ret;
}
long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
struct ib_uverbs_file *file = filp->private_data;
struct ib_uverbs_ioctl_hdr __user *user_hdr =
(struct ib_uverbs_ioctl_hdr __user *)arg;
struct ib_uverbs_ioctl_hdr hdr;
int srcu_key;
int err;
if (unlikely(cmd != RDMA_VERBS_IOCTL))
return -ENOIOCTLCMD;
err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
if (err)
return -EFAULT;
if (hdr.length > PAGE_SIZE ||
hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
return -EINVAL;
if (hdr.reserved1 || hdr.reserved2)
return -EPROTONOSUPPORT;
srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
return err;
}
int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
const struct uverbs_attr *attr;
u64 flags;
attr = uverbs_attr_get(attrs_bundle, idx);
/* Missing attribute means 0 flags */
if (IS_ERR(attr)) {
*to = 0;
return 0;
}
/*
* New userspace code should use 8 bytes to pass flags, but we
* transparently support old userspaces that were using 4 bytes as
* well.
*/
if (attr->ptr_attr.len == 8)
flags = attr->ptr_attr.data;
else if (attr->ptr_attr.len == 4)
flags = *(u32 *)&attr->ptr_attr.data;
else
return -EINVAL;
if (flags & ~allowed_bits)
return -EINVAL;
*to = flags;
return 0;
}
EXPORT_SYMBOL(uverbs_get_flags64);
int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
u64 flags;
int ret;
ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
if (ret)
return ret;
if (flags > U32_MAX)
return -EINVAL;
*to = flags;
return 0;
}
EXPORT_SYMBOL(uverbs_get_flags32);
/*
* This is for ease of conversion. The purpose is to convert all drivers to
* use uverbs_attr_bundle instead of ib_udata. Assume attr == 0 is input and
* attr == 1 is output.
*/
void create_udata(struct uverbs_attr_bundle *bundle, struct ib_udata *udata)
{
struct bundle_priv *pbundle =
container_of(bundle, struct bundle_priv, bundle);
const struct uverbs_attr *uhw_in =
uverbs_attr_get(bundle, UVERBS_ATTR_UHW_IN);
const struct uverbs_attr *uhw_out =
uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT);
if (!IS_ERR(uhw_in)) {
udata->inlen = uhw_in->ptr_attr.len;
if (uverbs_attr_ptr_is_inline(uhw_in))
udata->inbuf =
&pbundle->user_attrs[uhw_in->ptr_attr.uattr_idx]
.data;
else
udata->inbuf = u64_to_user_ptr(uhw_in->ptr_attr.data);
} else {
udata->inbuf = NULL;
udata->inlen = 0;
}
if (!IS_ERR(uhw_out)) {
udata->outbuf = u64_to_user_ptr(uhw_out->ptr_attr.data);
udata->outlen = uhw_out->ptr_attr.len;
} else {
udata->outbuf = NULL;
udata->outlen = 0;
}
}
int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
const void *from, size_t size)
{
struct bundle_priv *pbundle =
container_of(bundle, struct bundle_priv, bundle);
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
u16 flags;
size_t min_size;
if (IS_ERR(attr))
return PTR_ERR(attr);
min_size = min_t(size_t, attr->ptr_attr.len, size);
if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
return -EFAULT;
flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
UVERBS_ATTR_F_VALID_OUTPUT;
if (put_user(flags,
&pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
return -EFAULT;
return 0;
}
EXPORT_SYMBOL(uverbs_copy_to);
File diff suppressed because it is too large. Load diff
+215
View File
@@ -0,0 +1,215 @@
/*
* Copyright (c) 2005 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/export.h>
#include <rdma/ib_marshall.h>
#define OPA_DEFAULT_GID_PREFIX cpu_to_be64(0xfe80000000000000ULL)
static int rdma_ah_conv_opa_to_ib(struct ib_device *dev,
struct rdma_ah_attr *ib,
struct rdma_ah_attr *opa)
{
struct ib_port_attr port_attr;
int ret = 0;
/* Do structure copy and the over-write fields */
*ib = *opa;
ib->type = RDMA_AH_ATTR_TYPE_IB;
rdma_ah_set_grh(ib, NULL, 0, 0, 1, 0);
if (ib_query_port(dev, opa->port_num, &port_attr)) {
/* Set to default subnet to indicate error */
rdma_ah_set_subnet_prefix(ib, OPA_DEFAULT_GID_PREFIX);
ret = -EINVAL;
} else {
rdma_ah_set_subnet_prefix(ib,
cpu_to_be64(port_attr.subnet_prefix));
}
rdma_ah_set_interface_id(ib, OPA_MAKE_ID(rdma_ah_get_dlid(opa)));
return ret;
}
void ib_copy_ah_attr_to_user(struct ib_device *device,
struct ib_uverbs_ah_attr *dst,
struct rdma_ah_attr *ah_attr)
{
struct rdma_ah_attr *src = ah_attr;
struct rdma_ah_attr conv_ah;
memset(&dst->grh.reserved, 0, sizeof(dst->grh.reserved));
if ((ah_attr->type == RDMA_AH_ATTR_TYPE_OPA) &&
(rdma_ah_get_dlid(ah_attr) > be16_to_cpu(IB_LID_PERMISSIVE)) &&
(!rdma_ah_conv_opa_to_ib(device, &conv_ah, ah_attr)))
src = &conv_ah;
dst->dlid = rdma_ah_get_dlid(src);
dst->sl = rdma_ah_get_sl(src);
dst->src_path_bits = rdma_ah_get_path_bits(src);
dst->static_rate = rdma_ah_get_static_rate(src);
dst->is_global = rdma_ah_get_ah_flags(src) &
IB_AH_GRH ? 1 : 0;
if (dst->is_global) {
const struct ib_global_route *grh = rdma_ah_read_grh(src);
memcpy(dst->grh.dgid, grh->dgid.raw, sizeof(grh->dgid));
dst->grh.flow_label = grh->flow_label;
dst->grh.sgid_index = grh->sgid_index;
dst->grh.hop_limit = grh->hop_limit;
dst->grh.traffic_class = grh->traffic_class;
}
dst->port_num = rdma_ah_get_port_num(src);
dst->reserved = 0;
}
EXPORT_SYMBOL(ib_copy_ah_attr_to_user);
void ib_copy_qp_attr_to_user(struct ib_device *device,
struct ib_uverbs_qp_attr *dst,
struct ib_qp_attr *src)
{
dst->qp_state = src->qp_state;
dst->cur_qp_state = src->cur_qp_state;
dst->path_mtu = src->path_mtu;
dst->path_mig_state = src->path_mig_state;
dst->qkey = src->qkey;
dst->rq_psn = src->rq_psn;
dst->sq_psn = src->sq_psn;
dst->dest_qp_num = src->dest_qp_num;
dst->qp_access_flags = src->qp_access_flags;
dst->max_send_wr = src->cap.max_send_wr;
dst->max_recv_wr = src->cap.max_recv_wr;
dst->max_send_sge = src->cap.max_send_sge;
dst->max_recv_sge = src->cap.max_recv_sge;
dst->max_inline_data = src->cap.max_inline_data;
ib_copy_ah_attr_to_user(device, &dst->ah_attr, &src->ah_attr);
ib_copy_ah_attr_to_user(device, &dst->alt_ah_attr, &src->alt_ah_attr);
dst->pkey_index = src->pkey_index;
dst->alt_pkey_index = src->alt_pkey_index;
dst->en_sqd_async_notify = src->en_sqd_async_notify;
dst->sq_draining = src->sq_draining;
dst->max_rd_atomic = src->max_rd_atomic;
dst->max_dest_rd_atomic = src->max_dest_rd_atomic;
dst->min_rnr_timer = src->min_rnr_timer;
dst->port_num = src->port_num;
dst->timeout = src->timeout;
dst->retry_cnt = src->retry_cnt;
dst->rnr_retry = src->rnr_retry;
dst->alt_port_num = src->alt_port_num;
dst->alt_timeout = src->alt_timeout;
memset(dst->reserved, 0, sizeof(dst->reserved));
}
EXPORT_SYMBOL(ib_copy_qp_attr_to_user);
static void __ib_copy_path_rec_to_user(struct ib_user_path_rec *dst,
struct sa_path_rec *src)
{
memcpy(dst->dgid, src->dgid.raw, sizeof(src->dgid));
memcpy(dst->sgid, src->sgid.raw, sizeof(src->sgid));
dst->dlid = htons(ntohl(sa_path_get_dlid(src)));
dst->slid = htons(ntohl(sa_path_get_slid(src)));
dst->raw_traffic = sa_path_get_raw_traffic(src);
dst->flow_label = src->flow_label;
dst->hop_limit = src->hop_limit;
dst->traffic_class = src->traffic_class;
dst->reversible = src->reversible;
dst->numb_path = src->numb_path;
dst->pkey = src->pkey;
dst->sl = src->sl;
dst->mtu_selector = src->mtu_selector;
dst->mtu = src->mtu;
dst->rate_selector = src->rate_selector;
dst->rate = src->rate;
dst->packet_life_time = src->packet_life_time;
dst->preference = src->preference;
dst->packet_life_time_selector = src->packet_life_time_selector;
}
void ib_copy_path_rec_to_user(struct ib_user_path_rec *dst,
struct sa_path_rec *src)
{
struct sa_path_rec rec;
if (src->rec_type == SA_PATH_REC_TYPE_OPA) {
sa_convert_path_opa_to_ib(&rec, src);
__ib_copy_path_rec_to_user(dst, &rec);
return;
}
__ib_copy_path_rec_to_user(dst, src);
}
EXPORT_SYMBOL(ib_copy_path_rec_to_user);
void ib_copy_path_rec_from_user(struct sa_path_rec *dst,
struct ib_user_path_rec *src)
{
u32 slid, dlid;
memset(dst, 0, sizeof(*dst));
if ((ib_is_opa_gid((union ib_gid *)src->sgid)) ||
(ib_is_opa_gid((union ib_gid *)src->dgid))) {
dst->rec_type = SA_PATH_REC_TYPE_OPA;
slid = opa_get_lid_from_gid((union ib_gid *)src->sgid);
dlid = opa_get_lid_from_gid((union ib_gid *)src->dgid);
} else {
dst->rec_type = SA_PATH_REC_TYPE_IB;
slid = ntohs(src->slid);
dlid = ntohs(src->dlid);
}
memcpy(dst->dgid.raw, src->dgid, sizeof dst->dgid);
memcpy(dst->sgid.raw, src->sgid, sizeof dst->sgid);
sa_path_set_dlid(dst, dlid);
sa_path_set_slid(dst, slid);
sa_path_set_raw_traffic(dst, src->raw_traffic);
dst->flow_label = src->flow_label;
dst->hop_limit = src->hop_limit;
dst->traffic_class = src->traffic_class;
dst->reversible = src->reversible;
dst->numb_path = src->numb_path;
dst->pkey = src->pkey;
dst->sl = src->sl;
dst->mtu_selector = src->mtu_selector;
dst->mtu = src->mtu;
dst->rate_selector = src->rate_selector;
dst->rate = src->rate;
dst->packet_life_time = src->packet_life_time;
dst->preference = src->preference;
dst->packet_life_time_selector = src->packet_life_time_selector;
/* TODO: No need to set this */
sa_path_set_dmac_zero(dst);
}
EXPORT_SYMBOL(ib_copy_path_rec_from_user);
+286
View File
@@ -0,0 +1,286 @@
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <rdma/uverbs_std_types.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/ib_verbs.h>
#include <linux/bug.h>
#include <linux/file.h>
#include <rdma/restrack.h>
#include "rdma_core.h"
#include "uverbs.h"
static int uverbs_free_ah(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
return rdma_destroy_ah((struct ib_ah *)uobject->object);
}
static int uverbs_free_flow(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_flow *flow = (struct ib_flow *)uobject->object;
struct ib_uflow_object *uflow =
container_of(uobject, struct ib_uflow_object, uobject);
struct ib_qp *qp = flow->qp;
int ret;
ret = flow->device->destroy_flow(flow);
if (!ret) {
if (qp)
atomic_dec(&qp->usecnt);
ib_uverbs_flow_resources_free(uflow->resources);
}
return ret;
}
static int uverbs_free_mw(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
return uverbs_dealloc_mw((struct ib_mw *)uobject->object);
}
static int uverbs_free_qp(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_qp *qp = uobject->object;
struct ib_uqp_object *uqp =
container_of(uobject, struct ib_uqp_object, uevent.uobject);
int ret;
/*
* If this is a user triggered destroy then do not allow destruction
* until the user cleans up all the mcast bindings. Unlike in other
* places we forcibly clean up the mcast attachments for !DESTROY
* because the mcast attaches are not ubojects and will not be
* destroyed by anything else during cleanup processing.
*/
if (why == RDMA_REMOVE_DESTROY) {
if (!list_empty(&uqp->mcast_list))
return -EBUSY;
} else if (qp == qp->real_qp) {
ib_uverbs_detach_umcast(qp, uqp);
}
ret = ib_destroy_qp(qp);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
if (uqp->uxrcd)
atomic_dec(&uqp->uxrcd->refcnt);
ib_uverbs_release_uevent(uobject->context->ufile, &uqp->uevent);
return ret;
}
static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_rwq_ind_table *rwq_ind_tbl = uobject->object;
struct ib_wq **ind_tbl = rwq_ind_tbl->ind_tbl;
int ret;
ret = ib_destroy_rwq_ind_table(rwq_ind_tbl);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
kfree(ind_tbl);
return ret;
}
static int uverbs_free_wq(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_wq *wq = uobject->object;
struct ib_uwq_object *uwq =
container_of(uobject, struct ib_uwq_object, uevent.uobject);
int ret;
ret = ib_destroy_wq(wq);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
ib_uverbs_release_uevent(uobject->context->ufile, &uwq->uevent);
return ret;
}
static int uverbs_free_srq(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_srq *srq = uobject->object;
struct ib_uevent_object *uevent =
container_of(uobject, struct ib_uevent_object, uobject);
enum ib_srq_type srq_type = srq->srq_type;
int ret;
ret = ib_destroy_srq(srq);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
if (srq_type == IB_SRQT_XRC) {
struct ib_usrq_object *us =
container_of(uevent, struct ib_usrq_object, uevent);
atomic_dec(&us->uxrcd->refcnt);
}
ib_uverbs_release_uevent(uobject->context->ufile, uevent);
return ret;
}
static int uverbs_free_xrcd(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_xrcd *xrcd = uobject->object;
struct ib_uxrcd_object *uxrcd =
container_of(uobject, struct ib_uxrcd_object, uobject);
int ret;
ret = ib_destroy_usecnt(&uxrcd->refcnt, why, uobject);
if (ret)
return ret;
mutex_lock(&uobject->context->ufile->device->xrcd_tree_mutex);
ret = ib_uverbs_dealloc_xrcd(uobject, xrcd, why);
mutex_unlock(&uobject->context->ufile->device->xrcd_tree_mutex);
return ret;
}
static int uverbs_free_pd(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_pd *pd = uobject->object;
int ret;
ret = ib_destroy_usecnt(&pd->usecnt, why, uobject);
if (ret)
return ret;
ib_dealloc_pd((struct ib_pd *)uobject->object);
return 0;
}
static int uverbs_hot_unplug_completion_event_file(struct ib_uobject *uobj,
enum rdma_remove_reason why)
{
struct ib_uverbs_completion_event_file *comp_event_file =
container_of(uobj, struct ib_uverbs_completion_event_file,
uobj);
struct ib_uverbs_event_queue *event_queue = &comp_event_file->ev_queue;
spin_lock_irq(&event_queue->lock);
event_queue->is_closed = 1;
spin_unlock_irq(&event_queue->lock);
if (why == RDMA_REMOVE_DRIVER_REMOVE) {
wake_up_interruptible(&event_queue->poll_wait);
kill_fasync(&event_queue->async_queue, SIGIO, POLL_IN);
}
return 0;
};
int uverbs_destroy_def_handler(struct ib_uverbs_file *file,
struct uverbs_attr_bundle *attrs)
{
return 0;
}
EXPORT_SYMBOL(uverbs_destroy_def_handler);
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_COMP_CHANNEL,
UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_completion_event_file),
uverbs_hot_unplug_completion_event_file,
&uverbs_event_fops,
"[infinibandevent]",
O_RDONLY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_QP,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), uverbs_free_qp));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_MW,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_mw));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_SRQ,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_usrq_object),
uverbs_free_srq));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_AH,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_ah));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_FLOW,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uflow_object),
uverbs_free_flow));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_WQ,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uwq_object), uverbs_free_wq));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_RWQ_IND_TBL,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_rwq_ind_tbl));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_XRCD,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uxrcd_object),
uverbs_free_xrcd));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_PD,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_pd));
DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE);
DECLARE_UVERBS_OBJECT_TREE(uverbs_default_objects,
&UVERBS_OBJECT(UVERBS_OBJECT_DEVICE),
&UVERBS_OBJECT(UVERBS_OBJECT_PD),
&UVERBS_OBJECT(UVERBS_OBJECT_MR),
&UVERBS_OBJECT(UVERBS_OBJECT_COMP_CHANNEL),
&UVERBS_OBJECT(UVERBS_OBJECT_CQ),
&UVERBS_OBJECT(UVERBS_OBJECT_QP),
&UVERBS_OBJECT(UVERBS_OBJECT_AH),
&UVERBS_OBJECT(UVERBS_OBJECT_MW),
&UVERBS_OBJECT(UVERBS_OBJECT_SRQ),
&UVERBS_OBJECT(UVERBS_OBJECT_FLOW),
&UVERBS_OBJECT(UVERBS_OBJECT_WQ),
&UVERBS_OBJECT(UVERBS_OBJECT_RWQ_IND_TBL),
&UVERBS_OBJECT(UVERBS_OBJECT_XRCD),
&UVERBS_OBJECT(UVERBS_OBJECT_FLOW_ACTION),
&UVERBS_OBJECT(UVERBS_OBJECT_DM),
&UVERBS_OBJECT(UVERBS_OBJECT_COUNTERS));
const struct uverbs_object_tree_def *uverbs_default_get_objects(void)
{
return &uverbs_default_objects;
}
@@ -0,0 +1,151 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "uverbs.h"
#include <rdma/uverbs_std_types.h>
static int uverbs_free_counters(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_counters *counters = uobject->object;
int ret;
ret = ib_destroy_usecnt(&counters->usecnt, why, uobject);
if (ret)
return ret;
return counters->device->destroy_counters(counters);
}
static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_CREATE)(
struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj = uverbs_attr_get_uobject(
attrs, UVERBS_ATTR_CREATE_COUNTERS_HANDLE);
struct ib_device *ib_dev = uobj->context->device;
struct ib_counters *counters;
int ret;
/*
* This check should be removed once the infrastructure
* have the ability to remove methods from parse tree once
* such condition is met.
*/
if (!ib_dev->create_counters)
return -EOPNOTSUPP;
counters = ib_dev->create_counters(ib_dev, attrs);
if (IS_ERR(counters)) {
ret = PTR_ERR(counters);
goto err_create_counters;
}
counters->device = ib_dev;
counters->uobject = uobj;
uobj->object = counters;
atomic_set(&counters->usecnt, 0);
return 0;
err_create_counters:
return ret;
}
static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_READ)(
struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
{
struct ib_counters_read_attr read_attr = {};
const struct uverbs_attr *uattr;
struct ib_counters *counters =
uverbs_attr_get_obj(attrs, UVERBS_ATTR_READ_COUNTERS_HANDLE);
int ret;
if (!counters->device->read_counters)
return -EOPNOTSUPP;
if (!atomic_read(&counters->usecnt))
return -EINVAL;
ret = uverbs_get_flags32(&read_attr.flags, attrs,
UVERBS_ATTR_READ_COUNTERS_FLAGS,
IB_UVERBS_READ_COUNTERS_PREFER_CACHED);
if (ret)
return ret;
uattr = uverbs_attr_get(attrs, UVERBS_ATTR_READ_COUNTERS_BUFF);
read_attr.ncounters = uattr->ptr_attr.len / sizeof(u64);
read_attr.counters_buff = uverbs_zalloc(
attrs, array_size(read_attr.ncounters, sizeof(u64)));
if (IS_ERR(read_attr.counters_buff))
return PTR_ERR(read_attr.counters_buff);
ret = counters->device->read_counters(counters, &read_attr, attrs);
if (ret)
return ret;
return uverbs_copy_to(attrs, UVERBS_ATTR_READ_COUNTERS_BUFF,
read_attr.counters_buff,
read_attr.ncounters * sizeof(u64));
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_COUNTERS_CREATE,
UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_COUNTERS_HANDLE,
UVERBS_OBJECT_COUNTERS,
UVERBS_ACCESS_NEW,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_COUNTERS_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_COUNTERS_HANDLE,
UVERBS_OBJECT_COUNTERS,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_COUNTERS_READ,
UVERBS_ATTR_IDR(UVERBS_ATTR_READ_COUNTERS_HANDLE,
UVERBS_OBJECT_COUNTERS,
UVERBS_ACCESS_READ,
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_READ_COUNTERS_BUFF,
UVERBS_ATTR_MIN_SIZE(0),
UA_MANDATORY),
UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_READ_COUNTERS_FLAGS,
enum ib_uverbs_read_counters_flags));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_COUNTERS,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_counters),
&UVERBS_METHOD(UVERBS_METHOD_COUNTERS_CREATE),
&UVERBS_METHOD(UVERBS_METHOD_COUNTERS_DESTROY),
&UVERBS_METHOD(UVERBS_METHOD_COUNTERS_READ));
@@ -0,0 +1,209 @@
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <rdma/uverbs_std_types.h>
#include "rdma_core.h"
#include "uverbs.h"
static int uverbs_free_cq(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_cq *cq = uobject->object;
struct ib_uverbs_event_queue *ev_queue = cq->cq_context;
struct ib_ucq_object *ucq =
container_of(uobject, struct ib_ucq_object, uobject);
int ret;
ret = ib_destroy_cq(cq);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
ib_uverbs_release_ucq(
uobject->context->ufile,
ev_queue ? container_of(ev_queue,
struct ib_uverbs_completion_event_file,
ev_queue) :
NULL,
ucq);
return ret;
}
static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
{
struct ib_ucq_object *obj = container_of(
uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE),
typeof(*obj), uobject);
struct ib_device *ib_dev = obj->uobject.context->device;
struct ib_udata uhw;
int ret;
u64 user_handle;
struct ib_cq_init_attr attr = {};
struct ib_cq *cq;
struct ib_uverbs_completion_event_file *ev_file = NULL;
struct ib_uobject *ev_file_uobj;
if (!ib_dev->create_cq || !ib_dev->destroy_cq)
return -EOPNOTSUPP;
ret = uverbs_copy_from(&attr.comp_vector, attrs,
UVERBS_ATTR_CREATE_CQ_COMP_VECTOR);
if (!ret)
ret = uverbs_copy_from(&attr.cqe, attrs,
UVERBS_ATTR_CREATE_CQ_CQE);
if (!ret)
ret = uverbs_copy_from(&user_handle, attrs,
UVERBS_ATTR_CREATE_CQ_USER_HANDLE);
if (ret)
return ret;
ret = uverbs_get_flags32(&attr.flags, attrs,
UVERBS_ATTR_CREATE_CQ_FLAGS,
IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION |
IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN);
if (ret)
return ret;
ev_file_uobj = uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_CQ_COMP_CHANNEL);
if (!IS_ERR(ev_file_uobj)) {
ev_file = container_of(ev_file_uobj,
struct ib_uverbs_completion_event_file,
uobj);
uverbs_uobject_get(ev_file_uobj);
}
if (attr.comp_vector >= file->device->num_comp_vectors) {
ret = -EINVAL;
goto err_event_file;
}
obj->comp_events_reported = 0;
obj->async_events_reported = 0;
INIT_LIST_HEAD(&obj->comp_list);
INIT_LIST_HEAD(&obj->async_list);
/* Temporary, only until drivers get the new uverbs_attr_bundle */
create_udata(attrs, &uhw);
cq = ib_dev->create_cq(ib_dev, &attr, obj->uobject.context, &uhw);
if (IS_ERR(cq)) {
ret = PTR_ERR(cq);
goto err_event_file;
}
cq->device = ib_dev;
cq->uobject = &obj->uobject;
cq->comp_handler = ib_uverbs_comp_handler;
cq->event_handler = ib_uverbs_cq_event_handler;
cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
obj->uobject.object = cq;
obj->uobject.user_handle = user_handle;
atomic_set(&cq->usecnt, 0);
cq->res.type = RDMA_RESTRACK_CQ;
rdma_restrack_add(&cq->res);
ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &cq->cqe,
sizeof(cq->cqe));
if (ret)
goto err_cq;
return 0;
err_cq:
ib_destroy_cq(cq);
err_event_file:
if (ev_file)
uverbs_uobject_put(ev_file_uobj);
return ret;
};
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_CQ_CREATE,
UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_CQ_HANDLE,
UVERBS_OBJECT_CQ,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_CQE,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_USER_HANDLE,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_CQ_COMP_CHANNEL,
UVERBS_OBJECT_COMP_CHANNEL,
UVERBS_ACCESS_READ,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_COMP_VECTOR,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_CREATE_CQ_FLAGS,
enum ib_uverbs_ex_create_cq_flags),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_CQ_RESP_CQE,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_UHW());
static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)(
struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj =
uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_CQ_HANDLE);
struct ib_ucq_object *obj =
container_of(uobj, struct ib_ucq_object, uobject);
struct ib_uverbs_destroy_cq_resp resp = {
.comp_events_reported = obj->comp_events_reported,
.async_events_reported = obj->async_events_reported
};
return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_CQ_RESP, &resp,
sizeof(resp));
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_CQ_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_CQ_HANDLE,
UVERBS_OBJECT_CQ,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_DESTROY_CQ_RESP,
UVERBS_ATTR_TYPE(struct ib_uverbs_destroy_cq_resp),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_CQ,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_ucq_object), uverbs_free_cq),
#if IS_ENABLED(CONFIG_INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI)
&UVERBS_METHOD(UVERBS_METHOD_CQ_CREATE),
&UVERBS_METHOD(UVERBS_METHOD_CQ_DESTROY)
#endif
);
@@ -0,0 +1,111 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "uverbs.h"
#include <rdma/uverbs_std_types.h>
static int uverbs_free_dm(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_dm *dm = uobject->object;
int ret;
ret = ib_destroy_usecnt(&dm->usecnt, why, uobject);
if (ret)
return ret;
return dm->device->dealloc_dm(dm);
}
static int
UVERBS_HANDLER(UVERBS_METHOD_DM_ALLOC)(struct ib_uverbs_file *file,
struct uverbs_attr_bundle *attrs)
{
struct ib_dm_alloc_attr attr = {};
struct ib_uobject *uobj =
uverbs_attr_get(attrs, UVERBS_ATTR_ALLOC_DM_HANDLE)
->obj_attr.uobject;
struct ib_device *ib_dev = uobj->context->device;
struct ib_dm *dm;
int ret;
if (!ib_dev->alloc_dm)
return -EOPNOTSUPP;
ret = uverbs_copy_from(&attr.length, attrs,
UVERBS_ATTR_ALLOC_DM_LENGTH);
if (ret)
return ret;
ret = uverbs_copy_from(&attr.alignment, attrs,
UVERBS_ATTR_ALLOC_DM_ALIGNMENT);
if (ret)
return ret;
dm = ib_dev->alloc_dm(ib_dev, uobj->context, &attr, attrs);
if (IS_ERR(dm))
return PTR_ERR(dm);
dm->device = ib_dev;
dm->length = attr.length;
dm->uobject = uobj;
atomic_set(&dm->usecnt, 0);
uobj->object = dm;
return 0;
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_DM_ALLOC,
UVERBS_ATTR_IDR(UVERBS_ATTR_ALLOC_DM_HANDLE,
UVERBS_OBJECT_DM,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_ALLOC_DM_LENGTH,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_ALLOC_DM_ALIGNMENT,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_DM_FREE,
UVERBS_ATTR_IDR(UVERBS_ATTR_FREE_DM_HANDLE,
UVERBS_OBJECT_DM,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_DM,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_dm),
&UVERBS_METHOD(UVERBS_METHOD_DM_ALLOC),
&UVERBS_METHOD(UVERBS_METHOD_DM_FREE));
@@ -0,0 +1,443 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "uverbs.h"
#include <rdma/uverbs_std_types.h>
static int uverbs_free_flow_action(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
struct ib_flow_action *action = uobject->object;
int ret;
ret = ib_destroy_usecnt(&action->usecnt, why, uobject);
if (ret)
return ret;
return action->device->destroy_flow_action(action);
}
static u64 esp_flags_uverbs_to_verbs(struct uverbs_attr_bundle *attrs,
u32 flags, bool is_modify)
{
u64 verbs_flags = flags;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_ESN))
verbs_flags |= IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED;
if (is_modify && uverbs_attr_is_valid(attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS))
verbs_flags |= IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS;
return verbs_flags;
};
static int validate_flow_action_esp_keymat_aes_gcm(struct ib_flow_action_attrs_esp_keymats *keymat)
{
struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm =
&keymat->keymat.aes_gcm;
if (aes_gcm->iv_algo > IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)
return -EOPNOTSUPP;
if (aes_gcm->key_len != 32 &&
aes_gcm->key_len != 24 &&
aes_gcm->key_len != 16)
return -EINVAL;
if (aes_gcm->icv_len != 16 &&
aes_gcm->icv_len != 8 &&
aes_gcm->icv_len != 12)
return -EINVAL;
return 0;
}
static int (* const flow_action_esp_keymat_validate[])(struct ib_flow_action_attrs_esp_keymats *keymat) = {
[IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM] = validate_flow_action_esp_keymat_aes_gcm,
};
static int flow_action_esp_replay_none(struct ib_flow_action_attrs_esp_replays *replay,
bool is_modify)
{
/* This is used in order to modify an esp flow action with an enabled
* replay protection to a disabled one. This is only supported via
* modify, as in create verb we can simply drop the REPLAY attribute and
* achieve the same thing.
*/
return is_modify ? 0 : -EINVAL;
}
static int flow_action_esp_replay_def_ok(struct ib_flow_action_attrs_esp_replays *replay,
bool is_modify)
{
/* Some replay protections could always be enabled without validating
* anything.
*/
return 0;
}
static int (* const flow_action_esp_replay_validate[])(struct ib_flow_action_attrs_esp_replays *replay,
bool is_modify) = {
[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_NONE] = flow_action_esp_replay_none,
[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_BMP] = flow_action_esp_replay_def_ok,
};
static int parse_esp_ip(enum ib_flow_spec_type proto,
const void __user *val_ptr,
size_t len, union ib_flow_spec *out)
{
int ret;
const struct ib_uverbs_flow_ipv4_filter ipv4 = {
.src_ip = cpu_to_be32(0xffffffffUL),
.dst_ip = cpu_to_be32(0xffffffffUL),
.proto = 0xff,
.tos = 0xff,
.ttl = 0xff,
.flags = 0xff,
};
const struct ib_uverbs_flow_ipv6_filter ipv6 = {
.src_ip = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
.dst_ip = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
.flow_label = cpu_to_be32(0xffffffffUL),
.next_hdr = 0xff,
.traffic_class = 0xff,
.hop_limit = 0xff,
};
union {
struct ib_uverbs_flow_ipv4_filter ipv4;
struct ib_uverbs_flow_ipv6_filter ipv6;
} user_val = {};
const void *user_pmask;
size_t val_len;
/* If the flow IPv4/IPv6 flow specifications are extended, the mask
* should be changed as well.
*/
BUILD_BUG_ON(offsetof(struct ib_uverbs_flow_ipv4_filter, flags) +
sizeof(ipv4.flags) != sizeof(ipv4));
BUILD_BUG_ON(offsetof(struct ib_uverbs_flow_ipv6_filter, reserved) +
sizeof(ipv6.reserved) != sizeof(ipv6));
switch (proto) {
case IB_FLOW_SPEC_IPV4:
if (len > sizeof(user_val.ipv4) &&
!ib_is_buffer_cleared(val_ptr + sizeof(user_val.ipv4),
len - sizeof(user_val.ipv4)))
return -EOPNOTSUPP;
val_len = min_t(size_t, len, sizeof(user_val.ipv4));
ret = copy_from_user(&user_val.ipv4, val_ptr,
val_len);
if (ret)
return -EFAULT;
user_pmask = &ipv4;
break;
case IB_FLOW_SPEC_IPV6:
if (len > sizeof(user_val.ipv6) &&
!ib_is_buffer_cleared(val_ptr + sizeof(user_val.ipv6),
len - sizeof(user_val.ipv6)))
return -EOPNOTSUPP;
val_len = min_t(size_t, len, sizeof(user_val.ipv6));
ret = copy_from_user(&user_val.ipv6, val_ptr,
val_len);
if (ret)
return -EFAULT;
user_pmask = &ipv6;
break;
default:
return -EOPNOTSUPP;
}
return ib_uverbs_kern_spec_to_ib_spec_filter(proto, user_pmask,
&user_val,
val_len, out);
}
static int flow_action_esp_get_encap(struct ib_flow_spec_list *out,
struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_flow_action_esp_encap uverbs_encap;
int ret;
ret = uverbs_copy_from(&uverbs_encap, attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP);
if (ret)
return ret;
/* We currently support only one encap */
if (uverbs_encap.next_ptr)
return -EOPNOTSUPP;
if (uverbs_encap.type != IB_FLOW_SPEC_IPV4 &&
uverbs_encap.type != IB_FLOW_SPEC_IPV6)
return -EOPNOTSUPP;
return parse_esp_ip(uverbs_encap.type,
u64_to_user_ptr(uverbs_encap.val_ptr),
uverbs_encap.len,
&out->spec);
}
struct ib_flow_action_esp_attr {
struct ib_flow_action_attrs_esp hdr;
struct ib_flow_action_attrs_esp_keymats keymat;
struct ib_flow_action_attrs_esp_replays replay;
/* We currently support only one spec */
struct ib_flow_spec_list encap;
};
#define ESP_LAST_SUPPORTED_FLAG IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW
static int parse_flow_action_esp(struct ib_device *ib_dev,
struct ib_uverbs_file *file,
struct uverbs_attr_bundle *attrs,
struct ib_flow_action_esp_attr *esp_attr,
bool is_modify)
{
struct ib_uverbs_flow_action_esp uverbs_esp = {};
int ret;
/* Optional param, if it doesn't exist, we get -ENOENT and skip it */
ret = uverbs_copy_from(&esp_attr->hdr.esn, attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_ESN);
if (IS_UVERBS_COPY_ERR(ret))
return ret;
/* This can be called from FLOW_ACTION_ESP_MODIFY where
* UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS is optional
*/
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS)) {
ret = uverbs_copy_from_or_zero(&uverbs_esp, attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS);
if (ret)
return ret;
if (uverbs_esp.flags & ~((ESP_LAST_SUPPORTED_FLAG << 1) - 1))
return -EOPNOTSUPP;
esp_attr->hdr.spi = uverbs_esp.spi;
esp_attr->hdr.seq = uverbs_esp.seq;
esp_attr->hdr.tfc_pad = uverbs_esp.tfc_pad;
esp_attr->hdr.hard_limit_pkts = uverbs_esp.hard_limit_pkts;
}
esp_attr->hdr.flags = esp_flags_uverbs_to_verbs(attrs, uverbs_esp.flags,
is_modify);
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT)) {
esp_attr->keymat.protocol =
uverbs_attr_get_enum_id(attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT);
ret = uverbs_copy_from_or_zero(&esp_attr->keymat.keymat,
attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT);
if (ret)
return ret;
ret = flow_action_esp_keymat_validate[esp_attr->keymat.protocol](&esp_attr->keymat);
if (ret)
return ret;
esp_attr->hdr.keymat = &esp_attr->keymat;
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY)) {
esp_attr->replay.protocol =
uverbs_attr_get_enum_id(attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY);
ret = uverbs_copy_from_or_zero(&esp_attr->replay.replay,
attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY);
if (ret)
return ret;
ret = flow_action_esp_replay_validate[esp_attr->replay.protocol](&esp_attr->replay,
is_modify);
if (ret)
return ret;
esp_attr->hdr.replay = &esp_attr->replay;
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP)) {
ret = flow_action_esp_get_encap(&esp_attr->encap, attrs);
if (ret)
return ret;
esp_attr->hdr.encap = &esp_attr->encap;
}
return 0;
}
static int UVERBS_HANDLER(UVERBS_METHOD_FLOW_ACTION_ESP_CREATE)(
struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj = uverbs_attr_get_uobject(
attrs, UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE);
struct ib_device *ib_dev = uobj->context->device;
int ret;
struct ib_flow_action *action;
struct ib_flow_action_esp_attr esp_attr = {};
if (!ib_dev->create_flow_action_esp)
return -EOPNOTSUPP;
ret = parse_flow_action_esp(ib_dev, file, attrs, &esp_attr, false);
if (ret)
return ret;
/* No need to check as this attribute is marked as MANDATORY */
action = ib_dev->create_flow_action_esp(ib_dev, &esp_attr.hdr, attrs);
if (IS_ERR(action))
return PTR_ERR(action);
atomic_set(&action->usecnt, 0);
action->device = ib_dev;
action->type = IB_FLOW_ACTION_ESP;
action->uobject = uobj;
uobj->object = action;
return 0;
}
static int UVERBS_HANDLER(UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY)(
struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj = uverbs_attr_get_uobject(
attrs, UVERBS_ATTR_MODIFY_FLOW_ACTION_ESP_HANDLE);
struct ib_flow_action *action = uobj->object;
int ret;
struct ib_flow_action_esp_attr esp_attr = {};
if (!action->device->modify_flow_action_esp)
return -EOPNOTSUPP;
ret = parse_flow_action_esp(action->device, file, attrs, &esp_attr,
true);
if (ret)
return ret;
if (action->type != IB_FLOW_ACTION_ESP)
return -EINVAL;
return action->device->modify_flow_action_esp(action, &esp_attr.hdr,
attrs);
}
static const struct uverbs_attr_spec uverbs_flow_action_esp_keymat[] = {
[IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM] = {
.type = UVERBS_ATTR_TYPE_PTR_IN,
UVERBS_ATTR_STRUCT(
struct ib_uverbs_flow_action_esp_keymat_aes_gcm,
aes_key),
},
};
static const struct uverbs_attr_spec uverbs_flow_action_esp_replay[] = {
[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_NONE] = {
.type = UVERBS_ATTR_TYPE_PTR_IN,
UVERBS_ATTR_NO_DATA(),
},
[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_BMP] = {
.type = UVERBS_ATTR_TYPE_PTR_IN,
UVERBS_ATTR_STRUCT(struct ib_uverbs_flow_action_esp_replay_bmp,
size),
},
};
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_FLOW_ACTION_ESP_CREATE,
UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE,
UVERBS_OBJECT_FLOW_ACTION,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS,
UVERBS_ATTR_STRUCT(struct ib_uverbs_flow_action_esp,
hard_limit_pkts),
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ESN,
UVERBS_ATTR_TYPE(__u32),
UA_OPTIONAL),
UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT,
uverbs_flow_action_esp_keymat,
UA_MANDATORY),
UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY,
uverbs_flow_action_esp_replay,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(
UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP,
UVERBS_ATTR_TYPE(struct ib_uverbs_flow_action_esp_encap),
UA_OPTIONAL));
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY,
UVERBS_ATTR_IDR(UVERBS_ATTR_MODIFY_FLOW_ACTION_ESP_HANDLE,
UVERBS_OBJECT_FLOW_ACTION,
UVERBS_ACCESS_WRITE,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS,
UVERBS_ATTR_STRUCT(struct ib_uverbs_flow_action_esp,
hard_limit_pkts),
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ESN,
UVERBS_ATTR_TYPE(__u32),
UA_OPTIONAL),
UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT,
uverbs_flow_action_esp_keymat,
UA_OPTIONAL),
UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY,
uverbs_flow_action_esp_replay,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(
UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP,
UVERBS_ATTR_TYPE(struct ib_uverbs_flow_action_esp_encap),
UA_OPTIONAL));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_FLOW_ACTION_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_FLOW_ACTION_HANDLE,
UVERBS_OBJECT_FLOW_ACTION,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_FLOW_ACTION,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_flow_action),
&UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_ESP_CREATE),
&UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_DESTROY),
&UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY));
@@ -0,0 +1,149 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "uverbs.h"
#include <rdma/uverbs_std_types.h>
static int uverbs_free_mr(struct ib_uobject *uobject,
enum rdma_remove_reason why)
{
return ib_dereg_mr((struct ib_mr *)uobject->object);
}
static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)(
struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
{
struct ib_dm_mr_attr attr = {};
struct ib_uobject *uobj =
uverbs_attr_get_uobject(attrs, UVERBS_ATTR_REG_DM_MR_HANDLE);
struct ib_dm *dm =
uverbs_attr_get_obj(attrs, UVERBS_ATTR_REG_DM_MR_DM_HANDLE);
struct ib_pd *pd =
uverbs_attr_get_obj(attrs, UVERBS_ATTR_REG_DM_MR_PD_HANDLE);
struct ib_device *ib_dev = pd->device;
struct ib_mr *mr;
int ret;
if (!ib_dev->reg_dm_mr)
return -EOPNOTSUPP;
ret = uverbs_copy_from(&attr.offset, attrs, UVERBS_ATTR_REG_DM_MR_OFFSET);
if (ret)
return ret;
ret = uverbs_copy_from(&attr.length, attrs,
UVERBS_ATTR_REG_DM_MR_LENGTH);
if (ret)
return ret;
ret = uverbs_get_flags32(&attr.access_flags, attrs,
UVERBS_ATTR_REG_DM_MR_ACCESS_FLAGS,
IB_ACCESS_SUPPORTED);
if (ret)
return ret;
if (!(attr.access_flags & IB_ZERO_BASED))
return -EINVAL;
ret = ib_check_mr_access(attr.access_flags);
if (ret)
return ret;
if (attr.offset > dm->length || attr.length > dm->length ||
attr.length > dm->length - attr.offset)
return -EINVAL;
mr = pd->device->reg_dm_mr(pd, dm, &attr, attrs);
if (IS_ERR(mr))
return PTR_ERR(mr);
mr->device = pd->device;
mr->pd = pd;
mr->dm = dm;
mr->uobject = uobj;
atomic_inc(&pd->usecnt);
atomic_inc(&dm->usecnt);
uobj->object = mr;
ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DM_MR_RESP_LKEY, &mr->lkey,
sizeof(mr->lkey));
if (ret)
goto err_dereg;
ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DM_MR_RESP_RKEY,
&mr->rkey, sizeof(mr->rkey));
if (ret)
goto err_dereg;
return 0;
err_dereg:
ib_dereg_mr(mr);
return ret;
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_DM_MR_REG,
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_HANDLE,
UVERBS_OBJECT_MR,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_DM_MR_OFFSET,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_DM_MR_LENGTH,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_PD_HANDLE,
UVERBS_OBJECT_PD,
UVERBS_ACCESS_READ,
UA_MANDATORY),
UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_REG_DM_MR_ACCESS_FLAGS,
enum ib_access_flags),
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_DM_HANDLE,
UVERBS_OBJECT_DM,
UVERBS_ACCESS_READ,
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_DM_MR_RESP_LKEY,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_DM_MR_RESP_RKEY,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_MR,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_mr),
&UVERBS_METHOD(UVERBS_METHOD_DM_MR_REG));
+349
View File
@@ -0,0 +1,349 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*/
#include <rdma/uverbs_ioctl.h>
#include <rdma/rdma_user_ioctl.h>
#include <linux/bitops.h>
#include "rdma_core.h"
#include "uverbs.h"
static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)
{
void *elm;
int rc;
if (key == UVERBS_API_KEY_ERR)
return ERR_PTR(-EOVERFLOW);
elm = kzalloc(alloc_size, GFP_KERNEL);
if (!elm)
return ERR_PTR(-ENOMEM);
rc = radix_tree_insert(&uapi->radix, key, elm);
if (rc) {
kfree(elm);
return ERR_PTR(rc);
}
return elm;
}
static int uapi_merge_method(struct uverbs_api *uapi,
struct uverbs_api_object *obj_elm, u32 obj_key,
const struct uverbs_method_def *method,
bool is_driver)
{
u32 method_key = obj_key | uapi_key_ioctl_method(method->id);
struct uverbs_api_ioctl_method *method_elm;
unsigned int i;
if (!method->attrs)
return 0;
method_elm = uapi_add_elm(uapi, method_key, sizeof(*method_elm));
if (IS_ERR(method_elm)) {
if (method_elm != ERR_PTR(-EEXIST))
return PTR_ERR(method_elm);
/*
* This occurs when a driver uses ADD_UVERBS_ATTRIBUTES_SIMPLE
*/
if (WARN_ON(method->handler))
return -EINVAL;
method_elm = radix_tree_lookup(&uapi->radix, method_key);
if (WARN_ON(!method_elm))
return -EINVAL;
} else {
WARN_ON(!method->handler);
rcu_assign_pointer(method_elm->handler, method->handler);
if (method->handler != uverbs_destroy_def_handler)
method_elm->driver_method = is_driver;
}
for (i = 0; i != method->num_attrs; i++) {
const struct uverbs_attr_def *attr = (*method->attrs)[i];
struct uverbs_api_attr *attr_slot;
if (!attr)
continue;
/*
* ENUM_IN contains the 'ids' pointer to the driver's .rodata,
* so if it is specified by a driver then it always makes this
* into a driver method.
*/
if (attr->attr.type == UVERBS_ATTR_TYPE_ENUM_IN)
method_elm->driver_method |= is_driver;
attr_slot =
uapi_add_elm(uapi, method_key | uapi_key_attr(attr->id),
sizeof(*attr_slot));
/* Attributes are not allowed to be modified by drivers */
if (IS_ERR(attr_slot))
return PTR_ERR(attr_slot);
attr_slot->spec = attr->attr;
}
return 0;
}
static int uapi_merge_tree(struct uverbs_api *uapi,
const struct uverbs_object_tree_def *tree,
bool is_driver)
{
unsigned int i, j;
int rc;
if (!tree->objects)
return 0;
for (i = 0; i != tree->num_objects; i++) {
const struct uverbs_object_def *obj = (*tree->objects)[i];
struct uverbs_api_object *obj_elm;
u32 obj_key;
if (!obj)
continue;
obj_key = uapi_key_obj(obj->id);
obj_elm = uapi_add_elm(uapi, obj_key, sizeof(*obj_elm));
if (IS_ERR(obj_elm)) {
if (obj_elm != ERR_PTR(-EEXIST))
return PTR_ERR(obj_elm);
/* This occurs when a driver uses ADD_UVERBS_METHODS */
if (WARN_ON(obj->type_attrs))
return -EINVAL;
obj_elm = radix_tree_lookup(&uapi->radix, obj_key);
if (WARN_ON(!obj_elm))
return -EINVAL;
} else {
obj_elm->type_attrs = obj->type_attrs;
if (obj->type_attrs) {
obj_elm->type_class =
obj->type_attrs->type_class;
/*
* Today drivers are only permitted to use
* idr_class types. They cannot use FD types
* because we currently have no way to revoke
* the fops pointer after device
* disassociation.
*/
if (WARN_ON(is_driver &&
obj->type_attrs->type_class !=
&uverbs_idr_class))
return -EINVAL;
}
}
if (!obj->methods)
continue;
for (j = 0; j != obj->num_methods; j++) {
const struct uverbs_method_def *method =
(*obj->methods)[j];
if (!method)
continue;
rc = uapi_merge_method(uapi, obj_elm, obj_key, method,
is_driver);
if (rc)
return rc;
}
}
return 0;
}
static int
uapi_finalize_ioctl_method(struct uverbs_api *uapi,
struct uverbs_api_ioctl_method *method_elm,
u32 method_key)
{
struct radix_tree_iter iter;
unsigned int num_attrs = 0;
unsigned int max_bkey = 0;
bool single_uobj = false;
void __rcu **slot;
method_elm->destroy_bkey = UVERBS_API_ATTR_BKEY_LEN;
radix_tree_for_each_slot (slot, &uapi->radix, &iter,
uapi_key_attrs_start(method_key)) {
struct uverbs_api_attr *elm =
rcu_dereference_protected(*slot, true);
u32 attr_key = iter.index & UVERBS_API_ATTR_KEY_MASK;
u32 attr_bkey = uapi_bkey_attr(attr_key);
u8 type = elm->spec.type;
if (uapi_key_attr_to_method(iter.index) !=
uapi_key_attr_to_method(method_key))
break;
if (elm->spec.mandatory)
__set_bit(attr_bkey, method_elm->attr_mandatory);
if (type == UVERBS_ATTR_TYPE_IDR ||
type == UVERBS_ATTR_TYPE_FD) {
u8 access = elm->spec.u.obj.access;
/*
* Verbs specs may only have one NEW/DESTROY, we don't
* have the infrastructure to abort multiple NEW's or
* cope with multiple DESTROY failure.
*/
if (access == UVERBS_ACCESS_NEW ||
access == UVERBS_ACCESS_DESTROY) {
if (WARN_ON(single_uobj))
return -EINVAL;
single_uobj = true;
if (WARN_ON(!elm->spec.mandatory))
return -EINVAL;
}
if (access == UVERBS_ACCESS_DESTROY)
method_elm->destroy_bkey = attr_bkey;
}
max_bkey = max(max_bkey, attr_bkey);
num_attrs++;
}
method_elm->key_bitmap_len = max_bkey + 1;
WARN_ON(method_elm->key_bitmap_len > UVERBS_API_ATTR_BKEY_LEN);
uapi_compute_bundle_size(method_elm, num_attrs);
return 0;
}
static int uapi_finalize(struct uverbs_api *uapi)
{
struct radix_tree_iter iter;
void __rcu **slot;
int rc;
radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
struct uverbs_api_ioctl_method *method_elm =
rcu_dereference_protected(*slot, true);
if (uapi_key_is_ioctl_method(iter.index)) {
rc = uapi_finalize_ioctl_method(uapi, method_elm,
iter.index);
if (rc)
return rc;
}
}
return 0;
}
void uverbs_destroy_api(struct uverbs_api *uapi)
{
struct radix_tree_iter iter;
void __rcu **slot;
if (!uapi)
return;
radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
kfree(rcu_dereference_protected(*slot, true));
radix_tree_iter_delete(&uapi->radix, &iter, slot);
}
kfree(uapi);
}
struct uverbs_api *uverbs_alloc_api(
const struct uverbs_object_tree_def *const *driver_specs,
enum rdma_driver_id driver_id)
{
struct uverbs_api *uapi;
int rc;
uapi = kzalloc(sizeof(*uapi), GFP_KERNEL);
if (!uapi)
return ERR_PTR(-ENOMEM);
INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);
uapi->driver_id = driver_id;
rc = uapi_merge_tree(uapi, uverbs_default_get_objects(), false);
if (rc)
goto err;
for (; driver_specs && *driver_specs; driver_specs++) {
rc = uapi_merge_tree(uapi, *driver_specs, true);
if (rc)
goto err;
}
rc = uapi_finalize(uapi);
if (rc)
goto err;
return uapi;
err:
if (rc != -ENOMEM)
pr_err("Setup of uverbs_api failed, kernel parsing tree description is not valid (%d)??\n",
rc);
uverbs_destroy_api(uapi);
return ERR_PTR(rc);
}
/*
* The pre version is done before destroying the HW objects, it only blocks
* off method access. All methods that require the ib_dev or the module data
* must test one of these assignments prior to continuing.
*/
void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev)
{
struct uverbs_api *uapi = uverbs_dev->uapi;
struct radix_tree_iter iter;
void __rcu **slot;
rcu_assign_pointer(uverbs_dev->ib_dev, NULL);
radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
if (uapi_key_is_ioctl_method(iter.index)) {
struct uverbs_api_ioctl_method *method_elm =
rcu_dereference_protected(*slot, true);
if (method_elm->driver_method)
rcu_assign_pointer(method_elm->handler, NULL);
}
}
synchronize_srcu(&uverbs_dev->disassociate_srcu);
}
/*
* Called when a driver disassociates from the ib_uverbs_device. The
* assumption is that the driver module will unload after. Replace everything
* related to the driver with NULL as a safety measure.
*/
void uverbs_disassociate_api(struct uverbs_api *uapi)
{
struct radix_tree_iter iter;
void __rcu **slot;
radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
if (uapi_key_is_object(iter.index)) {
struct uverbs_api_object *object_elm =
rcu_dereference_protected(*slot, true);
/*
* Some type_attrs are in the driver module. We don't
* bother to keep track of which since there should be
* no use of this after disassociate.
*/
object_elm->type_attrs = NULL;
} else if (uapi_key_is_attr(iter.index)) {
struct uverbs_api_attr *elm =
rcu_dereference_protected(*slot, true);
if (elm->spec.type == UVERBS_ATTR_TYPE_ENUM_IN)
elm->spec.u2.enum_def.ids = NULL;
}
}
}
File diff suppressed because it is too large. Load diff