08ff32352d
ConnectX-3 devices can use either 64- or 32-byte completion queue entries (CQEs) and event queue entries (EQEs). Using 64-byte EQEs/CQEs performs better because each entry is aligned to a complete cacheline. This patch queries the HCA's capabilities, and if it supports 64-byte CQEs and EQES the driver will configure the HW to work in 64-byte mode. The 32-byte vs 64-byte mode is global per HCA and not per CQ or EQ. Since this mode is global, userspace (libmlx4) must be updated to work with the configured CQE size, and guests using SR-IOV virtual functions need to know both EQE and CQE size. In case one of the 64-byte CQE/EQE capabilities is activated, the patch makes sure that older guest drivers that use the QUERY_DEV_FUNC command (e.g as done in mlx4_core of Linux 3.3..3.6) will notice that they need an update to be able to work with the PPF. This is done by changing the returned pf_context_behaviour not to be zero any more. In case none of these capabilities is activated that value remains zero and older guest drivers can run OK. The SRIOV related flow is as follows 1. the PPF does the detection of the new capabilities using QUERY_DEV_CAP command. 2. the PPF activates the new capabilities using INIT_HCA. 3. the VF detects if the PPF activated the capabilities using QUERY_HCA, and if this is the case activates them for itself too. Note that the VF detects that it must be aware to the new PF behaviour using QUERY_FUNC_CAP. Steps 1 and 2 apply also for native mode. User space notification is done through a new field introduced in struct mlx4_ib_ucontext which holds device capabilities for which user space must take action. This changes the binary interface so the ABI towards libmlx4 exposed through uverbs is bumped from 3 to 4 but only when **needed** i.e. only when the driver does use 64-byte CQEs or future device capabilities which must be in sync by user space. This practice allows to work with unmodified libmlx4 on older devices (e.g A0, B0) which don't support 64-byte CQEs. In order to keep existing systems functional when they update to a newer kernel that contains these changes in VF and userspace ABI, a module parameter enable_64b_cqe_eqe must be set to enable 64-byte mode; the default is currently false. Signed-off-by: Eli Cohen <eli@mellanox.com> Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com> Signed-off-by: Roland Dreier <roland@purestorage.com>
108 lines
2.8 KiB
C
108 lines
2.8 KiB
C
/*
|
|
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
|
|
* Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
|
|
*
|
|
* This software is available to you under a choice of one of two
|
|
* licenses. You may choose to be licensed under the terms of the GNU
|
|
* General Public License (GPL) Version 2, available from the file
|
|
* COPYING in the main directory of this source tree, or the
|
|
* OpenIB.org BSD license below:
|
|
*
|
|
* Redistribution and use in source and binary forms, with or
|
|
* without modification, are permitted provided that the following
|
|
* conditions are met:
|
|
*
|
|
* - Redistributions of source code must retain the above
|
|
* copyright notice, this list of conditions and the following
|
|
* disclaimer.
|
|
*
|
|
* - Redistributions in binary form must reproduce the above
|
|
* copyright notice, this list of conditions and the following
|
|
* disclaimer in the documentation and/or other materials
|
|
* provided with the distribution.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
|
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
|
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
* SOFTWARE.
|
|
*/
|
|
|
|
#ifndef MLX4_IB_USER_H
|
|
#define MLX4_IB_USER_H
|
|
|
|
#include <linux/types.h>
|
|
|
|
/*
|
|
* Increment this value if any changes that break userspace ABI
|
|
* compatibility are made.
|
|
*/
|
|
|
|
#define MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION 3
|
|
#define MLX4_IB_UVERBS_ABI_VERSION 4
|
|
|
|
/*
|
|
* Make sure that all structs defined in this file remain laid out so
|
|
* that they pack the same way on 32-bit and 64-bit architectures (to
|
|
* avoid incompatibility between 32-bit userspace and 64-bit kernels).
|
|
* In particular do not use pointer types -- pass pointers in __u64
|
|
* instead.
|
|
*/
|
|
|
|
struct mlx4_ib_alloc_ucontext_resp_v3 {
|
|
__u32 qp_tab_size;
|
|
__u16 bf_reg_size;
|
|
__u16 bf_regs_per_page;
|
|
};
|
|
|
|
struct mlx4_ib_alloc_ucontext_resp {
|
|
__u32 dev_caps;
|
|
__u32 qp_tab_size;
|
|
__u16 bf_reg_size;
|
|
__u16 bf_regs_per_page;
|
|
__u32 cqe_size;
|
|
};
|
|
|
|
struct mlx4_ib_alloc_pd_resp {
|
|
__u32 pdn;
|
|
__u32 reserved;
|
|
};
|
|
|
|
struct mlx4_ib_create_cq {
|
|
__u64 buf_addr;
|
|
__u64 db_addr;
|
|
};
|
|
|
|
struct mlx4_ib_create_cq_resp {
|
|
__u32 cqn;
|
|
__u32 reserved;
|
|
};
|
|
|
|
struct mlx4_ib_resize_cq {
|
|
__u64 buf_addr;
|
|
};
|
|
|
|
struct mlx4_ib_create_srq {
|
|
__u64 buf_addr;
|
|
__u64 db_addr;
|
|
};
|
|
|
|
struct mlx4_ib_create_srq_resp {
|
|
__u32 srqn;
|
|
__u32 reserved;
|
|
};
|
|
|
|
struct mlx4_ib_create_qp {
|
|
__u64 buf_addr;
|
|
__u64 db_addr;
|
|
__u8 log_sq_bb_count;
|
|
__u8 log_sq_stride;
|
|
__u8 sq_no_prefetch;
|
|
__u8 reserved[5];
|
|
};
|
|
|
|
#endif /* MLX4_IB_USER_H */
|