Skip to content

Commit

Permalink
Adding Doca Communication Channel API
Browse files Browse the repository at this point in the history
  • Loading branch information
@EldarShalev
Eldar Shalev authored and EldarShalev committed Dec 7, 2023
1 parent 8c23bf7 commit 3da6652
Show file tree
Hide file tree
Showing 12 changed files with 1,268 additions and 81 deletions.
41 changes: 35 additions & 6 deletions configure.ac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ AC_CANONICAL_SYSTEM

TARGETDIR="unknown"
case "$host" in

i?86-*-*) TARGET=X86; TARGETDIR=x86;;
ia64*-*-*) TARGET=IA64; TARGETDIR=ia64;;
powerpc*-*-linux* | powerpc-*-sysv*) TARGET=POWERPC; TARGETDIR=powerpc;;
Expand All @@ -26,14 +26,14 @@ case "$host" in
aarch64-*-*) TARGET=AARCH64; TARGETDIR=aarch64;;
s390*-*-*) TARGET=S390; TARGETDIR=s390;;
esac

AC_SUBST(AM_RUNTESTFLAGS)
AC_SUBST(AM_LTLDFLAGS)

if test $TARGETDIR = unknown; then
AC_MSG_ERROR(["it has not been ported to $host."])
fi

AM_CONDITIONAL(X86, test x$TARGET = xX86)
AM_CONDITIONAL(IA64, test x$TARGET = xIA64)
AM_CONDITIONAL(POWERPC, test x$TARGET = xPOWERPC)
Expand Down Expand Up @@ -100,6 +100,34 @@ AC_MSG_CHECKING(
[for vma extra api])
AC_MSG_RESULT([${have_vma_api}])

##########################################################################
# check DOCA communication channel API
#
AC_ARG_ENABLE(
[doca-communication-channel-api],
AC_HELP_STRING([--enable-doca-communication-channel-api],
[SOCKPERF: enable DOCA communication channel extra api support (default=no)]),
[have_doca_comm_channel_api=$enableval],
[have_doca_comm_channel_api=no])
AS_IF([test "x${have_doca_comm_channel_api}" == "xyes"],
if test "$have_doca_comm_channel_api" = "yes"
then
have_doca_comm_channel_api=/opt/mellanox/doca
fi

CPPFLAGS="$CPPFLAGS -I$have_doca_comm_channel_api/include -I$have_doca_comm_channel_api/lib -I$have_doca_comm_channel_api/samples"
LIBS="$LIBS -ldoca_comm_channel -ldoca_common"

[AC_CHECK_HEADERS([$have_doca_comm_channel_api/include/doca_comm_channel.h $have_doca_comm_channel_api/include/doca_dev.h $have_doca_comm_channel_api/samples/common.h],
[AC_DEFINE([USING_DOCA_COMM_CHANNEL_API],[1],[[Enable using DOCA communication channel extra API]])
],
[AC_MSG_ERROR([doca_comm_channel.h file not found])]
[have_doca_comm_channel_api=no])])
AC_MSG_CHECKING(
[for doca communication channel extra api])
AC_MSG_RESULT([${have_doca_comm_channel_api}])


##########################################################################
# check XLIO extra API
#
Expand Down Expand Up @@ -141,7 +169,7 @@ AM_CONDITIONAL(DOC, test "x$have_doc" = "xyes")


##########################
# Enable tests
# Enable tests
#
SP_ARG_ENABLE_BOOL(
[test],
Expand All @@ -151,7 +179,7 @@ AM_CONDITIONAL(TEST, test "x$have_test" = "xyes")


##########################
# Enable tools
# Enable tools
#
SP_ARG_ENABLE_BOOL(
[tool],
Expand Down Expand Up @@ -233,6 +261,7 @@ AC_MSG_RESULT([
test: ${have_test}
tool: ${have_tool}
vma_api: ${have_vma_api}
doca_api: ${have_doca_comm_channel_api}
xlio_api: ${have_xlio_api}
debug: ${have_debug}
])
13 changes: 13 additions & 0 deletions src/client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -885,7 +885,20 @@ int Client<IoType, SwitchCycleDuration, PongModeCare>::initBeforeLoop() {
if (!(data && (data->active_fd_list))) continue;

const sockaddr_store_t *p_client_bind_addr = &g_pApp->m_const_params.client_bind_info;
#if defined(USING_DOCA_COMM_CHANNEL_API)
if (s_user_params.doca_comm_channel) {
// Waiting for connection
struct cc_ctx_client *ctx_client = (struct cc_ctx_client *)data->doca_cc_ctx;
while (ctx_client->state != CC_CONNECTED) {
doca_pe_progress(s_user_params.pe);
}
log_dbg("[fd=%d] Client connected successfully", ifd);
}
// Avoid Client binding in Com Channel mode
if (p_client_bind_addr->addr.sa_family != AF_UNSPEC && !s_user_params.doca_comm_channel) {
#else
if (p_client_bind_addr->addr.sa_family != AF_UNSPEC) {
#endif //USING_DOCA_COMM_CHANNEL_API
socklen_t client_bind_addr_len = g_pApp->m_const_params.client_bind_info_len;
std::string hostport = sockaddr_to_hostport(p_client_bind_addr);
#if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__)
Expand Down
6 changes: 5 additions & 1 deletion src/common.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,11 @@ std::string sockaddr_to_hostport(const struct sockaddr *addr)
if (addr->sa_family == AF_INET6) {
return "[" + std::string(hbuf) + "]:" + std::string(pbuf);
} else if (addr->sa_family == AF_UNIX) {
return std::string(addr->sa_data);
#if defined(USING_DOCA_COMM_CHANNEL_API)
if (s_user_params.doca_comm_channel)
return std::string(pbuf) + " [DOCA]";
#endif
return std::string(pbuf) + " [UNIX]";
} else {
return std::string(hbuf) + ":" + std::string(pbuf);
}
Expand Down
50 changes: 50 additions & 0 deletions src/common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,56 @@ static inline int msg_sendto(int fd, uint8_t *buf, int nbytes,
ret = tls_write(g_fds_array[fd]->tls_handle, buf, nbytes);
} else
#endif /* DEFINED_TLS */
#if defined(USING_DOCA_COMM_CHANNEL_API)
if (s_user_params.doca_comm_channel) {
doca_error_t doca_error;
struct doca_cc_send_task *task;
struct doca_task *task_obj;
struct timespec ts = {
.tv_sec = 0,
.tv_nsec = NANOS_10_X_1000,
};
struct cc_ctx *ctx = g_fds_array[fd]->doca_cc_ctx;
do {
if (s_user_params.mode == MODE_SERVER) {
struct cc_ctx_server *ctx_server = (struct cc_ctx_server*)ctx;
doca_error = doca_cc_server_send_task_alloc_init(ctx_server->server, ctx_server->ctx.connection, buf,
nbytes, &task);
} else { // MODE_CLIENT
struct cc_ctx_client *ctx_client = (struct cc_ctx_client *)ctx;
doca_error = doca_cc_client_send_task_alloc_init(ctx_client->client, ctx_client->ctx.connection, buf,
nbytes, &task);
}
if (doca_error == DOCA_ERROR_NO_MEMORY) {
// Queue is full of tasks, need to free tasks with completion callback
doca_pe_progress(s_user_params.pe);
}
} while (doca_error == DOCA_ERROR_NO_MEMORY);

if (doca_error != DOCA_SUCCESS) {
log_err("Doca task_alloc_init failed");
return RET_SOCKET_SHUTDOWN;
}

task_obj = doca_cc_send_task_as_task(task);
do {
doca_error = doca_task_submit(task_obj);
if (doca_error == DOCA_ERROR_AGAIN) {
// Queue is full of tasks, need to free tasks with completion callback
doca_pe_progress(s_user_params.pe);
}
} while (doca_error == DOCA_ERROR_AGAIN);

if (doca_error != DOCA_SUCCESS) {
log_err("Doca doca_task_submit failed");
return RET_SOCKET_SHUTDOWN;
}

// Additional call for better performance- release pressure on send queue
doca_pe_progress(s_user_params.pe);
ret = nbytes;
} else
#endif /* USING_DOCA_COMM_CHANNEL_API */
{
ret = sendto(fd, buf, nbytes, flags, sendto_addr, addrlen);
}
Expand Down
22 changes: 21 additions & 1 deletion src/defs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,11 @@ typedef unsigned short int sa_family_t;
#endif // USING_XLIO_EXTRA_API
#endif // !defined(__windows__) && !defined(__FreeBSD__) && !defined(__APPLE__)

#ifdef USING_DOCA_COMM_CHANNEL_API
#include "doca_cc_helper.h"
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
#endif /* USING_DOCA_COMM_CHANNEL_API */

#define MIN_PAYLOAD_SIZE (MsgHeader::EFFECTIVE_SIZE)
extern int MAX_PAYLOAD_SIZE;
extern int max_fds_num;
Expand Down Expand Up @@ -291,8 +296,14 @@ enum {
OPT_LOAD_XLIO, // 47
OPT_TCP_NB_CONN_TIMEOUT_MS, // 48
#if defined(DEFINED_TLS)
OPT_TLS
OPT_TLS,
#endif /* DEFINED_TLS */
#if defined(USING_DOCA_COMM_CHANNEL_API)
OPT_DOCA,
OPT_PCI,
OPT_PCI_REP
#endif /* USING_DOCA_COMM_CHANNEL_API */

};

static const char *const round_trip_str[] = { "latency", "rtt" };
Expand Down Expand Up @@ -571,6 +582,9 @@ struct fds_data {
#if defined(DEFINED_TLS)
void *tls_handle = nullptr;
#endif /* DEFINED_TLS */
#if defined(USING_DOCA_COMM_CHANNEL_API)
struct cc_ctx *doca_cc_ctx = nullptr;
#endif /* USING_DOCA_COMM_CHANNEL_API */

fds_data()
{
Expand Down Expand Up @@ -804,6 +818,12 @@ struct user_params_t {
#if defined(DEFINED_TLS)
bool tls = false;
#endif /* DEFINED_TLS */
#if defined(USING_DOCA_COMM_CHANNEL_API)
bool doca_comm_channel = false; /* Flag to indicate using Com Channel*/
char cc_dev_pci_addr[PCI_ADDR_LEN]; /* Comm Channel DOCA device PCI address */
char cc_dev_rep_pci_addr[PCI_ADDR_LEN]; /* Comm Channel DOCA device representor PCI address */
struct doca_pe *pe = nullptr; /* Progress engine for doca, one per thread*/
#endif /* USING_DOCA_COMM_CHANNEL_API */

user_params_t() {
memset(&client_bind_info, 0, sizeof(client_bind_info));
Expand Down
Loading

0 comments on commit 3da6652

Please sign in to comment.