diff options
Diffstat (limited to 'kiro-rdma.h')
| -rw-r--r-- | kiro-rdma.h | 230 |
1 files changed, 0 insertions, 230 deletions
diff --git a/kiro-rdma.h b/kiro-rdma.h deleted file mode 100644 index fa16fd1..0000000 --- a/kiro-rdma.h +++ /dev/null @@ -1,230 +0,0 @@ -/* Copyright (C) 2014 Timo Dritschler <timo.dritschler@kit.edu> - (Karlsruhe Institute of Technology) - - This library is free software; you can redistribute it and/or modify it - under the terms of the GNU Lesser General Public License as published by the - Free Software Foundation; either version 2.1 of the License, or (at your - option) any later version. - - This library is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more - details. - - You should have received a copy of the GNU Lesser General Public License along - with this library; if not, write to the Free Software Foundation, Inc., 51 - Franklin St, Fifth Floor, Boston, MA 02110, USA -*/ - -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> -#ifndef __KIRO_RDMA_H__ -#define __KIRO_RDMA_H__ - - -#include <rdma/rdma_cma.h> - - -struct kiro_connection_context { - - // Information and necessary structurs - uint32_t identifier; // Unique Identifier for this connection (Application Specific) - struct kiro_rdma_mem *cf_mr_recv; // Control-Flow Memory Region Receive - struct kiro_rdma_mem *cf_mr_send; // Control-Flow Memory Region Send - struct kiro_rdma_mem *rdma_mr; // Memory Region for RDMA Operations - - struct ibv_mr peer_mr; // RDMA Memory Region Information of the peer - - enum { - KIRO_IDLE, - KIRO_MRI_REQUESTED, // Memory Region Information Requested - KIRO_RDMA_ESTABLISHED, // MRI Exchange complete. RDMA is ready - KIRO_RDMA_ACTIVE // RDMA Operation is being performed - } rdma_state; - -}; - - -struct kiro_connection { - - uint32_t identifier; - struct rdma_cm_id *id; - struct kiro_connection *next; - -}; - - -struct kiro_ctrl_msg { - - enum { - KIRO_REQ_RDMA, // Requesting RDMA Access to/from the peer - KIRO_ACK_RDMA, // acknowledge RDMA Request and provide Memory Region Information - KIRO_REJ_RDMA // RDMA Request rejected :( (peer_mri will be invalid) - } msg_type; - - struct ibv_mr peer_mri; - -}; - - -struct kiro_rdma_mem { - - void *mem; // Pointer to the beginning of the memory block - struct ibv_mr *mr; // Memory Region associated with the memory - size_t size; // Size in Bytes of the memory block - -}; - - -static int kiro_attach_qp (struct rdma_cm_id *id) -{ - if(!id) - return -1; - - id->pd = ibv_alloc_pd(id->verbs); - id->send_cq_channel = ibv_create_comp_channel(id->verbs); - id->recv_cq_channel = id->send_cq_channel; //we use one shared completion channel - id->send_cq = ibv_create_cq(id->verbs, 1, id, id->send_cq_channel, 0); - id->recv_cq = id->send_cq; //we use one shared completion queue - - struct ibv_qp_init_attr qp_attr; - memset(&qp_attr, 0, sizeof(struct ibv_qp_init_attr)); - qp_attr.qp_context = (uintptr_t)id; - qp_attr.send_cq = id->send_cq; - qp_attr.recv_cq = id->recv_cq; - qp_attr.qp_type = IBV_QPT_RC; - qp_attr.cap.max_send_wr = 1; - qp_attr.cap.max_recv_wr = 1; - qp_attr.cap.max_send_sge = 1; - qp_attr.cap.max_recv_sge = 1; - - return rdma_create_qp(id, id->pd, &qp_attr); -} - - -static int kiro_register_rdma_memory (struct ibv_pd *pd, struct ibv_mr **mr, void *mem, size_t mem_size, int access) -{ - - if(mem_size == 0) - { - printf("Cant allocate memory of size '0'.\n"); - return -1; - } - - void *mem_handle = mem; - - if(!mem_handle) - mem_handle = malloc(mem_size); - - if(!mem_handle) - { - printf("Failed to allocate memory [Register Memory]."); - return -1; - } - - *mr = ibv_reg_mr(pd, mem_handle, mem_size, access); - if(!(*mr)) - { - // Memory Registration failed - printf("Failed to register memory region!\n"); - free(mem_handle); - return -1; - } - - return 0; -} - - -static struct kiro_rdma_mem* kiro_create_rdma_memory (struct ibv_pd *pd, size_t mem_size, int access) -{ - if(mem_size == 0) - { - printf("Cant allocate memory of size '0'.\n"); - return NULL; - } - - struct kiro_rdma_mem *krm = (struct kiro_rdma_mem *)calloc(1, sizeof(struct kiro_rdma_mem)); - if(!krm) - { - printf("Failed to create new KIRO RDMA Memory.\n"); - return NULL; - } - - if(kiro_register_rdma_memory(pd, &(krm->mr), krm->mem, mem_size, access)) - { - free(krm); - return NULL; - } - - if(!krm->mem) - krm->mem = krm->mr->addr; - - - return krm; - -} - - -static void kiro_destroy_rdma_memory (struct kiro_rdma_mem *krm) -{ - if(!krm) - return; - - if(krm->mr) - ibv_dereg_mr(krm->mr); - - if(krm->mem) - free(krm->mem); - - free(krm); - krm = NULL; -} - - -static void kiro_destroy_connection_context (struct kiro_connection_context **ctx) -{ - if(!ctx) - return; - - if(!(*ctx)) - return; - - if((*ctx)->cf_mr_recv) - kiro_destroy_rdma_memory((*ctx)->cf_mr_recv); - if((*ctx)->cf_mr_send) - kiro_destroy_rdma_memory((*ctx)->cf_mr_send); - - //The RDMA-Memory Region normally contains allocated memory from the USER that has - //just been 'registered' for RDMA. DON'T free it! Just deregister it. The user is - //responsible for freeing this memory. - if((*ctx)->rdma_mr) - { - if((*ctx)->rdma_mr->mr) - ibv_dereg_mr((*ctx)->rdma_mr->mr); - - free((*ctx)->rdma_mr); - (*ctx)->rdma_mr = NULL; - } - - free(*ctx); - *ctx = NULL; -} - - -static void kiro_destroy_connection (struct rdma_cm_id **conn) -{ - if(!(*conn)) - return; - - rdma_disconnect(*conn); - struct kiro_connection_context *ctx = (struct kiro_connection_context *)((*conn)->context); - if(ctx) - kiro_destroy_connection_context(&ctx); - - rdma_destroy_ep(*conn); - *conn = NULL; -} - - -#endif //__KIRO_RDMA_H__
\ No newline at end of file |