diff --git a/CMakeLists.txt b/CMakeLists.txt index 90edd0d1fc2864cba4611460ca7166b61c0274ad..2d9ef88350513bb29be45f7fe5e6ec48821b85f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ pack_resources(TARGET "${PROJECT_NAME}" set_target_properties("${PROJECT_NAME}" PROPERTIES - MACOSX_BUNDLE_INFO_PLIST "${CMAKE_CURRENT_SOURCE_DIR}/Samples/Info.plist.in" + #MACOSX_BUNDLE_INFO_PLIST "${CMAKE_CURRENT_SOURCE_DIR}/Samples/Info.plist.in" XCODE_ATTRIBUTE_BUNDLE_IDENTIFIER "com.ravbug.${APPNAME}" # with templated plist we can set this XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER "com.ravbug.${APPNAME}" XCODE_ATTRIBUTE_CURRENTYEAR "${CURRENTYEAR}" @@ -46,4 +46,8 @@ set_target_properties("${PROJECT_NAME}" VS_WINDOWS_TARGET_PLATFORM_MIN_VERSION "10.0.19041.0" ) -target_link_libraries("${PROJECT_NAME}" PUBLIC RavEngine "ibverbs") \ No newline at end of file +target_link_libraries("${PROJECT_NAME}" PUBLIC RavEngine) + +if (LINUX) +target_link_libraries("${PROJECT_NAME}" PUBLIC "ibverbs") +endif() diff --git a/README.md b/README.md index f5e6f77bb03814240edc4e47dce72d2e97609256..3db64c99d8ce704fa5bba25331584dfdfa05b3a2 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,16 @@ On GreatLakes, create two servers on two different terminals, by running the fol salloc --account=cse589s011f24_class --nodes=1 --ntasks-per-node=1 --mem-per-cpu=1GB --cpus-per-task=1 ``` -To tell the server which instance it is (A or B), use the `SERVER` environment variable. If the variable is not set, it will default to Server A. +To tell the server which instance it is (A or B), use the `SERVER` environment variable. If the variable is not set, it will default to Server A. ```sh # on GreatLakes SERVER=B ./RDMAGame ``` -This property can also be set inside Visual Studio via Properties->Debugging->Environment. +To disable RDMA entirely and run a single server only, set the `NORDMA` environment variable to 1. On non-Linux platforms, RDMA is force-disabled automatically. +```sh +# on GreatLakes +NORDMA=1 ./RDMAGame +``` +These properties can also be set inside Visual Studio via Properties->Debugging->Environment. -After running the application, it should tell you the server's RDMA local ID and queue pair number, and prompt you for the other server's local ID and queue pair number. Input the proper information into both servers, and the application should work. \ No newline at end of file +After running the application, it should tell you the server's RDMA local ID and queue pair number, and prompt you for the other server's local ID and queue pair number. Input the proper information into both servers, and the application should work. diff --git a/src/App.cpp b/src/App.cpp index 52c89530a1c4c8a02cff36ed11c48a87d0056e10..392d88cf89557a8617d218a45516f3f27d9c8efb 100644 --- a/src/App.cpp +++ b/src/App.cpp @@ -59,7 +59,23 @@ struct NetApp : public RavEngine::App { } } + const char* rdmaInactiveReason = nullptr; Debug::Log("I am server {}", isServerA? "A" :"B"); + + +#if RDMA_ENABLED + // is RDMA forced off? + auto rdmaEnv = std::getenv("NORDMA"); + if (rdmaEnv){ + rdmaActive = false; + rdmaInactiveReason = "Disabled via environment variable"; + } +#else + rdmaInactiveReason = "Current OS does not support RDMA"; + rdmaActive = false; +#endif + + Debug::Log("RDMA is {}, reason = {}", rdmaActive ? "active" : "inactive", rdmaInactiveReason); } // otherwise we will launch as a client else { diff --git a/src/Level.cpp b/src/Level.cpp index 947e36614cf6070fc677b1ebc6769545ce9fa7d1..ef0d6f0616c486776ab7aa33d43e469a61c7ed9b 100644 --- a/src/Level.cpp +++ b/src/Level.cpp @@ -98,13 +98,15 @@ void Level::SetupServer() // replicate entities to connected clients EmplaceTimedSystem<SyncNetTransforms>(std::chrono::milliseconds(100)); +#if RDMA_ENABLED EmplaceSystem<RDMASendSystem>(); EmplaceSystem<RDMARecieveSystem>(); CreateDependency<RDMASendSystem, MoveEntities>(); CreateDependency<RDMARecieveSystem, MoveEntities>(); CreateDependency<RDMARecieveSystem, RDMASendSystem>(); // these don't have a dependency but it'll make debugging easier - +#endif + #if !RVE_SERVER auto guientity = Instantiate<Entity>(); auto& guic = guientity.EmplaceComponent<GUIComponent>(); @@ -118,7 +120,11 @@ void Level::SetupServer() im->BindAxis("MouseY", gh, &GUIComponent::MouseY, CID::ANY, 0); im->BindAnyAction(gh->GetData()); #endif - RDMAInit(); +#if RDMA_ENABLED + if (rdmaActive){ + RDMAInit(); + } +#endif } void Level::SetupClient() @@ -142,21 +148,33 @@ void Level::SetupClient() void Level::PostTick(float) { if (GetApp()->networkManager.IsServer()) { - RDMAFinalize(); +#if RDMA_ENABLED + if (rdmaActive){ + RDMAFinalize(); + } +#endif } } void Level::PreTick(float) { if (GetApp()->networkManager.IsServer()) { - RDMABegin(); +#if RDMA_ENABLED + if (rdmaActive){ + RDMABegin(); + } +#endif } } Level::~Level() { if (GetApp()->networkManager.IsServer()) { - RDMAShutdown(); +#if RDMA_ENABLED + if (rdmaActive){ + RDMAShutdown(); + } +#endif } } diff --git a/src/rdma.cpp b/src/rdma.cpp index 8cd2d033e90d1152f4fbe20df450ceadb819f3ee..15b2b672ff1392cc21b5ac0cf25d753c0b19a752 100644 --- a/src/rdma.cpp +++ b/src/rdma.cpp @@ -1,14 +1,25 @@ #include "rdma.hpp" +#include "Level.hpp" + +STATIC(isServerA) = true; // default to server A +STATIC(rdmaActive) = true; +bool isLocalToThisInstance(uint32_t pathDataID) { + if (rdmaActive){ + return (isServerA && pathDataID < (num_entities / 2)) || (!isServerA && pathDataID >= (num_entities / 2)); + } + else{ + return true; // if RDMA is inactive, then the current server owns ALL objects + } +} + +#if RDMA_ENABLED #include <RavEngine/Utilities.hpp> #include <RavEngine/Transform.hpp> -#include "Level.hpp" #include <RavEngine/Debug.hpp> -STATIC(isServerA) = true; // default to server A - STATIC(rdma_initialized) = false; -STATIC(dest_id) = 0; +STATIC(dest_id) = 0; STATIC(dest_qp) = 0; STATIC(context) = nullptr; //Protection Domain @@ -29,9 +40,6 @@ STATIC(memory_region_A) = nullptr; STATIC(memory_region_B) = nullptr; -bool isLocalToThisInstance(uint32_t pathDataID) { - return (isServerA && pathDataID < (num_entities / 2)) || (!isServerA && pathDataID >= (num_entities / 2)); -} //this might cause race conditions when rdma communication happens during read/write but i dont think thatll happen void RDMASendSystem::operator()(const PathData& pathData, const RavEngine::Transform& transform) const @@ -402,3 +410,4 @@ bool pollCompletion(struct ibv_cq* cq) { printf("Poll failed with status %s (work request ID: %llu)\n", ibv_wc_status_str(wc.status), wc.wr_id); return false; } +#endif diff --git a/src/rdma.hpp b/src/rdma.hpp index d209e78eddfa9980e51b27b187831e239ca1cd5d..74aaa3bfcf7786f0f680264a0ff429116eda7678 100644 --- a/src/rdma.hpp +++ b/src/rdma.hpp @@ -1,5 +1,22 @@ #pragma once +#include <cstdint> #include "NetEntity.hpp" + +extern bool isServerA; // evil global but I don't care +extern bool rdmaActive; +/** +* Returns true if the current server instance is responsible for simulating this object +* @param pathDataID the ID of the object +*/ +bool isLocalToThisInstance(uint32_t pathDataID); + +#if __linux__ +#define RDMA_ENABLED 1 +#else +#define RDMA_ENABLED 0 +#endif + +#if RDMA_ENABLED #include <infiniband/verbs.h> #include <iostream> @@ -7,7 +24,6 @@ #include <unordered_map> #include <vector> -extern bool isServerA; // evil global but I don't care extern bool rdma_initialized; extern uint16_t dest_id; // evil globals but i also don't care extern uint32_t dest_qp; @@ -52,12 +68,6 @@ struct RDMARecieveSystem { }; -/** -* Returns true if the current server instance is responsible for simulating this object -* @param pathDataID the ID of the object -*/ -bool isLocalToThisInstance(uint32_t pathDataID); - //RDMA boilerplate whee struct ibv_context* createContext(const std::string& device_name); @@ -70,4 +80,6 @@ bool changeQueuePairStateToRTS(struct ibv_qp* queue_pair); struct ibv_mr* registerMemoryRegion(struct ibv_pd* pd, void* buffer, size_t size); bool postReceiveRequest(const std::vector<struct ibv_mr*>& sge, struct ibv_qp* queue_pair_); bool postSendRequest(const std::vector<struct ibv_mr*>& sge, struct ibv_qp* queue_pair_); -bool pollCompletion(struct ibv_cq* cq); \ No newline at end of file +bool pollCompletion(struct ibv_cq* cq); + +#endif