LLVM Discussion Forums

Help with adding CUDA files to a LLVM scheduler project

Hello, I am new here so I felt that the beginner forum was appropriate. Please direct me to correct forum if it is not.

I am working on a research project where we are working on an exhaustive non-greedy scheduler for LLVM. I am using CUDA to add GPU acceleration to our scheduler, but I am running into issues when trying to add the new CUDA files to the projects CMakeLists.txt. As of now the build.ninja file generated by cmake is not passing the CXX compiler flags through the -Xcompiler flag to nvcc, and is also not passing the -fPIC flag that is needed for the scheduler to build properly. This causes a nvcc: fatal error since the flags being passed are not nvcc flags.
I am able to get them to compile by manually modifying the flags in the ninja.build file, but this is becoming quite time consuming and is obviously terrible for others who will have to work on this.

So currently for the .cu files build.ninja generates:
FLAGS = -g -fno-exceptions -fno-rtti
Which causes an error.
I would like it to instead generate:
FLAGS = -Xcompiler "-fPIC -g -fno-exceptions -fno-rtti"
Which build correctly.

I am unsure of how to accomplish this due to the complicated nature of the LLVM build and my inexperience with cmake. I am not sure where flags are set, as they are not specified in our projects CMakeLists.

Based on what I read in the cmake documentation I came up with the following command to accomplish this:
SET(CMAKE_CUDA_FLAGS "-Xcompiler \\\"-fPIC ${CMAKE_CUDA_FLAGS}\\\"")
But this does not work correctly and generates the following:
FLAGS = -Xcompiler \"-fPIC \" -g -fno-exceptions -fno-rtti
which does not work properly.

I am unsure why this happens, though it seems like the flags are added after my command.
I would really appreciate some help with this problem.

For more information, here is the CMakeLists.txt file that is located in llvm-project/llvm/projects/MyProject, where I specified CUDA as a language, added the .cu files, and tried to add the cuda flag command:

CMakeLists.txt:

cmake_minimum_required(VERSION 3.4.3)
project(OptSched LANGUAGES CXX CUDA)

include(CTest)

option(OPTSCHED_INCLUDE_TESTS "Generate build targets for the OptSched unit tests." ON)

# Exit if attempting to build as a standalone project.
IF(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
  message(FATAL_ERROR "You cannot build opt-sched as a standalone project. Place opt-sched in the llvm tree under the \"projects\" directory.")
ENDIF()

IF(NOT CMAKE_BUILD_TYPE)
  message(FATAL_ERROR "Must set cmake build type.")
ENDIF()

IF(${PACKAGE_VERSION} VERSION_LESS "7.0")
  add_definitions("-DLLVM_DEBUG=DEBUG")
ENDIF()

# Add OptSched debug defines eg: SET(OPT_SCHED_DEBUG_FLAGS "-DIS_DEBUG_DEFS_AND_USES -DIS_DEBUG_DEF_USE_COUNT")
SET(OPT_SCHED_DEBUG_FLAGS "")

# If asserts are enabled opt-sched must be built with "IS_DEBUG".
SET(CMAKE_CXX_FLAGS_DEBUG " -DIS_DEBUG")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPT_SCHED_DEBUG_FLAGS}")

#Add -fPIC flag and enclose all flags in -Xcompiler
SET(CMAKE_CUDA_FLAGS "-Xcompiler \\\"-fPIC ${CMAKE_CUDA_FLAGS}\\\"")

IF(CMAKE_BUILD_TYPE EQUAL "DEBUG")
  set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
ENDIF()

set(OPTSCHED_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
set(OPTSCHED_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})

# OptSched sources
SET(OPT_SCHED_SRCS Scheduler/aco.cpp
  Scheduler/bb_spill.cpp
  Scheduler/buffers.cpp
  Scheduler/config.cpp
  Scheduler/data_dep.cpp
  Scheduler/enumerator.cpp
  Scheduler/cuda_gen_sched.cu
  Scheduler/graph.cpp
  Scheduler/graph_trans.cpp
  Scheduler/hist_table.cpp
  Scheduler/cuda_list_sched.cu
  Scheduler/logger.cpp
  Scheduler/reg_alloc.cpp
  Scheduler/utilities.cpp
  Scheduler/machine_model.cpp
  Scheduler/random.cpp
  Scheduler/cuda_ready_list.cu
  Scheduler/cuda_register.cu
  Scheduler/relaxed_sched.cpp
  Scheduler/cuda_sched_basic_data.cu
  Scheduler/sched_region.cpp
  Scheduler/stats.cpp
  Wrapper/OptimizingScheduler.cpp
  Wrapper/OptSchedMachineWrapper.cpp
  Wrapper/OptSchedDDGWrapperBasic.cpp
  Wrapper/OptSchedGenericTarget.cpp)

SET(OPT_SCHED_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/include)
SET(OPT_SCHED_TARGET_DEPS "LLVMCodeGen")

function(check_if_AMDGPU_supported)
  IF(${PACKAGE_VERSION} VERSION_LESS "7.0")
    message(WARNING "OptSched requries LLVM version >= 7.0 to build the AMDGPU scheduler.")
    SET(AMDGPU_SUPPORTED False PARENT_SCOPE)

  ELSE()
    SET(AMDGPU_SUPPORTED True PARENT_SCOPE)

  ENDIF()
endfunction(check_if_AMDGPU_supported)

# Only build and include AMDGPU functionality if it is being targeted.
IF("AMDGPU" IN_LIST LLVM_TARGETS_TO_BUILD)
  check_if_AMDGPU_supported()
  IF(${AMDGPU_SUPPORTED})
    SET(OPT_SCHED_TARGET_DEPS ${OPT_SCHED_TARGET_DEPS} AMDGPUCommonTableGen)
    SET(OPT_SCHED_SRCS ${OPT_SCHED_SRCS} Wrapper/AMDGPU/GCNOptSched.cpp
                                          Wrapper/AMDGPU/OptSchedGCNTarget.cpp
                                          Wrapper/AMDGPU/OptSchedDDGWrapperGCN.cpp)
    SET(OPT_SCHED_INCLUDE_DIRS ${OPT_SCHED_INCLUDE_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/lib
                                ${LLVM_MAIN_SRC_DIR}/lib/Target/AMDGPU
                                ${CMAKE_CURRENT_BINARY_DIR}/../../lib/Target/AMDGPU)
  ENDIF()
ENDIF()

include_directories(${OPT_SCHED_INCLUDE_DIRS})
add_subdirectory(lib)

if(OPTSCHED_INCLUDE_TESTS)
  add_subdirectory(unittests)
  list(APPEND OPTSCHED_TEST_DEPS OptSchedUnitTests)
  list(APPEND OPTSCHED_TEST_PARAMS
    clang_unit_site_config=${CMAKE_CURRENT_BINARY_DIR}/test/Unit/lit.site.cfg
    )

  add_subdirectory(test)
endif()