[07/15] clover/spirv: Add functions for parsing arguments, linking programs, etc.

Submitted by Karol Herbst on May 11, 2019, 2:07 p.m.

Details

Message ID 20190511140712.27095-8-kherbst@redhat.com
State New
Headers show
Series "Clover: support CL through SPIR-V" ( rev: 1 ) in Mesa

Not browsing as part of any series.

Commit Message

Karol Herbst May 11, 2019, 2:07 p.m.
From: Pierre Moreau <pierre.morrow@free.fr>

v2 (Karol Herbst):
  silence warnings about unhandled enum values
---
 .../clover/spirv/invocation.cpp               | 598 ++++++++++++++++++
 .../clover/spirv/invocation.hpp               |  12 +
 2 files changed, 610 insertions(+)

Patch hide | download patch | download mbox

diff --git a/src/gallium/state_trackers/clover/spirv/invocation.cpp b/src/gallium/state_trackers/clover/spirv/invocation.cpp
index b874f2f061c..62886e77495 100644
--- a/src/gallium/state_trackers/clover/spirv/invocation.cpp
+++ b/src/gallium/state_trackers/clover/spirv/invocation.cpp
@@ -22,10 +22,24 @@ 
 
 #include "invocation.hpp"
 
+#include <stack>
+#include <tuple>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
 #ifdef CLOVER_ALLOW_SPIRV
 #include <spirv-tools/libspirv.hpp>
+#include <spirv-tools/linker.hpp>
 #endif
 
+#include "core/error.hpp"
+#include "core/platform.hpp"
+#include "invocation.hpp"
+#include "llvm/util.hpp"
+#include "pipe/p_state.h"
+#include "util/algorithm.hpp"
+#include "util/functional.hpp"
 #include "util/u_math.h"
 
 #include "compiler/spirv/spirv.h"
@@ -34,6 +48,472 @@  using namespace clover;
 
 namespace {
 
+   template<typename T>
+   T get(const char *source, size_t index) {
+      const uint32_t *word_ptr = reinterpret_cast<const uint32_t *>(source);
+      return static_cast<T>(word_ptr[index]);
+   }
+
+   enum module::argument::type
+   convertStorageClass(SpvStorageClass storage_class, std::string &err) {
+      switch (storage_class) {
+      case SpvStorageClassFunction:
+         return module::argument::scalar;
+      case SpvStorageClassUniformConstant:
+         return module::argument::constant;
+      case SpvStorageClassWorkgroup:
+         return module::argument::local;
+      case SpvStorageClassCrossWorkgroup:
+         return module::argument::global;
+      default:
+         err += "Invalid storage type " + std::to_string(storage_class) + "\n";
+         throw build_error();
+      }
+   }
+
+   enum module::argument::type
+   convertImageType(SpvId id, SpvDim dim, SpvAccessQualifier access,
+                    std::string &err) {
+#define APPEND_DIM(d) \
+      switch(access) { \
+      case SpvAccessQualifierReadOnly: \
+         return module::argument::image##d##_rd; \
+      case SpvAccessQualifierWriteOnly: \
+         return module::argument::image##d##_wr; \
+      default: \
+         err += "Unsupported access qualifier " #d " for image " + \
+                std::to_string(id); \
+         throw build_error(); \
+      }
+
+      switch (dim) {
+      case SpvDim2D:
+         APPEND_DIM(2d)
+      case SpvDim3D:
+         APPEND_DIM(3d)
+      default:
+         err += "Unsupported dimension " + std::to_string(dim) + " for image " +
+                std::to_string(id);
+         throw build_error();
+      }
+
+#undef APPEND_DIM
+   }
+
+   module::section
+   make_text_section(const std::vector<char> &code,
+                     enum module::section::type section_type) {
+      const pipe_llvm_program_header header { uint32_t(code.size()) };
+      module::section text { 0, section_type, header.num_bytes, {} };
+
+      text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header),
+                       reinterpret_cast<const char *>(&header) + sizeof(header));
+      text.data.insert(text.data.end(), code.begin(), code.end());
+
+      return text;
+   }
+
+   module
+   create_module_from_spirv(const std::vector<char> &source,
+                            size_t pointer_byte_size,
+                            std::string &err) {
+      const size_t length = source.size() / sizeof(uint32_t);
+      size_t i = 5u; // Skip header
+
+      std::string kernel_name;
+      size_t kernel_nb = 0u;
+      std::vector<module::argument> args;
+
+      module m;
+
+      std::unordered_map<SpvId, std::string> kernels;
+      std::unordered_map<SpvId, module::argument> types;
+      std::unordered_map<SpvId, SpvId> pointer_types;
+      std::unordered_map<SpvId, unsigned int> constants;
+      std::unordered_set<SpvId> packed_structures;
+      std::unordered_map<SpvId, std::vector<SpvFunctionParameterAttribute>>
+         func_param_attr_map;
+
+#define GET_OPERAND(type, operand_id) get<type>(source.data(), i + operand_id)
+
+      while (i < length) {
+         const auto desc_word = get<uint32_t>(source.data(), i);
+         const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
+         const unsigned int num_operands = desc_word >> SpvWordCountShift;
+
+         switch (opcode) {
+         case SpvOpEntryPoint:
+            if (GET_OPERAND(SpvExecutionModel, 1) == SpvExecutionModelKernel)
+               kernels.emplace(GET_OPERAND(SpvId, 2),
+                               source.data() + (i + 3u) * sizeof(uint32_t));
+            break;
+
+         case SpvOpDecorate: {
+            const auto id = GET_OPERAND(SpvId, 1);
+            const auto decoration = GET_OPERAND(SpvDecoration, 2);
+            if (decoration == SpvDecorationCPacked)
+               packed_structures.emplace(id);
+            else if (decoration == SpvDecorationFuncParamAttr)
+               func_param_attr_map[id].push_back(GET_OPERAND(SpvFunctionParameterAttribute, 3u));
+            break;
+         }
+
+         case SpvOpGroupDecorate: {
+            const auto group_id = GET_OPERAND(SpvId, 1);
+            if (packed_structures.count(group_id)) {
+               for (unsigned int i = 2u; i < num_operands; ++i)
+                  packed_structures.emplace(GET_OPERAND(SpvId, i));
+            }
+            const auto func_param_attr_iter =
+               func_param_attr_map.find(group_id);
+            if (func_param_attr_iter != func_param_attr_map.end()) {
+               for (unsigned int i = 2u; i < num_operands; ++i)
+                  func_param_attr_map.emplace(GET_OPERAND(SpvId, i),
+                                              func_param_attr_iter->second);
+            }
+            break;
+         }
+
+         case SpvOpConstant:
+            // We only care about constants that represents the size of arrays.
+            // If they are passed as argument, they will never be more than
+            // 4GB-wide, and even if they did, a clover::module::argument size
+            // is represented by an int.
+            constants[GET_OPERAND(SpvId, 2)] = GET_OPERAND(unsigned int, 3u);
+            break;
+
+         case SpvOpTypeInt: // FALLTHROUGH
+         case SpvOpTypeFloat: {
+            const auto size = GET_OPERAND(uint32_t, 2) / 8u;
+            types[GET_OPERAND(SpvId, 1)] = { module::argument::scalar,
+                                               size, size, size,
+                                               module::argument::zero_ext };
+            break;
+         }
+
+         case SpvOpTypeArray: {
+            const auto id = GET_OPERAND(SpvId, 1);
+            const auto type_id = GET_OPERAND(SpvId, 2);
+            const auto types_iter = types.find(type_id);
+            if (types_iter == types.end())
+               break;
+
+            const auto constant_id = GET_OPERAND(SpvId, 3);
+            const auto constants_iter = constants.find(constant_id);
+            if (constants_iter == constants.end()) {
+               err += "Constant " + std::to_string(constant_id) +
+                  " is missing\n";
+               throw build_error();
+            }
+            const auto elem_size = types_iter->second.size;
+            const auto elem_nbs = constants_iter->second;
+            const auto size = elem_size * elem_nbs;
+            types[id] = { module::argument::scalar, size, size,
+                          types_iter->second.target_align,
+                          module::argument::zero_ext };
+            break;
+         }
+
+         case SpvOpTypeStruct: {
+            const auto id = GET_OPERAND(SpvId, 1);
+            const bool is_packed = packed_structures.count(id);
+
+            unsigned struct_size = 0u;
+            unsigned max_elem_size = 0u;
+            for (unsigned j = 2u; j < num_operands; ++j) {
+               const auto type_id = GET_OPERAND(SpvId, j);
+               const auto types_iter = types.find(type_id);
+               if (types_iter == types.end())
+                  break;
+
+               const auto alignment = is_packed ? 1u
+                                                : types_iter->second.target_align;
+               const auto padding = (-struct_size) & (alignment - 1u);
+               struct_size += padding + types_iter->second.target_size;
+               max_elem_size = std::max(max_elem_size,
+                                        types_iter->second.target_align);
+            }
+            if (!is_packed)
+               struct_size += (-struct_size) & (max_elem_size - 1u);
+
+            types[id] = { module::argument::scalar, struct_size, struct_size,
+                          is_packed ? 1u
+                                    : max_elem_size, module::argument::zero_ext
+            };
+            break;
+         }
+
+         case SpvOpTypeVector: {
+            const auto id = GET_OPERAND(SpvId, 1);
+            const auto type_id = GET_OPERAND(SpvId, 2);
+            const auto types_iter = types.find(type_id);
+            if (types_iter == types.end())
+               break;
+
+            const auto elem_size = types_iter->second.size;
+            const auto elem_nbs = GET_OPERAND(uint32_t, 3);
+            const auto size = elem_size * elem_nbs;
+            types[id] = { module::argument::scalar, size, size, size,
+                          module::argument::zero_ext };
+            break;
+         }
+
+         case SpvOpTypeForwardPointer: // FALLTHROUGH
+         case SpvOpTypePointer: {
+            const auto id = GET_OPERAND(SpvId, 1);
+            const auto storage_class = GET_OPERAND(SpvStorageClass, 2);
+            // Input means this is for a builtin variable, which can not be
+            // passed as an argument to a kernel.
+            if (storage_class == SpvStorageClassInput)
+               break;
+            types[id] = { convertStorageClass(storage_class, err),
+                          sizeof(cl_mem),
+                          static_cast<module::size_t>(pointer_byte_size),
+                          static_cast<module::size_t>(pointer_byte_size),
+                          module::argument::zero_ext
+            };
+            if (opcode == SpvOpTypePointer)
+               pointer_types[id] = GET_OPERAND(SpvId, 3);
+            break;
+         }
+
+         case SpvOpTypeSampler:
+            types[GET_OPERAND(SpvId, 1)] = { module::argument::sampler,
+                                             sizeof(cl_sampler)
+            };
+            break;
+
+         case SpvOpTypeImage: {
+            const auto id = GET_OPERAND(SpvId, 1);
+            const auto dim = GET_OPERAND(SpvDim, 3);
+            const auto access = GET_OPERAND(SpvAccessQualifier, 9);
+            types[id] = { convertImageType(id, dim, access, err),
+                          sizeof(cl_mem), sizeof(cl_mem), sizeof(cl_mem),
+                          module::argument::zero_ext
+            };
+            break;
+         }
+
+         case SpvOpTypePipe: // FALLTHROUGH
+         case SpvOpTypeQueue: {
+            err += "TypePipe and TypeQueue are valid SPIR-V 1.0 types, but are "
+                   "not available in the currently supported OpenCL C version."
+                   "\n";
+            throw build_error();
+         }
+
+         case SpvOpFunction: {
+            const auto kernels_iter = kernels.find(GET_OPERAND(SpvId, 2));
+            if (kernels_iter != kernels.end())
+               kernel_name = kernels_iter->second;
+            break;
+         }
+
+         case SpvOpFunctionParameter: {
+            if (kernel_name.empty())
+               break;
+
+            const auto type_id = GET_OPERAND(SpvId, 1);
+            auto arg = types.find(type_id)->second;
+            const auto &func_param_attr_iter =
+               func_param_attr_map.find(GET_OPERAND(SpvId, 2));
+            if (func_param_attr_iter != func_param_attr_map.end()) {
+               for (auto &i : func_param_attr_iter->second) {
+                  switch (i) {
+                  case SpvFunctionParameterAttributeSext:
+                     arg.ext_type = module::argument::sign_ext;
+                     break;
+                  case SpvFunctionParameterAttributeZext:
+                     arg.ext_type = module::argument::zero_ext;
+                     break;
+                  case SpvFunctionParameterAttributeByVal: {
+                     const SpvId ptr_type_id =
+                        pointer_types.find(type_id)->second;
+                     arg = types.find(ptr_type_id)->second;
+                     break;
+                  }
+                  default:
+                     break;
+                  }
+               }
+            }
+            args.emplace_back(arg);
+            break;
+         }
+
+         case SpvOpFunctionEnd:
+            if (kernel_name.empty())
+               break;
+            m.syms.emplace_back(kernel_name, 0, kernel_nb, args);
+            ++kernel_nb;
+            kernel_name.clear();
+            args.clear();
+            break;
+
+         default:
+            break;
+         }
+
+         i += num_operands;
+      }
+
+#undef GET_OPERAND
+
+      m.secs.push_back(make_text_section(source,
+                                         module::section::text_intermediate));
+      return m;
+   }
+
+   bool
+   check_capabilities(const device &dev, const std::vector<char> &source,
+                      std::string &r_log) {
+      const size_t length = source.size() / sizeof(uint32_t);
+      size_t i = 5u; // Skip header
+
+      while (i < length) {
+         const auto desc_word = get<uint32_t>(source.data(), i);
+         const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
+         const unsigned int num_operands = desc_word >> SpvWordCountShift;
+
+         if (opcode != SpvOpCapability)
+            break;
+
+         const auto capability = get<SpvCapability>(source.data(), i + 1u);
+         switch (capability) {
+         // Mandatory capabilities
+         case SpvCapabilityAddresses:
+         case SpvCapabilityFloat16Buffer:
+         case SpvCapabilityGroups:
+         case SpvCapabilityInt64:
+         case SpvCapabilityInt16:
+         case SpvCapabilityInt8:
+         case SpvCapabilityKernel:
+         case SpvCapabilityLinkage:
+         case SpvCapabilityVector16:
+            break;
+         // Optional capabilities
+         case SpvCapabilityImageBasic:
+         case SpvCapabilityLiteralSampler:
+         case SpvCapabilitySampled1D:
+         case SpvCapabilityImage1D:
+         case SpvCapabilitySampledBuffer:
+         case SpvCapabilityImageBuffer:
+            if (!dev.image_support()) {
+               r_log += "Capability 'ImageBasic' is not supported.\n";
+               return false;
+            }
+            break;
+         case SpvCapabilityFloat64:
+            if (!dev.has_doubles()) {
+               r_log += "Capability 'Float64' is not supported.\n";
+               return false;
+            }
+            break;
+         // Enabled through extensions
+         case SpvCapabilityFloat16:
+            if (!dev.has_halves()) {
+               r_log += "Capability 'Float16' is not supported.\n";
+               return false;
+            }
+            break;
+         default:
+            r_log += "Capability '" + std::to_string(capability) +
+                     "' is not supported.\n";
+            return false;
+         }
+
+         i += num_operands;
+      }
+
+      return true;
+   }
+
+   bool
+   check_extensions(const device &dev, const std::vector<char> &source,
+                    std::string &r_log) {
+      const size_t length = source.size() / sizeof(uint32_t);
+      size_t i = 5u; // Skip header
+
+      while (i < length) {
+         const auto desc_word = get<uint32_t>(source.data(), i);
+         const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
+         const unsigned int num_operands = desc_word >> SpvWordCountShift;
+
+         if (opcode == SpvOpCapability) {
+            i += num_operands;
+            continue;
+         }
+         if (opcode != SpvOpExtension)
+            break;
+
+         const char *extension = source.data() + (i + 1u) * sizeof(uint32_t);
+         const std::string device_extensions = dev.supported_extensions();
+         const std::string platform_extensions =
+            dev.platform.supported_extensions();
+         if (device_extensions.find(extension) == std::string::npos &&
+             platform_extensions.find(extension) == std::string::npos) {
+            r_log += "Extension '" + std::string(extension) +
+                     "' is not supported.\n";
+            return false;
+         }
+
+         i += num_operands;
+      }
+
+      return true;
+   }
+
+   bool
+   check_memory_model(const device &dev, const std::vector<char> &source,
+                      std::string &r_log) {
+      const size_t length = source.size() / sizeof(uint32_t);
+      size_t i = 5u; // Skip header
+
+      while (i < length) {
+         const auto desc_word = get<uint32_t>(source.data(), i);
+         const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask);
+         const unsigned int num_operands = desc_word >> SpvWordCountShift;
+
+         switch (opcode) {
+         case SpvOpMemoryModel:
+            switch (get<SpvAddressingModel>(source.data(), i + 1u)) {
+            case SpvAddressingModelPhysical32:
+               return dev.address_bits() == 32;
+            case SpvAddressingModelPhysical64:
+               return dev.address_bits() == 64;
+            default:
+               unreachable("Only Physical32 and Physical64 are valid for OpenCL, and the binary was already validated");
+               return false;
+            }
+            break;
+         default:
+            break;
+         }
+
+         i += num_operands;
+      }
+
+      return false;
+   }
+
+   // Copies the input binary and convert it to the endianness of the host CPU.
+   std::vector<char>
+   spirv_to_cpu(const std::vector<char> &binary)
+   {
+      const uint32_t first_word = get<uint32_t>(binary.data(), 0u);
+      if (first_word == SpvMagicNumber)
+         return binary;
+
+      std::vector<char> cpu_endianness_binary(binary.size());
+      for (size_t i = 0; i < (binary.size() / 4u); ++i) {
+         const uint32_t word = get<uint32_t>(binary.data(), i);
+         reinterpret_cast<uint32_t *>(cpu_endianness_binary.data())[i] =
+            util_bswap32(word);
+      }
+
+      return cpu_endianness_binary;
+   }
+
 #ifdef CLOVER_ALLOW_SPIRV
    std::string
    format_validator_msg(spv_message_level_t level, const char * /* source */,
@@ -96,7 +576,117 @@  clover::spirv::is_binary_spirv(const char *il, size_t length)
           (util_bswap32(first_word) == SpvMagicNumber);
 }
 
+module
+clover::spirv::process_program(const std::vector<char> &binary,
+                               const device &dev, bool validate,
+                               std::string &r_log) {
+   std::vector<char> source = spirv_to_cpu(binary);
+
+   if (validate && !is_valid_spirv(
+         reinterpret_cast<const uint32_t *>(source.data()),
+         source.size() / 4u, dev.device_version(),
+         [&r_log](const char *log){ r_log += std::string(log); }))
+      throw build_error();
+
+   if (!check_capabilities(dev, source, r_log))
+      throw build_error();
+   if (!check_extensions(dev, source, r_log))
+      throw build_error();
+   if (!check_memory_model(dev, source, r_log))
+      throw build_error();
+
+   return create_module_from_spirv(source,
+                                   dev.address_bits() == 32 ? 4u : 8u, r_log);
+}
+
 #ifdef CLOVER_ALLOW_SPIRV
+module
+clover::spirv::link_program(const std::vector<module> &modules,
+                            const device &dev, const std::string &opts,
+                            std::string &r_log) {
+   std::vector<std::string> options = clover::llvm::tokenize(opts);
+
+   const bool create_library = count("-create-library", options);
+   erase_if(equals("-create-library"), options);
+   erase_if(equals("-enable-link-options"), options);
+
+   if (!options.empty()) {
+      r_log += "SPIR-V linker: Ignoring the following link options: ";
+      for (const auto &opt : options)
+         r_log += "'" + opt + "' ";
+   }
+
+   spvtools::LinkerOptions linker_options;
+   linker_options.SetCreateLibrary(create_library);
+
+   module m;
+
+   const auto section_type = create_library ? module::section::text_library :
+                                              module::section::text_executable;
+
+   std::vector<const uint32_t *> sections;
+   sections.reserve(modules.size());
+   std::vector<size_t> lengths;
+   lengths.reserve(modules.size());
+
+   auto const validator_consumer = [&r_log](spv_message_level_t level,
+                                            const char *source,
+                                            const spv_position_t &position,
+                                            const char *message) {
+      r_log += format_validator_msg(level, source, position, message);
+   };
+
+   for (const auto &mod : modules) {
+      const module::section *msec = nullptr;
+      try {
+         msec = &find([&](const module::section &sec) {
+                  return sec.type == module::section::text_intermediate ||
+                         sec.type == module::section::text_library;
+               }, mod.secs);
+      } catch (const std::out_of_range &e) {
+         // We should never reach this as validate_link_devices already checked
+         // that a binary was present.
+         assert(false);
+      }
+
+      const auto c_il = msec->data.data() +
+                        sizeof(struct pipe_llvm_program_header);
+      const auto length = msec->size;
+
+      sections.push_back(reinterpret_cast<const uint32_t *>(c_il));
+      lengths.push_back(length / sizeof(uint32_t));
+   }
+
+   std::vector<uint32_t> linked_binary;
+
+   const std::string opencl_version = dev.device_version();
+   const spv_target_env target_env =
+      convert_opencl_str_to_target_env(opencl_version);
+
+   const spvtools::MessageConsumer consumer = validator_consumer;
+   spvtools::Context context(target_env);
+   context.SetMessageConsumer(std::move(consumer));
+
+   if (Link(context, sections.data(), lengths.data(), sections.size(),
+            &linked_binary, linker_options) != SPV_SUCCESS)
+      throw error(CL_LINK_PROGRAM_FAILURE);
+
+   if (!is_valid_spirv(linked_binary.data(), linked_binary.size(),
+                       opencl_version,
+                       [&r_log](const char *log){ r_log += std::string(log); }))
+      throw error(CL_LINK_PROGRAM_FAILURE);
+
+   for (const auto &mod : modules)
+      m.syms.insert(m.syms.end(), mod.syms.begin(), mod.syms.end());
+
+   m.secs.emplace_back(make_text_section({
+            reinterpret_cast<char *>(linked_binary.data()),
+            reinterpret_cast<char *>(linked_binary.data()) +
+               linked_binary.size() * sizeof(uint32_t) }, section_type));
+
+   return m;
+}
+
 bool
 clover::spirv::is_valid_spirv(const uint32_t *binary, size_t length,
                               const std::string &opencl_version,
@@ -120,6 +710,14 @@  clover::spirv::is_valid_spirv(const uint32_t *binary, size_t length,
    return spvTool.Validate(binary, length);
 }
 #else
+module
+clover::spirv::link_program(const std::vector<module> &/*modules*/,
+                            const device &/*dev*/, const std::string &/*opts*/,
+                            std::string &r_log) {
+   r_log += "SPIRV-Tools and llvm-spirv are required for linking SPIR-V binaries.\n";
+   throw error(CL_LINKER_NOT_AVAILABLE);
+}
+
 bool
 clover::spirv::is_valid_spirv(const uint32_t * /*binary*/, size_t /*length*/,
                               const std::string &/*opencl_version*/,
diff --git a/src/gallium/state_trackers/clover/spirv/invocation.hpp b/src/gallium/state_trackers/clover/spirv/invocation.hpp
index 92255a68a8e..37cd1377cb2 100644
--- a/src/gallium/state_trackers/clover/spirv/invocation.hpp
+++ b/src/gallium/state_trackers/clover/spirv/invocation.hpp
@@ -24,6 +24,8 @@ 
 #define CLOVER_SPIRV_INVOCATION_HPP
 
 #include "core/context.hpp"
+#include "core/module.hpp"
+#include "core/program.hpp"
 
 namespace clover {
    namespace spirv {
@@ -41,6 +43,16 @@  namespace clover {
       bool is_valid_spirv(const uint32_t *binary, size_t length,
                           const std::string &opencl_version,
                           const context::notify_action &notify);
+
+      // Creates a clover module out of the given SPIR-V binary.
+      module process_program(const std::vector<char> &binary,
+                             const device &dev, bool validate,
+                             std::string &r_log);
+
+      // Combines multiple clover modules into a single one, resolving
+      // link dependencies between them.
+      module link_program(const std::vector<module> &modules, const device &dev,
+                          const std::string &opts, std::string &r_log);
    }
 }
 

Comments