/*
 * Copyright © 2015-2019 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/** @file
 *
 * This file implements a pass that validates shader assembly.
 *
 * The restrictions implemented herein are intended to verify that instructions
 * in shader assembly do not violate restrictions documented in the graphics
 * programming reference manuals.
 *
 * The restrictions are difficult for humans to quickly verify due to their
 * complexity and abundance.
 *
 * It is critical that this code is thoroughly unit tested because false
 * results will lead developers astray, which is worse than having no validator
 * at all. Functional changes to this file without corresponding unit tests (in
 * test_eu_validate.cpp) will be rejected.
 */

#include "brw_eu.h"
#include "brw_disasm_info.h"

enum brw_hw_instr_format {
   FORMAT_BASIC,
   FORMAT_BASIC_THREE_SRC,
   FORMAT_DPAS_THREE_SRC,
   FORMAT_SEND,
   FORMAT_BRANCH,
   FORMAT_ILLEGAL,
   FORMAT_NOP,
};

typedef struct brw_hw_decoded_inst {
   const brw_eu_inst *raw;

   enum brw_hw_instr_format format;

   enum opcode opcode;

   unsigned exec_size;
   unsigned access_mode;

   enum brw_conditional_mod cond_modifier;
   enum brw_predicate pred_control;
   bool saturate;

   bool has_dst;
   struct {
      enum brw_reg_file file;
      enum brw_reg_type type;
      unsigned address_mode;

      /* These are already physical register numbers. */
      unsigned nr;
      unsigned subnr;

      unsigned hstride;
   } dst;

   unsigned num_sources;
   struct {
      enum brw_reg_file file;
      enum brw_reg_type type;
      unsigned address_mode;
      bool negate;
      bool abs;

      /* These are already physical register numbers. */
      unsigned nr;
      unsigned subnr;

      unsigned vstride;
      unsigned width;
      unsigned hstride;
   } src[3];
} brw_hw_decoded_inst;

struct error {
   char *msg;
};

static void
report_error(struct error *error, const char *msg)
{
   assert(error);

   if (error->msg) {
      /* Ignore duplicate reports. */
      if (strstr(error->msg, msg))
         return;
      ralloc_asprintf_append(&error->msg, "\tERROR: %s\n", msg);
   } else {
      error->msg = ralloc_asprintf(NULL, "\tERROR: %s\n", msg);
   }
}

#define ERROR(msg) ERROR_IF(true, msg)
#define ERROR_IF(cond, msg)                             \
   do {                                                 \
      if ((cond))                                       \
         report_error(error, msg);                      \
   } while(0)

#define RETURN_ERROR(msg) RETURN_ERROR_IF(true, msg)
#define RETURN_ERROR_IF(cond, msg)                      \
   do {                                                 \
      if ((cond)) {                                     \
         report_error(error, msg);                      \
         return;                                        \
      }                                                 \
   } while(0)

#define STRIDE(stride) (stride != 0 ? 1 << ((stride) - 1) : 0)
#define WIDTH(width)   (1 << (width))

static bool
inst_is_send(const brw_hw_decoded_inst *inst)
{
   switch (inst->opcode) {
   case BRW_OPCODE_SEND:
   case BRW_OPCODE_SENDC:
   case BRW_OPCODE_SENDS:
   case BRW_OPCODE_SENDSC:
      return true;
   default:
      return false;
   }
}

static bool
inst_is_split_send(const struct brw_isa_info *isa, const brw_hw_decoded_inst *inst)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   if (devinfo->ver >= 12) {
      return inst_is_send(inst);
   } else {
      switch (inst->opcode) {
      case BRW_OPCODE_SENDS:
      case BRW_OPCODE_SENDSC:
         return true;
      default:
         return false;
      }
   }
}

static unsigned
signed_type(unsigned type)
{
   return brw_type_is_uint(type) ? (type | BRW_TYPE_BASE_SINT) : type;
}

static bool
inst_is_raw_move(const brw_hw_decoded_inst *inst)
{
   unsigned dst_type = signed_type(inst->dst.type);
   unsigned src_type = signed_type(inst->src[0].type);

   if (inst->src[0].file == IMM) {
      /* FIXME: not strictly true */
      if (brw_type_is_vector_imm(inst->src[0].type))
         return false;
   } else if (inst->src[0].negate || inst->src[0].abs) {
      return false;
   }

   return inst->opcode == BRW_OPCODE_MOV &&
          !inst->saturate &&
          dst_type == src_type;
}

static bool
dst_is_null(const brw_hw_decoded_inst *inst)
{
   return inst->dst.file == ARF && inst->dst.nr == BRW_ARF_NULL;
}

static bool
src0_is_null(const brw_hw_decoded_inst *inst)
{
   return inst->src[0].address_mode == BRW_ADDRESS_DIRECT &&
          inst->src[0].file == ARF &&
          inst->src[0].nr == BRW_ARF_NULL;
}

static bool
src1_is_null(const brw_hw_decoded_inst *inst)
{
   assert(inst->src[1].address_mode == BRW_ADDRESS_DIRECT);
   return inst->src[1].file == ARF &&
          inst->src[1].nr == BRW_ARF_NULL;
}

static bool
src0_is_acc(const brw_hw_decoded_inst *inst)
{
   return inst->src[0].address_mode == BRW_ADDRESS_DIRECT &&
          inst->src[0].file == ARF &&
          (inst->src[0].nr & 0xF0) == BRW_ARF_ACCUMULATOR;
}

static bool
src1_is_acc(const brw_hw_decoded_inst *inst)
{
   assert(inst->src[1].address_mode == BRW_ADDRESS_DIRECT);
   return inst->src[1].file == ARF &&
          (inst->src[1].nr & 0xF0) == BRW_ARF_ACCUMULATOR;
}

static bool
src_has_scalar_region(const brw_hw_decoded_inst *inst, int src)
{
   return inst->src[src].vstride == 0 &&
          inst->src[src].width == 1 &&
          inst->src[src].hstride == 0;
}

static void
invalid_values(const struct brw_isa_info *isa, const brw_hw_decoded_inst *inst,
               struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   if (devinfo->ver >= 12) {
      unsigned qtr_ctrl = brw_eu_inst_qtr_control(devinfo, inst->raw);
      unsigned nib_ctrl =
         devinfo->ver == 12 ? brw_eu_inst_nib_control(devinfo, inst->raw) : 0;

      unsigned chan_off = (qtr_ctrl * 2 + nib_ctrl) << 2;
      ERROR_IF(chan_off % inst->exec_size != 0,
               "The execution size must be a factor of the chosen offset");
   }
}

static void
sources_not_null(const struct brw_isa_info *isa,
                 const brw_hw_decoded_inst *inst,
                 struct error *error)
{
   /* Nothing to test. 3-src instructions can only have GRF sources, and
    * there's no bit to control the file.
    */
   if (inst->num_sources == 3)
      return;

   /* Nothing to test.  Split sends can only encode a file in sources that are
    * allowed to be NULL.
    */
   if (inst_is_split_send(isa, inst))
      return;

   if (inst->num_sources >= 1 && inst->opcode != BRW_OPCODE_SYNC)
      ERROR_IF(src0_is_null(inst), "src0 is null");

   if (inst->num_sources == 2)
      ERROR_IF(src1_is_null(inst), "src1 is null");
}

static bool
inst_uses_src_acc(const struct brw_isa_info *isa,
                  const brw_hw_decoded_inst *inst)
{
   /* Check instructions that use implicit accumulator sources */
   switch (inst->opcode) {
   case BRW_OPCODE_MAC:
   case BRW_OPCODE_MACH:
      return true;
   default:
      break;
   }

   /* FIXME: support 3-src instructions */
   assert(inst->num_sources < 3);

   return src0_is_acc(inst) || (inst->num_sources > 1 && src1_is_acc(inst));
}

static void
send_restrictions(const struct brw_isa_info *isa,
                  const brw_hw_decoded_inst *inst,
                  struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   if (inst_is_split_send(isa, inst)) {
      ERROR_IF(inst->src[1].file == ARF &&
               inst->src[1].nr != BRW_ARF_NULL,
               "src1 of split send must be a GRF or NULL");

      if (devinfo->ver < 30) {
         ERROR_IF(brw_eu_inst_eot(devinfo, inst->raw) &&
                  inst->src[0].nr < 112,
                  "send with EOT must use g112-g127");
         ERROR_IF(brw_eu_inst_eot(devinfo, inst->raw) &&
                  inst->src[1].file == FIXED_GRF &&
                  inst->src[1].nr < 112,
                  "send with EOT must use g112-g127");
      }

      if (inst->src[0].file == FIXED_GRF && inst->src[1].file == FIXED_GRF) {
         /* Assume minimums if we don't know */
         unsigned mlen = 1;
         if (!brw_eu_inst_send_sel_reg32_desc(devinfo, inst->raw)) {
            const uint32_t desc = brw_eu_inst_send_desc(devinfo, inst->raw);
            mlen = brw_message_desc_mlen(devinfo, desc) / reg_unit(devinfo);
         }

         unsigned ex_mlen = 1;
         if (!brw_eu_inst_send_sel_reg32_ex_desc(devinfo, inst->raw)) {
            const uint32_t ex_desc = brw_eu_inst_sends_ex_desc(devinfo, inst->raw, false);
            ex_mlen = brw_message_ex_desc_ex_mlen(devinfo, ex_desc) /
                      reg_unit(devinfo);
         }
         const unsigned src0_reg_nr = inst->src[0].nr;
         const unsigned src1_reg_nr = inst->src[1].nr;
         ERROR_IF((src0_reg_nr <= src1_reg_nr &&
                   src1_reg_nr < src0_reg_nr + mlen) ||
                  (src1_reg_nr <= src0_reg_nr &&
                   src0_reg_nr < src1_reg_nr + ex_mlen),
                   "split send payloads must not overlap");
      }
   } else if (inst_is_send(inst)) {
      ERROR_IF(inst->src[0].address_mode != BRW_ADDRESS_DIRECT,
               "send must use direct addressing");

      ERROR_IF(inst->src[0].file != FIXED_GRF,
               "send from non-GRF");
      ERROR_IF(brw_eu_inst_eot(devinfo, inst->raw) &&
               inst->src[0].nr < 112,
               "send with EOT must use g112-g127");

      if (devinfo->ver < 10) {
         ERROR_IF(!dst_is_null(inst) &&
                  (inst->dst.nr + brw_eu_inst_rlen(devinfo, inst->raw) > 127) &&
                  (inst->src[0].nr + brw_eu_inst_mlen(devinfo, inst->raw) > inst->dst.nr),
                  "r127 must not be used for return address when there is "
                  "a src and dest overlap");
      }
   }
}

static bool
is_unsupported_inst(const struct brw_isa_info *isa,
                    const brw_eu_inst *inst)
{
   return brw_eu_inst_opcode(isa, inst) == BRW_OPCODE_ILLEGAL;
}

/**
 * Returns whether a combination of two types would qualify as mixed float
 * operation mode
 */
static inline bool
types_are_mixed_float(enum brw_reg_type t0, enum brw_reg_type t1)
{
   return (t0 == BRW_TYPE_F && t1 == BRW_TYPE_HF) ||
          (t1 == BRW_TYPE_F && t0 == BRW_TYPE_HF);
}

static enum brw_reg_type
execution_type_for_type(enum brw_reg_type type)
{
   switch (type) {
   case BRW_TYPE_DF:
   case BRW_TYPE_F:
   case BRW_TYPE_HF:
      return type;

   case BRW_TYPE_VF:
   case BRW_TYPE_BF:
      return BRW_TYPE_F;

   case BRW_TYPE_Q:
   case BRW_TYPE_UQ:
      return BRW_TYPE_Q;

   case BRW_TYPE_D:
   case BRW_TYPE_UD:
      return BRW_TYPE_D;

   case BRW_TYPE_W:
   case BRW_TYPE_UW:
   case BRW_TYPE_B:
   case BRW_TYPE_UB:
   case BRW_TYPE_V:
   case BRW_TYPE_UV:
      return BRW_TYPE_W;

   default:
      return BRW_TYPE_INVALID;
   }
}

/**
 * Returns the execution type of an instruction \p inst
 */
static enum brw_reg_type
execution_type(const brw_hw_decoded_inst *inst)
{
   enum brw_reg_type src0_exec_type, src1_exec_type;

   /* Execution data type is independent of destination data type, except in
    * mixed F/HF instructions.
    */
   enum brw_reg_type dst_exec_type = inst->dst.type;

   src0_exec_type = execution_type_for_type(inst->src[0].type);
   if (inst->num_sources == 1) {
      if (src0_exec_type == BRW_TYPE_HF)
         return dst_exec_type;
      return src0_exec_type;
   }

   src1_exec_type = execution_type_for_type(inst->src[1].type);
   if (types_are_mixed_float(src0_exec_type, src1_exec_type) ||
       types_are_mixed_float(src0_exec_type, dst_exec_type) ||
       types_are_mixed_float(src1_exec_type, dst_exec_type)) {
      return BRW_TYPE_F;
   }

   if (src0_exec_type == src1_exec_type)
      return src0_exec_type;

   if (src0_exec_type == BRW_TYPE_Q ||
       src1_exec_type == BRW_TYPE_Q)
      return BRW_TYPE_Q;

   if (src0_exec_type == BRW_TYPE_D ||
       src1_exec_type == BRW_TYPE_D)
      return BRW_TYPE_D;

   if (src0_exec_type == BRW_TYPE_W ||
       src1_exec_type == BRW_TYPE_W)
      return BRW_TYPE_W;

   if (src0_exec_type == BRW_TYPE_DF ||
       src1_exec_type == BRW_TYPE_DF)
      return BRW_TYPE_DF;

   UNREACHABLE("not reached");
}

/**
 * Returns whether a region is packed
 *
 * A region is packed if its elements are adjacent in memory, with no
 * intervening space, no overlap, and no replicated values.
 */
static bool
is_packed(unsigned vstride, unsigned width, unsigned hstride)
{
   if (vstride == width) {
      if (vstride == 1) {
         return hstride == 0;
      } else {
         return hstride == 1;
      }
   }

   return false;
}

/**
 * Returns whether a region is linear
 *
 * A region is linear if its elements do not overlap and are not replicated.
 * Unlike a packed region, intervening space (i.e. strided values) is allowed.
 */
static bool
is_linear(unsigned vstride, unsigned width, unsigned hstride)
{
   return vstride == width * hstride ||
          (hstride == 0 && width == 1);
}

/**
 * Returns whether an instruction is an explicit or implicit conversion
 * to/from half-float.
 */
static bool
is_half_float_conversion(const brw_hw_decoded_inst *inst)
{
   enum brw_reg_type dst_type = inst->dst.type;

   enum brw_reg_type src0_type = inst->src[0].type;

   if (dst_type != src0_type &&
       (dst_type == BRW_TYPE_HF || src0_type == BRW_TYPE_HF)) {
      return true;
   } else if (inst->num_sources > 1) {
      enum brw_reg_type src1_type = inst->src[1].type;
      return dst_type != src1_type &&
            (dst_type == BRW_TYPE_HF ||
             src1_type == BRW_TYPE_HF);
   }

   return false;
}

/*
 * Returns whether an instruction is using mixed float operation mode
 */
static bool
is_mixed_float(const brw_hw_decoded_inst *inst)
{
   if (inst_is_send(inst))
      return false;

   if (!inst->has_dst)
      return false;

   /* FIXME: support 3-src instructions */
   assert(inst->num_sources < 3);

   enum brw_reg_type dst_type = inst->dst.type;
   enum brw_reg_type src0_type = inst->src[0].type;

   if (inst->num_sources == 1)
      return types_are_mixed_float(src0_type, dst_type);

   enum brw_reg_type src1_type = inst->src[1].type;

   return types_are_mixed_float(src0_type, src1_type) ||
          types_are_mixed_float(src0_type, dst_type) ||
          types_are_mixed_float(src1_type, dst_type);
}

static bool
is_pure_bfloat(const brw_hw_decoded_inst *inst)
{
   if (inst_is_send(inst))
      return false;

   if (inst->num_sources == 0 && !inst->has_dst)
      return false;

   for (int i = 0; i < inst->num_sources; i++) {
      if (!brw_type_is_bfloat(inst->src[i].type))
         return false;
   }

   if (inst->has_dst && !brw_type_is_bfloat(inst->dst.type))
      return false;

   return true;
}

static bool
is_mixed_bfloat(const brw_hw_decoded_inst *inst)
{
   if (inst_is_send(inst))
      return false;

   const int operands = inst->num_sources + inst->has_dst;

   int bfloat = 0;
   for (int i = 0; i < inst->num_sources; i++)
      bfloat += brw_type_is_bfloat(inst->src[i].type);
   if (inst->has_dst)
      bfloat += brw_type_is_bfloat(inst->dst.type);

   return bfloat > 0 && bfloat != operands;
}

/**
 * Returns whether an instruction is an explicit or implicit conversion
 * to/from byte.
 */
static bool
is_byte_conversion(const struct brw_isa_info *isa,
                   const brw_hw_decoded_inst *inst)
{
   enum brw_reg_type dst_type = inst->dst.type;

   enum brw_reg_type src0_type = inst->src[0].type;

   if (dst_type != src0_type &&
       (brw_type_size_bytes(dst_type) == 1 ||
        brw_type_size_bytes(src0_type) == 1)) {
      return true;
   } else if (inst->num_sources > 1) {
      enum brw_reg_type src1_type = inst->src[1].type;
      return dst_type != src1_type &&
            (brw_type_size_bytes(dst_type) == 1 ||
             brw_type_size_bytes(src1_type) == 1);
   }

   return false;
}

/**
 * Checks restrictions listed in "General Restrictions Based on Operand Types"
 * in the "Register Region Restrictions" section.
 */
static void
general_restrictions_based_on_operand_types(const struct brw_isa_info *isa,
                                            const brw_hw_decoded_inst *inst,
                                            struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   if (inst_is_send(inst))
      return;

   if (devinfo->ver >= 11) {
      /* A register type of B or UB for DPAS actually means 4 bytes packed into
       * a D or UD, so it is allowed.
       */
      if (inst->num_sources == 3 && inst->opcode != BRW_OPCODE_DPAS) {
         ERROR_IF(brw_type_size_bytes(inst->src[1].type) == 1 ||
                  brw_type_size_bytes(inst->src[2].type) == 1,
                  "Byte data type is not supported for src1/2 register regioning. This includes "
                  "byte broadcast as well.");
      }
      if (inst->num_sources == 2) {
         ERROR_IF(brw_type_size_bytes(inst->src[1].type) == 1,
                  "Byte data type is not supported for src1 register regioning. This includes "
                  "byte broadcast as well.");
      }
   }

   enum brw_reg_type dst_type = inst->dst.type;

   ERROR_IF(brw_type_is_bfloat(dst_type) &&
            !devinfo->has_bfloat16,
            "Bfloat destination, but platform does not support it");

   ERROR_IF(dst_type == BRW_TYPE_DF &&
            !devinfo->has_64bit_float,
            "64-bit float destination, but platform does not support it");

   ERROR_IF((dst_type == BRW_TYPE_Q ||
             dst_type == BRW_TYPE_UQ) &&
            !devinfo->has_64bit_int,
            "64-bit int destination, but platform does not support it");

   for (unsigned s = 0; s < inst->num_sources; s++) {
      enum brw_reg_type src_type = inst->src[s].type;

      ERROR_IF(brw_type_is_bfloat(src_type) &&
               !devinfo->has_bfloat16,
               "Bfloat source, but platform does not support it");

      ERROR_IF(src_type == BRW_TYPE_DF &&
               !devinfo->has_64bit_float,
               "64-bit float source, but platform does not support it");

      ERROR_IF((src_type == BRW_TYPE_Q ||
                src_type == BRW_TYPE_UQ) &&
               !devinfo->has_64bit_int,
               "64-bit int source, but platform does not support it");
      if (inst->access_mode == BRW_ALIGN_16 &&
          inst->num_sources == 3 && brw_type_size_bytes(src_type) > 4) {
         /* From the Broadwell PRM, Volume 7 "3D Media GPGPU", page 944:
          *
          *    "This is applicable to 32b datatypes and 16b datatype. 64b
          *    datatypes cannot use the replicate control."
          */
         switch (s) {
         case 0:
            ERROR_IF(brw_eu_inst_3src_a16_src0_rep_ctrl(devinfo, inst->raw),
                     "RepCtrl must be zero for 64-bit source 0");
            break;
         case 1:
            ERROR_IF(brw_eu_inst_3src_a16_src1_rep_ctrl(devinfo, inst->raw),
                     "RepCtrl must be zero for 64-bit source 1");
            break;
         case 2:
            ERROR_IF(brw_eu_inst_3src_a16_src2_rep_ctrl(devinfo, inst->raw),
                     "RepCtrl must be zero for 64-bit source 2");
            break;
         default: UNREACHABLE("invalid src");
         }
      }
   }

   if (inst->num_sources == 3)
      return;

   if (inst->exec_size == 1)
      return;

   if (!inst->has_dst)
      return;

   if (inst->opcode == BRW_OPCODE_MATH &&
       intel_needs_workaround(devinfo, 22016140776)) {
      /* Wa_22016140776:
       *
       *    Scalar broadcast on HF math (packed or unpacked) must not be
       *    used.  Compiler must use a mov instruction to expand the scalar
       *    value to a vector before using in a HF (packed or unpacked)
       *    math operation.
       */
      ERROR_IF(inst->src[0].type == BRW_TYPE_HF &&
               src_has_scalar_region(inst, 0),
               "Scalar broadcast on HF math (packed or unpacked) must not "
               "be used.");

      if (inst->num_sources > 1) {
         ERROR_IF(inst->src[1].type == BRW_TYPE_HF &&
                  src_has_scalar_region(inst, 1),
                  "Scalar broadcast on HF math (packed or unpacked) must not "
                  "be used.");
      }
   }

   /* The PRMs say:
    *
    *    Where n is the largest element size in bytes for any source or
    *    destination operand type, ExecSize * n must be <= 64.
    *
    * But we do not attempt to enforce it, because it is implied by other
    * rules:
    *
    *    - that the destination stride must match the execution data type
    *    - sources may not span more than two adjacent GRF registers
    *    - destination may not span more than two adjacent GRF registers
    *
    * In fact, checking it would weaken testing of the other rules.
    */

   unsigned dst_stride = inst->dst.hstride;
   bool dst_type_is_byte =
      inst->dst.type == BRW_TYPE_B ||
      inst->dst.type == BRW_TYPE_UB;

   if (dst_type_is_byte) {
      if (is_packed(inst->exec_size * dst_stride, inst->exec_size, dst_stride)) {
         ERROR_IF(!inst_is_raw_move(inst),
                  "Only raw MOV supports a packed-byte destination");
      }
   }

   unsigned exec_type = execution_type(inst);
   unsigned exec_type_size = brw_type_size_bytes(exec_type);
   unsigned dst_type_size = brw_type_size_bytes(dst_type);

   if (is_byte_conversion(isa, inst)) {
      /* From the BDW+ PRM, Volume 2a, Command Reference, Instructions - MOV:
       *
       *    "There is no direct conversion from B/UB to DF or DF to B/UB.
       *     There is no direct conversion from B/UB to Q/UQ or Q/UQ to B/UB."
       *
       * Even if these restrictions are listed for the MOV instruction, we
       * validate this more generally, since there is the possibility
       * of implicit conversions from other instructions.
       */
      enum brw_reg_type src0_type = inst->src[0].type;
      enum brw_reg_type src1_type = inst->num_sources > 1 ?
                                    inst->src[1].type : 0;

      ERROR_IF(brw_type_size_bytes(dst_type) == 1 &&
               (brw_type_size_bytes(src0_type) == 8 ||
                (inst->num_sources > 1 && brw_type_size_bytes(src1_type) == 8)),
               "There are no direct conversions between 64-bit types and B/UB");

      ERROR_IF(brw_type_size_bytes(dst_type) == 8 &&
               (brw_type_size_bytes(src0_type) == 1 ||
                (inst->num_sources > 1 && brw_type_size_bytes(src1_type) == 1)),
               "There are no direct conversions between 64-bit types and B/UB");
   }

   if (is_half_float_conversion(inst)) {
      /**
       * A helper to validate used in the validation of the following restriction
       * from the BDW+ PRM, Volume 2a, Command Reference, Instructions - MOV:
       *
       *    "There is no direct conversion from HF to DF or DF to HF.
       *     There is no direct conversion from HF to Q/UQ or Q/UQ to HF."
       *
       * Even if these restrictions are listed for the MOV instruction, we
       * validate this more generally, since there is the possibility
       * of implicit conversions from other instructions, such us implicit
       * conversion from integer to HF with the ADD instruction in SKL+.
       */
      enum brw_reg_type src0_type = inst->src[0].type;
      enum brw_reg_type src1_type = inst->num_sources > 1 ?
                                    inst->src[1].type : 0;
      ERROR_IF(dst_type == BRW_TYPE_HF &&
               (brw_type_size_bytes(src0_type) == 8 ||
                (inst->num_sources > 1 && brw_type_size_bytes(src1_type) == 8)),
               "There are no direct conversions between 64-bit types and HF");

      ERROR_IF(brw_type_size_bytes(dst_type) == 8 &&
               (src0_type == BRW_TYPE_HF ||
                (inst->num_sources > 1 && src1_type == BRW_TYPE_HF)),
               "There are no direct conversions between 64-bit types and HF");

      /* From the BDW+ PRM:
       *
       *   "Conversion between Integer and HF (Half Float) must be
       *    DWord-aligned and strided by a DWord on the destination."
       *
       * Also, the above restrictions seems to be expanded on CHV and SKL+ by:
       *
       *   "There is a relaxed alignment rule for word destinations. When
       *    the destination type is word (UW, W, HF), destination data types
       *    can be aligned to either the lowest word or the second lowest
       *    word of the execution channel. This means the destination data
       *    words can be either all in the even word locations or all in the
       *    odd word locations."
       *
       * We do not implement the second rule as is though, since empirical
       * testing shows inconsistencies:
       *   - It suggests that packed 16-bit is not allowed, which is not true.
       *   - It suggests that conversions from Q/DF to W (which need to be
       *     64-bit aligned on the destination) are not possible, which is
       *     not true.
       *
       * So from this rule we only validate the implication that conversions
       * from F to HF need to be DWord strided (except in Align1 mixed
       * float mode where packed fp16 destination is allowed so long as the
       * destination is oword-aligned).
       *
       * Finally, we only validate this for Align1 because Align16 always
       * requires packed destinations, so these restrictions can't possibly
       * apply to Align16 mode.
       */
      if (inst->access_mode == BRW_ALIGN_1) {
         if ((dst_type == BRW_TYPE_HF &&
              (brw_type_is_int(src0_type) ||
               (inst->num_sources > 1 && brw_type_is_int(src1_type)))) ||
             (brw_type_is_int(dst_type) &&
              (src0_type == BRW_TYPE_HF ||
               (inst->num_sources > 1 && src1_type == BRW_TYPE_HF)))) {
            ERROR_IF(dst_stride * dst_type_size != 4,
                     "Conversions between integer and half-float must be "
                     "strided by a DWord on the destination");

            ERROR_IF(inst->dst.subnr % 4 != 0,
                     "Conversions between integer and half-float must be "
                     "aligned to a DWord on the destination");
         } else if (dst_type == BRW_TYPE_HF) {
            ERROR_IF(dst_stride != 2 &&
                     !(is_mixed_float(inst) &&
                       dst_stride == 1 && inst->dst.subnr % 16 == 0),
                     "Conversions to HF must have either all words in even "
                     "word locations or all words in odd word locations or "
                     "be mixed-float with Oword-aligned packed destination");
         }
      }
   }

   /* There are special regioning rules for mixed-float mode in CHV and SKL that
    * override the general rule for the ratio of sizes of the destination type
    * and the execution type. We will add validation for those in a later patch.
    */
   bool validate_dst_size_and_exec_size_ratio = !is_mixed_float(inst) && !is_mixed_bfloat(inst);

   if (validate_dst_size_and_exec_size_ratio &&
       exec_type_size > dst_type_size) {
      if (!(dst_type_is_byte && inst_is_raw_move(inst))) {
         ERROR_IF(dst_stride * dst_type_size != exec_type_size,
                  "Destination stride must be equal to the ratio of the sizes "
                  "of the execution data type to the destination type");
      }

      unsigned subreg = inst->dst.subnr;

      if (inst->access_mode == BRW_ALIGN_1 &&
          inst->dst.address_mode == BRW_ADDRESS_DIRECT) {
         /* The i965 PRM says:
          *
          *    Implementation Restriction: The relaxed alignment rule for byte
          *    destination (#10.5) is not supported.
          */
         if (dst_type_is_byte) {
            ERROR_IF(subreg % exec_type_size != 0 &&
                     subreg % exec_type_size != 1,
                     "Destination subreg must be aligned to the size of the "
                     "execution data type (or to the next lowest byte for byte "
                     "destinations)");
         } else {
            ERROR_IF(subreg % exec_type_size != 0,
                     "Destination subreg must be aligned to the size of the "
                     "execution data type");
         }
      }
   }
}

/**
 * Checks restrictions listed in "General Restrictions on Regioning Parameters"
 * in the "Register Region Restrictions" section.
 */
static void
general_restrictions_on_region_parameters(const struct brw_isa_info *isa,
                                          const brw_hw_decoded_inst *inst,
                                          struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   if (inst->num_sources == 3)
      return;

   /* Split sends don't have the bits in the instruction to encode regions so
    * there's nothing to check.
    */
   if (inst_is_split_send(isa, inst))
      return;

   if (inst->access_mode == BRW_ALIGN_16) {
      if (inst->has_dst && !dst_is_null(inst))
         ERROR_IF(inst->dst.hstride != 1,
                  "Destination Horizontal Stride must be 1");

      if (inst->num_sources >= 1) {
         ERROR_IF(inst->src[0].file != IMM &&
                  inst->src[0].vstride != 0 &&
                  inst->src[0].vstride != 2 &&
                  inst->src[0].vstride != 4,
                  "In Align16 mode, only VertStride of 0, 2, or 4 is allowed");
      }

      if (inst->num_sources == 2) {
         ERROR_IF(inst->src[1].file != IMM &&
                  inst->src[1].vstride != 0 &&
                  inst->src[1].vstride != 2 &&
                  inst->src[1].vstride != 4,
                  "In Align16 mode, only VertStride of 0, 2, or 4 is allowed");
      }

      return;
   }

   for (unsigned i = 0; i < inst->num_sources; i++) {
      if (inst->src[i].file == IMM)
         continue;

      enum brw_reg_type type = inst->src[i].type;
      unsigned element_size = brw_type_size_bytes(type);
      unsigned subreg = inst->src[i].subnr;
      unsigned vstride = inst->src[i].vstride;
      unsigned width = inst->src[i].width;
      unsigned hstride = inst->src[i].hstride;

      /* ExecSize must be greater than or equal to Width. */
      ERROR_IF(inst->exec_size < width, "ExecSize must be greater than or equal "
                                        "to Width");

      /* If Width = 1, HorzStride must be 0 regardless of the values of
       * ExecSize and VertStride.
       */
      if (width == 1) {
         ERROR_IF(hstride != 0,
                  "If Width = 1, HorzStride must be 0 regardless "
                  "of the values of ExecSize and VertStride");
      }

      if (vstride == STRIDE(BRW_VERTICAL_STRIDE_ONE_DIMENSIONAL))
         continue;

      /* If ExecSize = Width and HorzStride ≠ 0,
       * VertStride must be set to Width * HorzStride.
       */
      if (inst->exec_size == width && hstride != 0) {
         ERROR_IF(vstride != width * hstride,
                  "If ExecSize = Width and HorzStride ≠ 0, "
                  "VertStride must be set to Width * HorzStride");
      }

      /* If ExecSize = Width = 1, both VertStride and HorzStride must be 0. */
      if (inst->exec_size == 1 && width == 1) {
         ERROR_IF(vstride != 0 || hstride != 0,
                  "If ExecSize = Width = 1, both VertStride "
                  "and HorzStride must be 0");
      }

      /* If VertStride = HorzStride = 0, Width must be 1 regardless of the
       * value of ExecSize.
       */
      if (vstride == 0 && hstride == 0) {
         ERROR_IF(width != 1,
                  "If VertStride = HorzStride = 0, Width must be "
                  "1 regardless of the value of ExecSize");
      }

      /* VertStride must be used to cross GRF register boundaries. This rule
       * implies that elements within a 'Width' cannot cross GRF boundaries.
       */
      if (inst->src[i].file == FIXED_GRF) {
         unsigned rowbase = subreg;
         assert(util_is_power_of_two_nonzero(reg_unit(devinfo)));
         unsigned grf_size_shift = ffs(REG_SIZE * reg_unit(devinfo)) - 1;

         for (int y = 0; y < inst->exec_size / width; y++) {
            bool spans_grfs = false;
            unsigned offset = rowbase;
            unsigned first_grf = offset >> grf_size_shift;

            for (int x = 0; x < width; x++) {
               const unsigned end_byte = offset + (element_size - 1);
               const unsigned end_grf = end_byte >> grf_size_shift;
               spans_grfs = end_grf != first_grf;
               if (spans_grfs)
                  break;
               offset += hstride * element_size;
            }

            rowbase += vstride * element_size;

            if (spans_grfs) {
               ERROR("VertStride must be used to cross GRF register boundaries");
               break;
            }
         }
      }
   }

   /* Dst.HorzStride must not be 0. */
   if (inst->has_dst && !dst_is_null(inst)) {
      ERROR_IF(inst->dst.hstride == 0,
               "Destination Horizontal Stride must not be 0");
   }
}

static bool
is_multiplier_instruction(const brw_hw_decoded_inst *inst)
{
   /* TODO: Complete this list. */
   switch (inst->opcode) {
   case BRW_OPCODE_MUL:
   case BRW_OPCODE_MAC:
   case BRW_OPCODE_MACH:
   case BRW_OPCODE_MAD:
      return true;
   default:
      return false;
   }
}

static void
special_restrictions_for_mixed_float_mode(const struct brw_isa_info *isa,
                                          const brw_hw_decoded_inst *inst,
                                          struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   /* See instruction_restrictions() for DPAS operand type validation. */
   if (inst->opcode == BRW_OPCODE_DPAS)
      return;

   ERROR_IF(is_pure_bfloat(inst),
            "Instructions with pure bfloat16 operands are not supported.");

   if (is_mixed_bfloat(inst)) {
      ERROR_IF(devinfo->ver < 20 && inst->exec_size > 8,
               "Execution size must not be greater than 8 in Gfx12.");
      ERROR_IF(devinfo->ver >= 20 && inst->exec_size > 16,
               "Execution size must not be greater than 8 in Gfx20+.");

      for (int i = 0; i < inst->num_sources; i++) {
         ERROR_IF(brw_type_is_bfloat(inst->src[i].type) &&
                  src_has_scalar_region(inst, i),
                  "Broadcast of bfloat16 scalar is not supported.");
      }

      if (is_multiplier_instruction(inst)) {
         if (inst->num_sources == 2) {
            ERROR_IF(brw_type_is_bfloat(inst->src[1].type),
                     "Bfloat16 not allowed in Src1 of 2-source instructions involving multiplier.");
         } else if (inst->num_sources == 3) {
            ERROR_IF(brw_type_is_bfloat(inst->src[2].type),
                     "Bfloat16 not allowed in Src2 of 3-source instructions involving multiplier.");
         }
      }

      const unsigned half_offset = REG_SIZE * reg_unit(devinfo) / 2;

      if (inst->has_dst && brw_type_is_bfloat(inst->dst.type)) {
         unsigned dst_stride = inst->dst.hstride;
         bool dst_is_packed = is_packed(inst->exec_size * dst_stride, inst->exec_size, dst_stride);

         if (dst_is_packed) {
            ERROR_IF(inst->dst.subnr != 0 && inst->dst.subnr != half_offset,
                     "Packed bfloat16 destination must have register offset 0 or half of GRF register.");
         } else {
            /* Offset in the restriction text is in terms of elements. */
            const unsigned elem_size = brw_type_size_bytes(inst->dst.type);
            ERROR_IF(dst_stride != 2 || (inst->dst.subnr != 0 &&
                                         inst->dst.subnr != 1 * elem_size),
                     "Unpacked bfloat16 destination must have stride 2 and register offset 0 or 1.");
         }
      }

      for (int i = 0; i < inst->num_sources; i++) {
         if (brw_type_is_bfloat(inst->src[i].type)) {
            bool src_is_packed = is_packed(inst->src[i].vstride, inst->src[i].width, inst->src[i].hstride);
            ERROR_IF(!src_is_packed,
                     "Bfloat16 source must be packed");
            ERROR_IF(inst->src[i].subnr != 0 && inst->src[i].subnr != half_offset,
                     "Bfloat16 source must have register offset 0 or half of GRF register.");
         }
      }
   }

   const unsigned opcode = inst->opcode;
   if (inst->num_sources >= 3)
      return;

   if (!is_mixed_float(inst))
      return;

   bool is_align16 = inst->access_mode == BRW_ALIGN_16;

   enum brw_reg_type src0_type = inst->src[0].type;
   enum brw_reg_type src1_type = inst->num_sources > 1 ?
                                 inst->src[1].type : 0;
   enum brw_reg_type dst_type = inst->dst.type;

   unsigned dst_stride = inst->dst.hstride;
   bool dst_is_packed = is_packed(inst->exec_size * dst_stride, inst->exec_size, dst_stride);

   /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
    * Float Operations:
    *
    *    "Indirect addressing on source is not supported when source and
    *     destination data types are mixed float."
    */
   ERROR_IF(inst->src[0].address_mode != BRW_ADDRESS_DIRECT ||
            (inst->num_sources > 1 &&
             inst->src[1].address_mode != BRW_ADDRESS_DIRECT),
            "Indirect addressing on source is not supported when source and "
            "destination data types are mixed float");

   /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
    * Float Operations:
    *
    *    "No SIMD16 in mixed mode when destination is f32. Instruction
    *     execution size must be no more than 8."
    */
   ERROR_IF(inst->exec_size > 8 && devinfo->ver < 20 &&
            dst_type == BRW_TYPE_F &&
            opcode != BRW_OPCODE_MOV,
            "Mixed float mode with 32-bit float destination is limited "
            "to SIMD8");

   if (is_align16) {
      /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
       * Float Operations:
       *
       *   "In Align16 mode, when half float and float data types are mixed
       *    between source operands OR between source and destination operands,
       *    the register content are assumed to be packed."
       *
       * Since Align16 doesn't have a concept of horizontal stride (or width),
       * it means that vertical stride must always be 4, since 0 and 2 would
       * lead to replicated data, and any other value is disallowed in Align16.
       */
      ERROR_IF(inst->src[0].vstride != 4,
               "Align16 mixed float mode assumes packed data (vstride must be 4");

      ERROR_IF(inst->num_sources >= 2 &&
               inst->src[1].vstride != 4,
               "Align16 mixed float mode assumes packed data (vstride must be 4");

      /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
       * Float Operations:
       *
       *   "For Align16 mixed mode, both input and output packed f16 data
       *    must be oword aligned, no oword crossing in packed f16."
       *
       * The previous rule requires that Align16 operands are always packed,
       * and since there is only one bit for Align16 subnr, which represents
       * offsets 0B and 16B, this rule is always enforced and we don't need to
       * validate it.
       */

      /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
       * Float Operations:
       *
       *    "No SIMD16 in mixed mode when destination is packed f16 for both
       *     Align1 and Align16."
       *
       * And:
       *
       *   "In Align16 mode, when half float and float data types are mixed
       *    between source operands OR between source and destination operands,
       *    the register content are assumed to be packed."
       *
       * Which implies that SIMD16 is not available in Align16. This is further
       * confirmed by:
       *
       *    "For Align16 mixed mode, both input and output packed f16 data
       *     must be oword aligned, no oword crossing in packed f16"
       *
       * Since oword-aligned packed f16 data would cross oword boundaries when
       * the execution size is larger than 8.
       */
      ERROR_IF(inst->exec_size > 8, "Align16 mixed float mode is limited to SIMD8");

      /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
       * Float Operations:
       *
       *    "No accumulator read access for Align16 mixed float."
       */
      ERROR_IF(inst_uses_src_acc(isa, inst),
               "No accumulator read access for Align16 mixed float");
   } else {
      assert(!is_align16);

      /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
       * Float Operations:
       *
       *    "No SIMD16 in mixed mode when destination is packed f16 for both
       *     Align1 and Align16."
       */
      ERROR_IF(inst->exec_size > 8 && dst_is_packed &&
               dst_type == BRW_TYPE_HF &&
               opcode != BRW_OPCODE_MOV,
               "Align1 mixed float mode is limited to SIMD8 when destination "
               "is packed half-float");

      /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
       * Float Operations:
       *
       *    "Math operations for mixed mode:
       *     - In Align1, f16 inputs need to be strided"
       */
      if (opcode == BRW_OPCODE_MATH) {
         if (src0_type == BRW_TYPE_HF) {
            ERROR_IF(inst->src[0].hstride <= 1,
                     "Align1 mixed mode math needs strided half-float inputs");
         }

         if (inst->num_sources >= 2 && src1_type == BRW_TYPE_HF) {
            ERROR_IF(inst->src[1].hstride <= 1,
                     "Align1 mixed mode math needs strided half-float inputs");
         }
      }

      if (dst_type == BRW_TYPE_HF && dst_stride == 1) {
         /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
          * Float Operations:
          *
          *    "In Align1, destination stride can be smaller than execution
          *     type. When destination is stride of 1, 16 bit packed data is
          *     updated on the destination. However, output packed f16 data
          *     must be oword aligned, no oword crossing in packed f16."
          *
          * The requirement of not crossing oword boundaries for 16-bit oword
          * aligned data means that execution size is limited to 8.
          */
         ERROR_IF(inst->dst.subnr % 16 != 0,
                  "Align1 mixed mode packed half-float output must be "
                  "oword aligned");
         ERROR_IF(inst->exec_size > 8,
                  "Align1 mixed mode packed half-float output must not "
                  "cross oword boundaries (max exec size is 8)");

         /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
          * Float Operations:
          *
          *    "When source is float or half float from accumulator register and
          *     destination is half float with a stride of 1, the source must
          *     register aligned. i.e., source must have offset zero."
          *
          * Align16 mixed float mode doesn't allow accumulator access on sources,
          * so we only need to check this for Align1.
          */
         if (src0_is_acc(inst) &&
             (src0_type == BRW_TYPE_F ||
              src0_type == BRW_TYPE_HF)) {
            ERROR_IF(inst->src[0].subnr != 0,
                     "Mixed float mode requires register-aligned accumulator "
                     "source reads when destination is packed half-float");

         }

         if (inst->num_sources > 1 &&
             src1_is_acc(inst) &&
             (src1_type == BRW_TYPE_F ||
              src1_type == BRW_TYPE_HF)) {
            ERROR_IF(inst->src[1].subnr != 0,
                     "Mixed float mode requires register-aligned accumulator "
                     "source reads when destination is packed half-float");
         }
      }

      /* From the SKL PRM, Special Restrictions for Handling Mixed Mode
       * Float Operations:
       *
       *    "No swizzle is allowed when an accumulator is used as an implicit
       *     source or an explicit source in an instruction. i.e. when
       *     destination is half float with an implicit accumulator source,
       *     destination stride needs to be 2."
       *
       * FIXME: it is not quite clear what the first sentence actually means
       *        or its link to the implication described after it, so we only
       *        validate the explicit implication, which is clearly described.
       */
      if (dst_type == BRW_TYPE_HF &&
          inst_uses_src_acc(isa, inst)) {
         ERROR_IF(dst_stride != 2,
                  "Mixed float mode with implicit/explicit accumulator "
                  "source and half-float destination requires a stride "
                  "of 2 on the destination");
      }
   }
}

/**
 * Creates a \p grf_access_mask for an \p exec_size, \p element_size, and a
 * region
 *
 * A \p grf_access_mask is a 32-element array of uint8_t, where each uint8_t
 * is a bitmask of grfs accessed by the region.
 *
 * For instance the access mask of the source gX.1<4,2,2>F in an exec_size = 4
 * instruction would be
 *
 *    access_mask[0] = 0x01 (bytes 7-4 of the 1st grf)
 *    access_mask[1] = 0x01 (bytes 15-12 of the 1st grf)
 *    access_mask[2] = 0x01 (bytes 23-20 of the 1st grf)
 *    access_mask[3] = 0x01 (bytes 31-28 of the 1st grf)
 *    access_mask[4-31] = 0
 *
 * Before Xe2, gX<1,1,0>F in an exec_size == 16 would yield:
 *
 *    access_mask[0] = 0x01 (bytes 3-0 of the 1st grf)
 *    access_mask[1] = 0x01 (bytes 7-4 of the 1st grf)
 *      ...
 *    access_mask[7] = 0x01 (bytes 31-28 of the 1st grf)
 *    access_mask[8] = 0x02 (bytes 3-0 of the 2nd grf)
 *      ...
 *    access_mask[15] = 0x02 (bytes 31-28 of the 2nd grf)
 *    access_mask[16-31] = 0
 *
 * Whereas on Xe2, gX<1,1,0>F in an exec_size of 16 would yield:
 *
 *    access_mask[0] = 0x01 (bytes 3-0 of the 1st grf)
 *    access_mask[1] = 0x01 (bytes 7-4 of the 1st grf)
 *      ...
 *    access_mask[7] = 0x01 (bytes 31-28 of the 1st grf)
 *    access_mask[8] = 0x01 (bytes 35-32 of the 1st grf)
 *      ...
 *    access_mask[15] = 0x01 (bytes 63-60 of the 1st grf)
 *    access_mask[4-31] = 0
 *
 */
static void
grfs_accessed(const struct intel_device_info *devinfo,
              uint8_t grf_access_mask[static 32],
              unsigned exec_size, unsigned element_size, unsigned subreg,
              unsigned vstride, unsigned width, unsigned hstride)
{
   unsigned rowbase = subreg;
   unsigned element = 0;
   assert(util_is_power_of_two_nonzero(reg_unit(devinfo)));
   unsigned grf_size_shift = (5 - 1) + ffs(reg_unit(devinfo));

   for (int y = 0; y < exec_size / width; y++) {
      unsigned offset = rowbase;

      for (int x = 0; x < width; x++) {
         const unsigned start_grf = (offset >> grf_size_shift) % 8;
         const unsigned end_byte = offset + (element_size - 1);
         const unsigned end_grf = (end_byte >> grf_size_shift) % 8;
         grf_access_mask[element++] = (1 << start_grf) | (1 << end_grf);
         offset += hstride * element_size;
      }

      rowbase += vstride * element_size;
   }

   assert(element == 0 || element == exec_size);
}

/**
 * Returns the number of registers accessed according to the \p access_mask
 */
static int
registers_read(const uint8_t grfs_accessed[static 32])
{
   uint8_t all_read = 0;

   for (unsigned i = 0; i < 32; i++)
      all_read |= grfs_accessed[i];

   return util_bitcount(all_read);
}

/**
 * Checks restrictions listed in "Region Alignment Rules" in the "Register
 * Region Restrictions" section.
 */
static void
region_alignment_rules(const struct brw_isa_info *isa,
                       const brw_hw_decoded_inst *inst,
                       struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;
   uint8_t dst_access_mask[32] = {}, src_access_mask[2][32] = {};

   if (inst->num_sources == 3)
      return;

   if (inst->access_mode == BRW_ALIGN_16)
      return;

   if (inst_is_send(inst))
      return;

   bool skip_detailed_grf_checks = false;

   for (unsigned i = 0; i < inst->num_sources; i++) {
      /* In Direct Addressing mode, a source cannot span more than 2 adjacent
       * GRF registers.
       */
      if (inst->src[i].file != FIXED_GRF ||
          inst->src[i].address_mode != BRW_ADDRESS_DIRECT)
         continue;

      enum brw_reg_type type = inst->src[i].type;
      unsigned element_size = brw_type_size_bytes(type);
      unsigned subreg = inst->src[i].subnr;
      unsigned vstride = inst->src[i].vstride;
      unsigned width = inst->src[i].width;
      unsigned hstride = inst->src[i].hstride;

      grfs_accessed(devinfo, src_access_mask[i],
                    inst->exec_size, element_size, subreg,
                    vstride, width, hstride);

      unsigned num_vstride = inst->exec_size / width;
      unsigned num_hstride = width;
      unsigned vstride_elements = (num_vstride - 1) * vstride;
      unsigned hstride_elements = (num_hstride - 1) * hstride;
      unsigned offset = (vstride_elements + hstride_elements) * element_size +
                        subreg;

      if (offset >= 64 * reg_unit(devinfo)) {
         ERROR("A source cannot span more than 2 adjacent GRF registers");
         skip_detailed_grf_checks = true;
      }
   }

   if (!inst->has_dst || dst_is_null(inst))
      return;

   unsigned stride = inst->dst.hstride;
   enum brw_reg_type dst_type = inst->dst.type;
   unsigned element_size = brw_type_size_bytes(dst_type);
   unsigned subreg = inst->dst.subnr;
   unsigned offset = ((inst->exec_size - 1) * stride * element_size) + subreg;
   if (offset >= 64 * reg_unit(devinfo)) {
      ERROR("A destination cannot span more than 2 adjacent GRF registers");
      skip_detailed_grf_checks = true;
   }

   if (skip_detailed_grf_checks)
      return;

   grfs_accessed(devinfo, dst_access_mask, inst->exec_size, element_size, subreg,
                 inst->exec_size == 1 ? 0 : inst->exec_size * stride,
                 inst->exec_size == 1 ? 1 : inst->exec_size,
                 inst->exec_size == 1 ? 0 : stride);

   unsigned dst_regs = registers_read(dst_access_mask);

   /* The SKL PRM says:
    *
    *    When destination of MATH instruction spans two registers, the
    *    destination elements must be evenly split between the two registers.
    *
    * It is not known whether this restriction applies to KBL other Gens after
    * SKL.
    */
   if (inst->opcode == BRW_OPCODE_MATH) {
      if (dst_regs == 2) {
         unsigned upper_reg_writes = 0, lower_reg_writes = 0;

         for (unsigned i = 0; i < inst->exec_size; i++) {
            if (dst_access_mask[i] == 2) {
               upper_reg_writes++;
            } else {
               assert(dst_access_mask[i] == 1);
               lower_reg_writes++;
            }
         }

         ERROR_IF(upper_reg_writes != lower_reg_writes,
                  "Writes must be evenly split between the two "
                  "destination registers");
      }
   }
}

static void
vector_immediate_restrictions(const struct brw_isa_info *isa,
                              const brw_hw_decoded_inst *inst,
                              struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;


   if (inst->num_sources == 3 || inst->num_sources == 0 ||
       (devinfo->ver >= 12 && inst_is_send(inst)))
      return;

   unsigned file = inst->src[inst->num_sources == 1 ? 0 : 1].file;
   if (file != IMM)
      return;

   enum brw_reg_type dst_type = inst->dst.type;
   unsigned dst_type_size = brw_type_size_bytes(dst_type);
   unsigned dst_subreg = inst->dst.subnr;
   unsigned dst_stride = inst->dst.hstride;
   enum brw_reg_type type = inst->src[inst->num_sources == 1 ? 0 : 1].type;

   /* The PRMs say:
    *
    *    When an immediate vector is used in an instruction, the destination
    *    must be 128-bit aligned with destination horizontal stride equivalent
    *    to a word for an immediate integer vector (v) and equivalent to a
    *    DWord for an immediate float vector (vf).
    *
    * The text has not been updated for the addition of the immediate unsigned
    * integer vector type (uv) on SNB, but presumably the same restriction
    * applies.
    */
   switch (type) {
   case BRW_TYPE_V:
   case BRW_TYPE_UV:
   case BRW_TYPE_VF:
      ERROR_IF(dst_subreg % (128 / 8) != 0,
               "Destination must be 128-bit aligned in order to use immediate "
               "vector types");

      if (type == BRW_TYPE_VF) {
         ERROR_IF(dst_type_size * dst_stride != 4,
                  "Destination must have stride equivalent to dword in order "
                  "to use the VF type");
      } else {
         ERROR_IF(dst_type_size * dst_stride != 2,
                  "Destination must have stride equivalent to word in order "
                  "to use the V or UV type");
      }
      break;
   default:
      break;
   }
}

static void
special_requirements_for_handling_double_precision_data_types(
                                       const struct brw_isa_info *isa,
                                       const brw_hw_decoded_inst *inst,
                                       struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   if (inst->num_sources == 3 || inst->num_sources == 0)
      return;

   /* Split sends don't have types so there's no doubles there. */
   if (inst_is_split_send(isa, inst))
      return;

   enum brw_reg_type exec_type = execution_type(inst);
   unsigned exec_type_size = brw_type_size_bytes(exec_type);

   enum brw_reg_type dst_type = inst->dst.type;
   unsigned dst_type_size = brw_type_size_bytes(dst_type);
   unsigned dst_hstride = inst->dst.hstride;
   unsigned dst_reg = inst->dst.nr;
   unsigned dst_subreg = inst->dst.subnr;
   unsigned dst_address_mode = inst->dst.address_mode;

   bool is_integer_dword_multiply =
      inst->opcode == BRW_OPCODE_MUL &&
      (inst->src[0].type == BRW_TYPE_D || inst->src[0].type == BRW_TYPE_UD) &&
      (inst->src[1].type == BRW_TYPE_D || inst->src[1].type == BRW_TYPE_UD);

   const bool is_double_precision =
      dst_type_size == 8 || exec_type_size == 8 || is_integer_dword_multiply;

   for (unsigned i = 0; i < inst->num_sources; i++) {
      enum brw_reg_file file = inst->src[i].file;
      if (file == IMM)
         continue;

      enum brw_reg_type type = inst->src[i].type;
      unsigned type_size = brw_type_size_bytes(type);
      unsigned address_mode = inst->src[i].address_mode;
      unsigned reg = inst->src[i].nr;
      unsigned subreg = inst->src[i].subnr;
      bool is_scalar_region = src_has_scalar_region(inst, i);
      unsigned vstride = inst->src[i].vstride;
      unsigned width = inst->src[i].width;
      unsigned hstride = inst->src[i].hstride;

      const unsigned src_stride = (hstride ? hstride : vstride) * type_size;
      const unsigned dst_stride = dst_hstride * dst_type_size;

      /* The PRMs say that for CHV, BXT:
       *
       *    When source or destination datatype is 64b or operation is integer
       *    DWord multiply, regioning in Align1 must follow these rules:
       *
       *    1. Source and Destination horizontal stride must be aligned to the
       *       same qword.
       *    2. Regioning must ensure Src.Vstride = Src.Width * Src.Hstride.
       *    3. Source and Destination offset must be the same, except the case
       *       of scalar source.
       *
       * We assume that the restriction applies to GLK as well.
       */
      if (is_double_precision &&
          inst->access_mode == BRW_ALIGN_1 &&
          intel_device_info_is_9lp(devinfo)) {
         ERROR_IF(!is_scalar_region &&
                  (src_stride % 8 != 0 ||
                   dst_stride % 8 != 0 ||
                   src_stride != dst_stride),
                  "Source and destination horizontal stride must equal and a "
                  "multiple of a qword when the execution type is 64-bit");

         ERROR_IF(vstride != width * hstride,
                  "Vstride must be Width * Hstride when the execution type is "
                  "64-bit");

         ERROR_IF(!is_scalar_region && dst_subreg != subreg,
                  "Source and destination offset must be the same when the "
                  "execution type is 64-bit");
      }

      /* The PRMs say that for CHV, BXT:
       *
       *    When source or destination datatype is 64b or operation is integer
       *    DWord multiply, indirect addressing must not be used.
       *
       * We assume that the restriction applies to GLK as well.
       */
      if (is_double_precision &&
          intel_device_info_is_9lp(devinfo)) {
         ERROR_IF(BRW_ADDRESS_REGISTER_INDIRECT_REGISTER == address_mode ||
                  BRW_ADDRESS_REGISTER_INDIRECT_REGISTER == dst_address_mode,
                  "Indirect addressing is not allowed when the execution type "
                  "is 64-bit");
      }

      /* The PRMs say that for CHV, BXT:
       *
       *    ARF registers must never be used with 64b datatype or when
       *    operation is integer DWord multiply.
       *
       * We assume that the restriction applies to GLK as well.
       *
       * We assume that the restriction does not apply to the null register.
       */
      if (is_double_precision &&
          intel_device_info_is_9lp(devinfo)) {
         ERROR_IF(inst->opcode == BRW_OPCODE_MAC ||
                  brw_eu_inst_acc_wr_control(devinfo, inst->raw) ||
                  (ARF == file &&
                   reg != BRW_ARF_NULL) ||
                  (ARF == inst->dst.file &&
                   dst_reg != BRW_ARF_NULL),
                  "Architecture registers cannot be used when the execution "
                  "type is 64-bit");
      }

      /* From the hardware spec section "Register Region Restrictions":
       *
       * There are two rules:
       *
       * "In case of all floating point data types used in destination:" and
       *
       * "In case where source or destination datatype is 64b or operation is
       *  integer DWord multiply:"
       *
       * both of which list the same restrictions:
       *
       *  "1. Register Regioning patterns where register data bit location
       *      of the LSB of the channels are changed between source and
       *      destination are not supported on Src0 and Src1 except for
       *      broadcast of a scalar.
       *
       *   2. Explicit ARF registers except null and accumulator must not be
       *      used."
       */
      if (devinfo->verx10 >= 125 &&
          (brw_type_is_float(dst_type) ||
           is_double_precision)) {
         ERROR_IF(!brw_type_is_bfloat(type) &&
                  !is_scalar_region &&
                  BRW_ADDRESS_REGISTER_INDIRECT_REGISTER != address_mode &&
                  (!is_linear(vstride, width, hstride) ||
                   src_stride != dst_stride ||
                   subreg != dst_subreg),
                  "Register Regioning patterns where register data bit "
                  "location of the LSB of the channels are changed between "
                  "source and destination are not supported except for "
                  "broadcast of a scalar.");

         /* NOTE: Expanded this to include Scalar.  See documentation issue
          * open in https://gfxspecs.intel.com/Predator/Home/Index/56640.
          */
         ERROR_IF((address_mode == BRW_ADDRESS_DIRECT && file == ARF &&
                   reg != BRW_ARF_SCALAR &&
                   reg != BRW_ARF_NULL && !(reg >= BRW_ARF_ACCUMULATOR && reg < BRW_ARF_FLAG)) ||
                  (inst->dst.file == ARF &&
                   dst_reg != BRW_ARF_SCALAR &&
                   dst_reg != BRW_ARF_NULL && (dst_reg & 0xF0) != BRW_ARF_ACCUMULATOR),
                  "Explicit ARF registers except null, accumulator, and scalar must not "
                  "be used.");
      }

      /* From the hardware spec section "Register Region Restrictions":
       *
       * "Vx1 and VxH indirect addressing for Float, Half-Float, Double-Float and
       *  Quad-Word data must not be used."
       *
       * and
       *
       * "Vx1 and VxH indirect addressing for BFloat16 (...) data
       *  must not be used."
       */
      if (devinfo->verx10 >= 125 &&
          (brw_type_is_float_or_bfloat(type) || brw_type_size_bytes(type) == 8)) {
         ERROR_IF(address_mode == BRW_ADDRESS_REGISTER_INDIRECT_REGISTER &&
                  vstride == BRW_VERTICAL_STRIDE_ONE_DIMENSIONAL,
                  "Vx1 and VxH indirect addressing for Float, Half-Float, "
                  "Double-Float, Quad-Word, and Bfloat16 data must not be used");
      }
   }

   /* The PRMs say that for BDW, SKL:
    *
    *    If Align16 is required for an operation with QW destination and non-QW
    *    source datatypes, the execution size cannot exceed 2.
    *
    * We assume that the restriction applies to all Gfx8+ parts.
    */
   if (is_double_precision) {
      enum brw_reg_type src0_type = inst->src[0].type;
      enum brw_reg_type src1_type =
         inst->num_sources > 1 ? inst->src[1].type : src0_type;
      unsigned src0_type_size = brw_type_size_bytes(src0_type);
      unsigned src1_type_size = brw_type_size_bytes(src1_type);

      ERROR_IF(inst->access_mode == BRW_ALIGN_16 &&
               dst_type_size == 8 &&
               (src0_type_size != 8 || src1_type_size != 8) &&
               inst->exec_size > 2,
               "In Align16 exec size cannot exceed 2 with a QWord destination "
               "and a non-QWord source");
   }

   /* The PRMs say that for CHV, BXT:
    *
    *    When source or destination datatype is 64b or operation is integer
    *    DWord multiply, DepCtrl must not be used.
    *
    * We assume that the restriction applies to GLK as well.
    */
   if (is_double_precision &&
       intel_device_info_is_9lp(devinfo)) {
      ERROR_IF(brw_eu_inst_no_dd_check(devinfo, inst->raw) ||
               brw_eu_inst_no_dd_clear(devinfo, inst->raw),
               "DepCtrl is not allowed when the execution type is 64-bit");
   }
}

static void
instruction_restrictions(const struct brw_isa_info *isa,
                         const brw_hw_decoded_inst *inst,
                         struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   /* From Wa_1604601757:
    *
    * "When multiplying a DW and any lower precision integer, source modifier
    *  is not supported."
    */
   if (devinfo->ver >= 12 &&
       inst->opcode == BRW_OPCODE_MUL) {
      enum brw_reg_type exec_type = execution_type(inst);
      const bool src0_valid =
         brw_type_size_bytes(inst->src[0].type) == 4 ||
         inst->src[0].file == IMM ||
         !(inst->src[0].negate || inst->src[0].abs);
      const bool src1_valid =
         brw_type_size_bytes(inst->src[1].type) == 4 ||
         inst->src[1].file == IMM ||
         !(inst->src[1].negate || inst->src[1].abs);

      ERROR_IF(!brw_type_is_float(exec_type) &&
               brw_type_size_bytes(exec_type) == 4 &&
               !(src0_valid && src1_valid),
               "When multiplying a DW and any lower precision integer, source "
               "modifier is not supported.");
   }

   if (inst->opcode == BRW_OPCODE_CMP ||
       inst->opcode == BRW_OPCODE_CMPN) {
      ERROR_IF(inst->cond_modifier == BRW_CONDITIONAL_NONE,
               "CMP (or CMPN) must have a condition.");
   }

   if (inst->opcode == BRW_OPCODE_SEL) {
      ERROR_IF((inst->cond_modifier != BRW_CONDITIONAL_NONE) ==
               (inst->pred_control != BRW_PREDICATE_NONE),
               "SEL must either be predicated or have a condition modifiers");
   }

   if (inst->opcode == BRW_OPCODE_MUL) {
      const enum brw_reg_type src0_type = inst->src[0].type;
      const enum brw_reg_type src1_type = inst->src[1].type;
      const enum brw_reg_type dst_type = inst->dst.type;

      /* Page 966 (page 982 of the PDF) of Broadwell PRM volume 2a says:
       *
       *    When multiplying a DW and any lower precision integer, the DW
       *    operand must on src0.
       *
       * Ivy Bridge, Haswell, Skylake, and Ice Lake PRMs contain the same
       * text.
       */
      ERROR_IF(brw_type_is_int(src1_type) &&
               brw_type_size_bytes(src0_type) < 4 &&
               brw_type_size_bytes(src1_type) == 4,
               "When multiplying a DW and any lower precision integer, the "
               "DW operand must be src0.");

      /* Page 971 (page 987 of the PDF), section "Accumulator
       * Restrictions," of the Broadwell PRM volume 7 says:
       *
       *    Integer source operands cannot be accumulators.
       *
       * The Skylake and Ice Lake PRMs contain the same text.
       */
      ERROR_IF((src0_is_acc(inst) &&
                brw_type_is_int(src0_type)) ||
               (src1_is_acc(inst) &&
                brw_type_is_int(src1_type)),
               "Integer source operands cannot be accumulators.");

      /* Page 935 (page 951 of the PDF) of the Ice Lake PRM volume 2a says:
       *
       *    When multiplying integer data types, if one of the sources is a
       *    DW, the resulting full precision data is stored in the
       *    accumulator. However, if the destination data type is either W or
       *    DW, the low bits of the result are written to the destination
       *    register and the remaining high bits are discarded. This results
       *    in undefined Overflow and Sign flags. Therefore, conditional
       *    modifiers and saturation (.sat) cannot be used in this case.
       *
       * Similar text appears in every version of the PRM.
       *
       * The wording of the last sentence is not very clear.  It could either
       * be interpreted as "conditional modifiers combined with saturation
       * cannot be used" or "neither conditional modifiers nor saturation can
       * be used."  I have interpreted it as the latter primarily because that
       * is the more restrictive interpretation.
       */
      ERROR_IF((src0_type == BRW_TYPE_UD ||
                src0_type == BRW_TYPE_D ||
                src1_type == BRW_TYPE_UD ||
                src1_type == BRW_TYPE_D) &&
               (dst_type == BRW_TYPE_UD ||
                dst_type == BRW_TYPE_D ||
                dst_type == BRW_TYPE_UW ||
                dst_type == BRW_TYPE_W) &&
               (inst->saturate || inst->cond_modifier != BRW_CONDITIONAL_NONE),
               "Neither Saturate nor conditional modifier allowed with DW "
               "integer multiply.");
   }

   if (inst->opcode == BRW_OPCODE_MATH) {
      unsigned math_function = brw_eu_inst_math_function(devinfo, inst->raw);
      switch (math_function) {
      case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER:
      case BRW_MATH_FUNCTION_INT_DIV_QUOTIENT:
      case BRW_MATH_FUNCTION_INT_DIV_REMAINDER: {
         /* Page 442 of the Broadwell PRM Volume 2a "Extended Math Function" says:
          *    INT DIV function does not support source modifiers.
          * Bspec 6647 extends it back to Ivy Bridge.
          */
         bool src0_valid = !inst->src[0].negate && !inst->src[0].abs;
         bool src1_valid = !inst->src[1].negate && !inst->src[1].abs;
         ERROR_IF(!src0_valid || !src1_valid,
                  "INT DIV function does not support source modifiers.");
         break;
      }
      default:
         break;
      }
   }

   if (inst->opcode == BRW_OPCODE_DP4A) {
      /* Page 396 (page 412 of the PDF) of the DG1 PRM volume 2a says:
       *
       *    Only one of src0 or src1 operand may be an the (sic) accumulator
       *    register (acc#).
       */
      ERROR_IF(src0_is_acc(inst) && src1_is_acc(inst),
               "Only one of src0 or src1 operand may be an accumulator "
               "register (acc#).");

   }

   if (inst->opcode == BRW_OPCODE_ADD3) {
      const enum brw_reg_type dst_type = inst->dst.type;

      ERROR_IF(dst_type != BRW_TYPE_D &&
               dst_type != BRW_TYPE_UD &&
               dst_type != BRW_TYPE_W &&
               dst_type != BRW_TYPE_UW,
               "Destination must be integer D, UD, W, or UW type.");

      for (unsigned i = 0; i < 3; i++) {
         enum brw_reg_type src_type = inst->src[i].type;

         ERROR_IF(src_type != BRW_TYPE_D &&
                  src_type != BRW_TYPE_UD &&
                  src_type != BRW_TYPE_W &&
                  src_type != BRW_TYPE_UW,
                  "Source must be integer D, UD, W, or UW type.");

         ERROR_IF(inst->src[i].file == IMM &&
                  src_type != BRW_TYPE_W &&
                  src_type != BRW_TYPE_UW,
                  "Immediate source must be integer W or UW type.");
      }
   }

   if (inst->opcode == BRW_OPCODE_OR ||
       inst->opcode == BRW_OPCODE_AND ||
       inst->opcode == BRW_OPCODE_XOR ||
       inst->opcode == BRW_OPCODE_NOT) {
      /* While the behavior of the negate source modifier is defined as
       * logical not, the behavior of abs source modifier is not
       * defined. Disallow it to be safe.
       */
      ERROR_IF(inst->src[0].abs,
               "Behavior of abs source modifier in logic ops is undefined.");
      ERROR_IF(inst->opcode != BRW_OPCODE_NOT &&
               inst->src[1].file != IMM &&
               inst->src[1].abs,
               "Behavior of abs source modifier in logic ops is undefined.");

      /* Page 479 (page 495 of the PDF) of the Broadwell PRM volume 2a says:
       *
       *    Source modifier is not allowed if source is an accumulator.
       *
       * The same text also appears for OR, NOT, and XOR instructions.
       */
      ERROR_IF((inst->src[0].abs || inst->src[0].negate) &&
               src0_is_acc(inst),
               "Source modifier is not allowed if source is an accumulator.");
      ERROR_IF(inst->num_sources > 1 &&
               (inst->src[1].abs || inst->src[1].negate) &&
               src1_is_acc(inst),
               "Source modifier is not allowed if source is an accumulator.");

      /* Page 479 (page 495 of the PDF) of the Broadwell PRM volume 2a says:
       *
       *    This operation does not produce sign or overflow conditions. Only
       *    the .e/.z or .ne/.nz conditional modifiers should be used.
       *
       * The same text also appears for OR, NOT, and XOR instructions.
       *
       * Per the comment around nir_op_imod in brw_from_nir.cpp, we have
       * determined this to not be true. The only conditions that seem
       * absolutely sketchy are O, R, and U.  Some OpenGL shaders from Doom
       * 2016 have been observed to generate and.g and operate correctly.
       */
      const enum brw_conditional_mod cmod = inst->cond_modifier;
      ERROR_IF(cmod == BRW_CONDITIONAL_O ||
               cmod == BRW_CONDITIONAL_R ||
               cmod == BRW_CONDITIONAL_U,
               "O, R, and U conditional modifiers should not be used.");
   }

   if (inst->opcode == BRW_OPCODE_BFI2) {
      ERROR_IF(inst->cond_modifier != BRW_CONDITIONAL_NONE,
               "BFI2 cannot have conditional modifier");

      ERROR_IF(inst->saturate,
               "BFI2 cannot have saturate modifier");

      enum brw_reg_type dst_type = inst->dst.type;

      ERROR_IF(dst_type != BRW_TYPE_D &&
               dst_type != BRW_TYPE_UD,
               "BFI2 destination type must be D or UD");

      for (unsigned s = 0; s < 3; s++) {
         enum brw_reg_type src_type = inst->src[s].type;

         ERROR_IF(src_type != dst_type,
                  "BFI2 source type must match destination type");
      }
   }

   if (inst->opcode == BRW_OPCODE_CSEL) {
      ERROR_IF(inst->pred_control != BRW_PREDICATE_NONE,
               "CSEL cannot be predicated");

      /* CSEL is CMP and SEL fused into one. The condition modifier, which
       * does not actually modify the flags, controls the built-in comparison.
       */
      ERROR_IF(inst->cond_modifier == BRW_CONDITIONAL_NONE,
               "CSEL must have a condition.");

      enum brw_reg_type dst_type = inst->dst.type;

      if (devinfo->ver == 9) {
         ERROR_IF(dst_type != BRW_TYPE_F,
                  "CSEL destination type must be F");
      } else {
         ERROR_IF(dst_type != BRW_TYPE_F &&
                  dst_type != BRW_TYPE_HF &&
                  dst_type != BRW_TYPE_D &&
                  dst_type != BRW_TYPE_W &&
                  dst_type != BRW_TYPE_UD &&
                  dst_type != BRW_TYPE_UW,
                  "CSEL destination type must be F, HF, *D, or *W");
      }

      for (unsigned s = 0; s < 3; s++) {
         enum brw_reg_type src_type = inst->src[s].type;

         if (devinfo->ver == 9) {
            ERROR_IF(src_type != BRW_TYPE_F,
                     "CSEL source type must be F");
         } else {
            ERROR_IF(src_type != BRW_TYPE_F && src_type != BRW_TYPE_HF &&
                     src_type != BRW_TYPE_D && src_type != BRW_TYPE_UD &&
                     src_type != BRW_TYPE_W && src_type != BRW_TYPE_UW,
                     "CSEL source type must be F, HF, *D, or *W");

            ERROR_IF(brw_type_is_float(src_type) != brw_type_is_float(dst_type),
                     "CSEL cannot mix float and integer types.");

            ERROR_IF(brw_type_size_bytes(src_type) !=
                     brw_type_size_bytes(dst_type),
                     "CSEL cannot mix different type sizes.");
         }
      }
   }

   if (inst->opcode == BRW_OPCODE_DPAS) {
      ERROR_IF(brw_eu_inst_dpas_3src_sdepth(devinfo, inst->raw) != BRW_SYSTOLIC_DEPTH_8,
               "Systolic depth must be 8.");

      const unsigned sdepth = 8;

      const enum brw_reg_type dst_type = inst->dst.type;
      const enum brw_reg_type src0_type = inst->src[0].type;
      const enum brw_reg_type src1_type = inst->src[1].type;
      const enum brw_reg_type src2_type = inst->src[2].type;

      const enum gfx12_sub_byte_precision src1_sub_byte =
         brw_eu_inst_dpas_3src_src1_subbyte(devinfo, inst->raw);

      if (src1_type != BRW_TYPE_B && src1_type != BRW_TYPE_UB) {
         ERROR_IF(src1_sub_byte != BRW_SUB_BYTE_PRECISION_NONE,
                  "Sub-byte precision must be None for source type larger than Byte.");
      } else {
         ERROR_IF(src1_sub_byte != BRW_SUB_BYTE_PRECISION_NONE &&
                  src1_sub_byte != BRW_SUB_BYTE_PRECISION_4BIT &&
                  src1_sub_byte != BRW_SUB_BYTE_PRECISION_2BIT,
                  "Invalid sub-byte precision.");
      }

      const enum gfx12_sub_byte_precision src2_sub_byte =
         brw_eu_inst_dpas_3src_src2_subbyte(devinfo, inst->raw);

      if (src2_type != BRW_TYPE_B && src2_type != BRW_TYPE_UB) {
         ERROR_IF(src2_sub_byte != BRW_SUB_BYTE_PRECISION_NONE,
                  "Sub-byte precision must be None.");
      } else {
         ERROR_IF(src2_sub_byte != BRW_SUB_BYTE_PRECISION_NONE &&
                  src2_sub_byte != BRW_SUB_BYTE_PRECISION_4BIT &&
                  src2_sub_byte != BRW_SUB_BYTE_PRECISION_2BIT,
                  "Invalid sub-byte precision.");
      }

      const unsigned src1_bits_per_element =
         brw_type_size_bits(src1_type) >>
         brw_eu_inst_dpas_3src_src1_subbyte(devinfo, inst->raw);

      const unsigned src2_bits_per_element =
         brw_type_size_bits(src2_type) >>
         brw_eu_inst_dpas_3src_src2_subbyte(devinfo, inst->raw);

      /* The MAX2(1, ...) is just to prevent possible division by 0 later. */
      const unsigned ops_per_chan =
         MAX2(1, 32 / MAX2(src1_bits_per_element, src2_bits_per_element));

      if (devinfo->ver < 20) {
         ERROR_IF(inst->exec_size != 8, "DPAS execution size must be 8.");
      } else {
         ERROR_IF(inst->exec_size != 16, "DPAS execution size must be 16.");
      }

      const unsigned dst_subnr  = inst->dst.subnr;
      const unsigned src0_subnr = inst->src[0].subnr;
      const unsigned src1_subnr = inst->src[1].subnr;
      const unsigned src2_subnr = inst->src[2].subnr;

      /* Until HF is supported as dst type, this is effectively subnr == 0. */
      ERROR_IF(dst_subnr % inst->exec_size != 0,
               "Destination subregister offset must be a multiple of ExecSize.");

      /* Until HF is supported as src0 type, this is effectively subnr == 0. */
      ERROR_IF(src0_subnr % inst->exec_size != 0,
               "Src0 subregister offset must be a multiple of ExecSize.");

      ERROR_IF(src1_subnr != 0,
               "Src1 subregister offsets must be 0.");

      /* In nearly all cases, this effectively requires that src2.subnr be
       * 0. It is only when src1 is 8 bits and src2 is 2 or 4 bits that the
       * ops_per_chan value can allow non-zero src2.subnr.
       */
      ERROR_IF(src2_subnr % (sdepth * ops_per_chan) != 0,
               "Src2 subregister offset must be a multiple of SystolicDepth "
               "times OPS_PER_CHAN.");

      ERROR_IF(dst_subnr * brw_type_size_bytes(dst_type) >= REG_SIZE,
               "Destination subregister specifies next register.");

      ERROR_IF(src0_subnr * brw_type_size_bytes(src0_type) >= REG_SIZE,
               "Src0 subregister specifies next register.");

      ERROR_IF((src1_subnr * brw_type_size_bytes(src1_type) * src1_bits_per_element) / 8 >= REG_SIZE,
               "Src1 subregister specifies next register.");

      ERROR_IF((src2_subnr * brw_type_size_bytes(src2_type) * src2_bits_per_element) / 8 >= REG_SIZE,
               "Src2 subregister specifies next register.");

      if (brw_eu_inst_3src_atomic_control(devinfo, inst->raw)) {
         /* FINISHME: When we start emitting DPAS with Atomic set, figure out
          * a way to validate it. Also add a test in test_eu_validate.cpp.
          */
         ERROR_IF(true,
                  "When instruction option Atomic is used it must be follwed by a "
                  "DPAS instruction.");
      }

      if (brw_eu_inst_dpas_3src_exec_type(devinfo, inst->raw) ==
          BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT) {
         ERROR_IF(src1_type != BRW_TYPE_HF &&
                  src1_type != BRW_TYPE_BF,
                  "DPAS src1 type must be HF or BF.");
         ERROR_IF(src2_type != BRW_TYPE_HF &&
                  src2_type != BRW_TYPE_BF,
                  "DPAS src2 type must be HF or BF.");
         ERROR_IF(src1_type != src2_type,
                  "DPAS src1 and src2 with types must match when using float types.");

         if (devinfo->ver < 20) {
            ERROR_IF(dst_type != BRW_TYPE_F,
                     "DPAS destination type must be F in Gfx12.");
            ERROR_IF(src0_type != BRW_TYPE_F,
                     "DPAS src0 type must be F in Gfx12.");
         } else {
            ERROR_IF(dst_type != BRW_TYPE_F &&
                     dst_type != src1_type,
                     "DPAS destination type must be F or match Src1/Src2 in Gfx20+.");
            ERROR_IF(src0_type != BRW_TYPE_F &&
                     src0_type != src1_type,
                     "DPAS src0 type must be F or match Src1/Src2 in Gfx20+.");
         }
      } else {
         ERROR_IF(dst_type != BRW_TYPE_D &&
                  dst_type != BRW_TYPE_UD,
                  "DPAS destination type must be D or UD.");
         ERROR_IF(src0_type != BRW_TYPE_D &&
                  src0_type != BRW_TYPE_UD,
                  "DPAS src0 type must be D or UD.");
         ERROR_IF(src1_type != BRW_TYPE_B &&
                  src1_type != BRW_TYPE_UB,
                  "DPAS src1 base type must be B or UB.");
         ERROR_IF(src2_type != BRW_TYPE_B &&
                  src2_type != BRW_TYPE_UB,
                  "DPAS src2 base type must be B or UB.");

         if (brw_type_is_uint(dst_type)) {
            ERROR_IF(!brw_type_is_uint(src0_type) ||
                     !brw_type_is_uint(src1_type) ||
                     !brw_type_is_uint(src2_type),
                     "If any source datatype is signed, destination datatype "
                     "must be signed.");
         }
      }

      /* FINISHME: Additional restrictions mentioned in the Bspec that are not
       * yet enforced here:
       *
       *    - General Accumulator registers access is not supported. This is
       *      currently enforced in brw_dpas_three_src (brw_eu_emit.c).
       *
       *    - Given any combination of datatypes in the sources of a DPAS
       *      instructions, the boundaries of a register should not be crossed.
       */
   }
}

static void
send_descriptor_restrictions(const struct brw_isa_info *isa,
                             const brw_hw_decoded_inst *inst,
                             struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   if (inst_is_split_send(isa, inst)) {
      /* We can only validate immediate descriptors */
      if (brw_eu_inst_send_sel_reg32_desc(devinfo, inst->raw))
         return;
   } else if (inst_is_send(inst)) {
      /* We can only validate immediate descriptors */
      if (inst->src[1].file != IMM)
         return;
   } else {
      return;
   }

   const uint32_t desc = brw_eu_inst_send_desc(devinfo, inst->raw);

   switch (brw_eu_inst_sfid(devinfo, inst->raw)) {
   case BRW_SFID_URB:
      if (devinfo->ver < 20)
         break;
      FALLTHROUGH;
   case BRW_SFID_TGM:
   case BRW_SFID_SLM:
   case BRW_SFID_UGM:
      ERROR_IF(!devinfo->has_lsc, "Platform does not support LSC");

      ERROR_IF(lsc_opcode_has_transpose(lsc_msg_desc_opcode(devinfo, desc)) &&
               lsc_msg_desc_transpose(devinfo, desc) &&
               inst->exec_size != 1,
               "Transposed vectors are restricted to Exec_Mask = 1.");
      break;

   default:
      break;
   }

   if (brw_eu_inst_sfid(devinfo, inst->raw) == BRW_SFID_URB && devinfo->ver < 20) {
      ERROR_IF(!brw_eu_inst_header_present(devinfo, inst->raw),
               "Header must be present for all URB messages.");

      switch (brw_eu_inst_urb_opcode(devinfo, inst->raw)) {
      case GFX7_URB_OPCODE_ATOMIC_INC:
      case GFX7_URB_OPCODE_ATOMIC_MOV:
      case GFX8_URB_OPCODE_ATOMIC_ADD:
      case GFX8_URB_OPCODE_SIMD8_WRITE:
         break;

      case GFX8_URB_OPCODE_SIMD8_READ:
         ERROR_IF(brw_eu_inst_rlen(devinfo, inst->raw) == 0,
                  "URB SIMD8 read message must read some data.");
         break;

      case GFX125_URB_OPCODE_FENCE:
         ERROR_IF(devinfo->verx10 < 125,
                  "URB fence message only valid on gfx >= 12.5");
         break;

      default:
         ERROR_IF(true, "Invalid URB message");
         break;
      }
   }
}

static void
register_region_special_restrictions(const struct brw_isa_info *isa,
                                     const brw_hw_decoded_inst *inst,
                                     struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   bool format_uses_regions = inst->format == FORMAT_BASIC ||
                              inst->format == FORMAT_BASIC_THREE_SRC;

   /* "Src0 Restrictions" in "Special Restrictions" in Bspec 56640 (r57070). */
   if (devinfo->ver >= 20 &&
       format_uses_regions &&
       inst->num_sources > 0 &&
       inst->src[0].file == FIXED_GRF) {
      const unsigned v = inst->src[0].vstride;
      const unsigned w = inst->src[0].width;
      const unsigned h = inst->src[0].hstride;

      const bool multi_indirect =
         inst->src[0].address_mode == BRW_ADDRESS_REGISTER_INDIRECT_REGISTER &&
         inst->src[0].vstride == STRIDE(BRW_VERTICAL_STRIDE_ONE_DIMENSIONAL);
      const bool is_Vx1 = multi_indirect && w != 1;
      const bool is_VxH = multi_indirect && w == 1;

      const unsigned src0_stride         = w == 1 ? v : h;
      const unsigned src0_uniform_stride = (w == 1) || (h * w == v) || is_Vx1;
      const unsigned dst_stride          = inst->dst.hstride;

      const unsigned src0_size  = brw_type_size_bytes(inst->src[0].type);
      const unsigned dst_size   = brw_type_size_bytes(inst->dst.type);
      const unsigned src0_subnr = inst->src[0].subnr / src0_size;
      const unsigned dst_subnr  = inst->dst.subnr / dst_size;

      const bool dst_dword_aligned = (dst_size >= 4) ||
                                     (dst_size == 2 && (dst_subnr % 2 == 0)) ||
                                     (dst_size == 1 && (dst_subnr % 4 == 0));

      /* The section below follows the pseudo-code in the spec to make
       * easier to verify.
       */
      bool allowed = false;
      if ((dst_size >= 4) ||
          (src0_size >= 4) ||
          (dst_size == 2 && dst_stride > 1) ||
          (dst_size == 1 && dst_stride > 2) ||
          is_VxH) {
         /* One element per DWord channel. */
         allowed = true;

      } else if (src0_uniform_stride || dst_dword_aligned) {
         if (src0_size == 2 && dst_size == 2) {
            if ((src0_stride < 2) ||
                (src0_stride == 2 && src0_uniform_stride && (dst_subnr % 16 == src0_subnr / 2)))
               allowed = true;

         } else if (src0_size == 2 && dst_size == 1 && dst_stride == 2) {
            if ((src0_stride < 2) ||
                (src0_stride == 2 && src0_uniform_stride && (dst_subnr % 32 == src0_subnr)))
               allowed = true;

         } else if (src0_size == 1 && dst_size == 2) {
            if ((src0_stride < 4) ||
                (src0_stride == 4 && src0_uniform_stride && ((2 * dst_subnr) % 16 == src0_subnr / 2)) ||
                (src0_stride == 8 && src0_uniform_stride && ((2 * dst_subnr) % 8 == src0_subnr / 4)))
               allowed = true;

         } else if (src0_size == 1 && dst_size == 1 && dst_stride == 2) {
            if ((src0_stride < 4) ||
                (src0_stride == 4 && src0_uniform_stride && (dst_subnr % 32 == src0_subnr / 2)) ||
                (src0_stride == 8 && src0_uniform_stride && (dst_subnr % 16 == src0_subnr / 4)))
               allowed = true;

         } else if (src0_size == 1 && dst_size == 1 && dst_stride == 1 && w != 2) {
            if ((src0_stride < 2) ||
                (src0_stride == 2 && src0_uniform_stride && (dst_subnr % 32 == src0_subnr / 2)) ||
                (src0_stride == 4 && src0_uniform_stride && (dst_subnr % 16 == src0_subnr / 4)))
               allowed = true;

         } else if (src0_size == 1 && dst_size == 1 && dst_stride == 1 && w == 2) {
            if ((h == 0 && v < 4) ||
                (h == 1 && v < 4) ||
                (h == 2 && v < 2) ||
                (h == 1 && v == 4 && (dst_subnr % 32 == 2 * (src0_subnr / 4)) && (src0_subnr % 2 == 0)) ||
                (h == 2 && v == 4 && (dst_subnr % 32 == src0_subnr / 2)) ||
                (h == 4 && v == 8 && (dst_subnr % 32 == src0_subnr / 4)))
               allowed = true;
         }
      }

      ERROR_IF(!allowed,
               "Invalid register region for source 0.  See special restrictions section.");
   }

   /* "Src1 Restrictions" in "Special Restrictions" in Bspec 56640 (r57070). */
   if (devinfo->ver >= 20 &&
       format_uses_regions &&
       inst->num_sources > 1 &&
       inst->src[1].file == FIXED_GRF) {
      const unsigned v = inst->src[1].vstride;
      const unsigned w = inst->src[1].width;
      const unsigned h = inst->src[1].hstride;

      const bool multi_indirect =
         inst->src[1].address_mode == BRW_ADDRESS_REGISTER_INDIRECT_REGISTER &&
         inst->src[1].vstride == STRIDE(BRW_VERTICAL_STRIDE_ONE_DIMENSIONAL);
      const bool is_Vx1 = multi_indirect && w != 1;

      const unsigned src1_stride         = w == 1 ? v : h;
      const unsigned src1_uniform_stride = (w == 1) || (h * w == v) || is_Vx1;
      const unsigned dst_stride          = inst->dst.hstride;

      const unsigned src1_size  = brw_type_size_bytes(inst->src[1].type);
      const unsigned dst_size   = brw_type_size_bytes(inst->dst.type);
      const unsigned src1_subnr = inst->src[1].subnr / src1_size;
      const unsigned dst_subnr  = inst->dst.subnr / dst_size;

      const bool dst_dword_aligned = (dst_size >= 4) ||
                                     (dst_size == 2 && (dst_subnr % 2 == 0)) ||
                                     (dst_size == 1 && (dst_subnr % 4 == 0));

      /* The section below follows the pseudo-code in the spec to make
       * easier to verify.
       */
      bool allowed = false;
      if ((dst_size >= 4) ||
          (src1_size >= 4) ||
          (dst_size == 2 && dst_stride > 1) ||
          (dst_size == 1 && dst_stride > 2)) {
         /* One element per DWord channel. */
         allowed = true;

      } else if (src1_uniform_stride || dst_dword_aligned) {
         if (src1_size == 2 && dst_size == 2) {
            if ((src1_stride < 2) ||
                (src1_stride == 2 && src1_uniform_stride && (dst_subnr % 16 == src1_subnr / 2)))
               allowed = true;

         } else if (src1_size == 2 && dst_size == 1 && dst_stride == 2) {
            if ((src1_stride < 2) ||
                (src1_stride == 2 && src1_uniform_stride && (dst_subnr % 32 == src1_subnr)))
               allowed = true;
         }
      }

      ERROR_IF(!allowed,
               "Invalid register region for source 1.  See special restrictions section.");
   }
}

static void
scalar_register_restrictions(const struct brw_isa_info *isa,
                             const brw_hw_decoded_inst *inst,
                             struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   /* Restrictions from BSpec 71168 (r55736). */

   if (devinfo->ver >= 30) {
      if (inst->dst.file == ARF && inst->dst.nr == BRW_ARF_SCALAR) {
         switch (inst->opcode) {
         case BRW_OPCODE_MOV: {
            unsigned dst_size_bits = brw_type_size_bits(inst->dst.type);
            ERROR_IF(inst->dst.type != inst->src[0].type,
                     "When destination is scalar register, "
                     "source and destination data-types must be the same.");
            ERROR_IF(!brw_type_is_int(inst->dst.type) || (dst_size_bits != 16 &&
                                                          dst_size_bits != 32 &&
                                                          dst_size_bits != 64),
                     "When destination is scalar register, "
                     "it must be an integer with size 16, 32, or 64 bits.");
            if (inst->src[0].file == IMM) {
               ERROR_IF(inst->exec_size != 1,
                        "When destination is scalar register with immediate source, "
                        "execution size must be 1.");
               ERROR_IF(inst->cond_modifier != BRW_CONDITIONAL_NONE,
                        "When destination is scalar register with immediate source, "
                        "conditional modifier must not be used.");
            }
            ERROR_IF((inst->dst.subnr / 32) != ((inst->dst.subnr + brw_type_size_bytes(inst->dst.type)) / 32),
                     "When destination is scalar register, it must not span across "
                     "the lower to upper 8 dword boundary of the register.");
            break;
         }

         default:
            ERROR("When destination is scalar register, opcode must be MOV.");
            break;
         }
      }

      if (inst->src[0].file == ARF && inst->src[0].nr == BRW_ARF_SCALAR) {
         switch (inst->opcode) {
         case BRW_OPCODE_MOV: {
            ERROR_IF(inst->dst.file == ARF && inst->dst.nr == BRW_ARF_SCALAR,
                     "When source is a scalar register, destination must not be a scalar register.");
            ERROR_IF(!src_has_scalar_region(inst, 0),
                     "When source is a scalar register and opcode is MOV, the scalar (broadcast) regioning must be used.");
            break;
         }

         case BRW_OPCODE_SEND:
         case BRW_OPCODE_SENDC: {
            ERROR_IF(!src1_is_null(inst),
                     "When source is a scalar and opcode is a SEND or SENDC, Src1 must be NULL.");
            break;
         }

         default:
            ERROR("When source is a scalar register, opcode must be MOV, SEND, or SENDC.");
            break;
         }
      }

      if ((inst->src[1].file == ARF && inst->src[1].nr == BRW_ARF_SCALAR) ||
          (inst->src[2].file == ARF && inst->src[2].nr == BRW_ARF_SCALAR)) {
         ERROR("When source is a scalar register, it must be on Source 0.");
      }
   } else {
      assert(devinfo->ver < 30);
      if ((inst->dst.file == ARF && inst->dst.nr == BRW_ARF_SCALAR) ||
          (inst->src[0].file == ARF && inst->src[0].nr == BRW_ARF_SCALAR) ||
          (inst->src[1].file == ARF && inst->src[1].nr == BRW_ARF_SCALAR) ||
          (inst->src[2].file == ARF && inst->src[2].nr == BRW_ARF_SCALAR))
         ERROR("Scalar register not available before Gfx30.");
   }
}

static unsigned
DST_STRIDE_3SRC(unsigned hstride)
{
   switch (hstride) {
   case BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1: return 1;
   case BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_2: return 2;
   }
   UNREACHABLE("invalid hstride");
}

static unsigned
VSTRIDE_3SRC(unsigned vstride)
{
   switch (vstride) {
   case BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0: return 0;
   case BRW_ALIGN1_3SRC_VERTICAL_STRIDE_1: return 1;
   case BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4: return 4;
   case BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8: return 8;
   }
   UNREACHABLE("invalid vstride");
}

static inline unsigned
brw_implied_width_for_3src_a1(unsigned v, unsigned h)
{
   /* "Regioning Rules for Align1 Ternary Operations" */

   /* TODO: Add remaining rules and de-duplicate with brw_disasm.c */

   if (v == 0) return 1;
   if (h == 0) return v;
   return v/h;
}

static void
brw_hw_decode_inst(const struct brw_isa_info *isa,
                   brw_hw_decoded_inst *inst,
                   const brw_eu_inst *raw, struct error *error)
{
   const struct intel_device_info *devinfo = isa->devinfo;

   inst->raw = raw;
   inst->opcode = brw_eu_inst_opcode(isa, raw);
   inst->num_sources = brw_num_sources_from_inst(isa, raw);

   const struct opcode_desc *desc = brw_opcode_desc(isa, inst->opcode);
   assert(desc->ndst == 0 || desc->ndst == 1);
   inst->has_dst = desc->ndst == 1;

   enum brw_execution_size exec_size = brw_eu_inst_exec_size(devinfo, raw);
   switch (exec_size) {
   case BRW_EXECUTE_1:
   case BRW_EXECUTE_2:
   case BRW_EXECUTE_4:
   case BRW_EXECUTE_8:
   case BRW_EXECUTE_16:
   case BRW_EXECUTE_32:
      inst->exec_size = 1 << exec_size;
      break;
   default:
      RETURN_ERROR("invalid execution size");
      break;
   }

   inst->access_mode = brw_eu_inst_access_mode(devinfo, raw);
   inst->pred_control = brw_eu_inst_pred_control(devinfo, raw);

   RETURN_ERROR_IF(inst->num_sources == 3 && inst->access_mode == BRW_ALIGN_1 && devinfo->ver == 9,
                   "Align1 mode not allowed on Gfx9 for 3-src instructions");

   RETURN_ERROR_IF(inst->access_mode == BRW_ALIGN_16 && devinfo->ver >= 11,
                   "Align16 mode doesn't exist on Gfx11+");

   switch (inst->opcode) {
   case BRW_OPCODE_DPAS:
      inst->format = FORMAT_DPAS_THREE_SRC;
      break;

   case BRW_OPCODE_SEND:
   case BRW_OPCODE_SENDC:
      inst->format = devinfo->ver >= 12 ? FORMAT_SEND : FORMAT_BASIC;
      break;

   case BRW_OPCODE_SENDS:
   case BRW_OPCODE_SENDSC:
      inst->format = FORMAT_SEND;
      break;

   case BRW_OPCODE_DO:
   case BRW_OPCODE_WHILE:
   case BRW_OPCODE_IF:
   case BRW_OPCODE_ELSE:
   case BRW_OPCODE_ENDIF:
   case BRW_OPCODE_BREAK:
   case BRW_OPCODE_CONTINUE:
   case BRW_OPCODE_JMPI:
   case BRW_OPCODE_BRD:
   case BRW_OPCODE_BRC:
   case BRW_OPCODE_HALT:
   case BRW_OPCODE_CALLA:
   case BRW_OPCODE_CALL:
   case BRW_OPCODE_GOTO:
   case BRW_OPCODE_JOIN:
      inst->format = FORMAT_BRANCH;
      break;

   case BRW_OPCODE_NOP:
      inst->format = FORMAT_NOP;
      break;

   case BRW_OPCODE_ILLEGAL:
      inst->format = FORMAT_ILLEGAL;
      break;

   default:
      if (inst->num_sources == 3) {
         inst->format = FORMAT_BASIC_THREE_SRC;
      } else {
         inst->format = FORMAT_BASIC;
      }
      break;
   }

   switch (inst->format) {
   case FORMAT_BASIC: {
      assert(inst->num_sources == 1 ||
             inst->num_sources == 2 ||
             inst->opcode == BRW_OPCODE_WAIT);
      assert(inst->has_dst ||
             inst->opcode == BRW_OPCODE_SYNC);

      if (inst->has_dst) {
         inst->dst.file = brw_eu_inst_dst_reg_file(devinfo, raw);
         inst->dst.type = brw_eu_inst_dst_type(devinfo, raw);
         inst->dst.address_mode = brw_eu_inst_dst_address_mode(devinfo, raw);
         if (inst->dst.address_mode == BRW_ADDRESS_DIRECT) {
            inst->dst.nr = brw_eu_inst_dst_da_reg_nr(devinfo, raw);
            if (inst->access_mode == BRW_ALIGN_1) {
               inst->dst.subnr = brw_eu_inst_dst_da1_subreg_nr(devinfo, raw);
            } else {
               inst->dst.subnr = brw_eu_inst_dst_da16_subreg_nr(devinfo, raw);
            }
         } else {
            inst->dst.subnr = brw_eu_inst_dst_ia_subreg_nr(devinfo, raw);
         }
         inst->dst.hstride = STRIDE(brw_eu_inst_dst_hstride(devinfo, raw));
      }

      inst->src[0].file = brw_eu_inst_src0_reg_file(devinfo, raw);
      inst->src[0].type = brw_eu_inst_src0_type(devinfo, raw);
      inst->src[0].address_mode = brw_eu_inst_src0_address_mode(devinfo, raw);
      inst->src[0].negate = brw_eu_inst_src0_negate(devinfo, raw);
      inst->src[0].abs = brw_eu_inst_src0_abs(devinfo, raw);
      if (inst->src[0].file != IMM) {
         if (inst->src[0].address_mode == BRW_ADDRESS_DIRECT) {
            inst->src[0].nr = brw_eu_inst_src0_da_reg_nr(devinfo, raw);
            if (inst->access_mode == BRW_ALIGN_1) {
               inst->src[0].subnr = brw_eu_inst_src0_da1_subreg_nr(devinfo, raw);
            } else {
               inst->src[0].subnr = brw_eu_inst_src0_da16_subreg_nr(devinfo, raw) * 16;
            }
         } else {
            inst->src[0].subnr = brw_eu_inst_src0_ia_subreg_nr(devinfo, raw);
         }

         inst->src[0].vstride = STRIDE(brw_eu_inst_src0_vstride(devinfo, raw));
         if (inst->access_mode == BRW_ALIGN_1) {
            inst->src[0].width = WIDTH(brw_eu_inst_src0_width(devinfo, raw));
            inst->src[0].hstride = STRIDE(brw_eu_inst_src0_hstride(devinfo, raw));
         }
      }

      if (inst->num_sources > 1) {
         inst->src[1].file = brw_eu_inst_src1_reg_file(devinfo, raw);
         inst->src[1].type = brw_eu_inst_src1_type(devinfo, raw);
         inst->src[1].negate = brw_eu_inst_src1_negate(devinfo, raw);
         inst->src[1].abs = brw_eu_inst_src1_abs(devinfo, raw);
         if (inst->src[1].file != IMM) {
            if (inst->src[1].address_mode == BRW_ADDRESS_DIRECT) {
               inst->src[1].nr = brw_eu_inst_src1_da_reg_nr(devinfo, raw);
               if (inst->access_mode == BRW_ALIGN_1) {
                  inst->src[1].subnr = brw_eu_inst_src1_da1_subreg_nr(devinfo, raw);
               } else {
                  inst->src[1].subnr = brw_eu_inst_src1_da16_subreg_nr(devinfo, raw) * 16;
               }
            } else {
               inst->src[1].subnr = brw_eu_inst_src1_ia_subreg_nr(devinfo, raw);
            }

            inst->src[1].vstride = STRIDE(brw_eu_inst_src1_vstride(devinfo, raw));
            if (inst->access_mode == BRW_ALIGN_1) {
               inst->src[1].width = WIDTH(brw_eu_inst_src1_width(devinfo, raw));
               inst->src[1].hstride = STRIDE(brw_eu_inst_src1_hstride(devinfo, raw));
            }
         }
      }

      break;
   }

   case FORMAT_BASIC_THREE_SRC: {
      assert(inst->num_sources == 3);
      assert(inst->has_dst);

      if (inst->access_mode == BRW_ALIGN_1) {
         inst->dst.file = brw_eu_inst_3src_a1_dst_reg_file(devinfo, raw);
         inst->dst.type = brw_eu_inst_3src_a1_dst_type(devinfo, raw);
         inst->dst.nr = brw_eu_inst_3src_dst_reg_nr(devinfo, raw);
         inst->dst.subnr = brw_eu_inst_3src_a1_dst_subreg_nr(devinfo, raw) * 8;
         inst->dst.hstride = DST_STRIDE_3SRC(brw_eu_inst_3src_a1_dst_hstride(devinfo, raw));

         inst->src[0].file = brw_eu_inst_3src_a1_src0_reg_file(devinfo, raw);
         inst->src[0].type = brw_eu_inst_3src_a1_src0_type(devinfo, raw);
         inst->src[0].negate = brw_eu_inst_3src_src0_negate(devinfo, raw);
         inst->src[0].abs = brw_eu_inst_3src_src0_abs(devinfo, raw);
         if (inst->src[0].file != IMM) {
            inst->src[0].nr = brw_eu_inst_3src_src0_reg_nr(devinfo, raw);
            inst->src[0].subnr = brw_eu_inst_3src_a1_src0_subreg_nr(devinfo, raw);
            inst->src[0].vstride = VSTRIDE_3SRC(brw_eu_inst_3src_a1_src0_vstride(devinfo, raw));
            inst->src[0].hstride = STRIDE(brw_eu_inst_3src_a1_src0_hstride(devinfo, raw));
            inst->src[0].width = brw_implied_width_for_3src_a1(inst->src[0].vstride, inst->src[0].hstride);
         }

         inst->src[1].file = brw_eu_inst_3src_a1_src1_reg_file(devinfo, raw);
         inst->src[1].type = brw_eu_inst_3src_a1_src1_type(devinfo, raw);
         inst->src[1].negate = brw_eu_inst_3src_src1_negate(devinfo, raw);
         inst->src[1].abs = brw_eu_inst_3src_src1_abs(devinfo, raw);
         inst->src[1].nr = brw_eu_inst_3src_src1_reg_nr(devinfo, raw);
         inst->src[1].subnr = brw_eu_inst_3src_a1_src1_subreg_nr(devinfo, raw);
         inst->src[1].vstride = VSTRIDE_3SRC(brw_eu_inst_3src_a1_src1_vstride(devinfo, raw));
         inst->src[1].hstride = STRIDE(brw_eu_inst_3src_a1_src1_hstride(devinfo, raw));
         inst->src[1].width = brw_implied_width_for_3src_a1(inst->src[1].vstride, inst->src[1].hstride);

         inst->src[2].file = brw_eu_inst_3src_a1_src2_reg_file(devinfo, raw);
         inst->src[2].type = brw_eu_inst_3src_a1_src2_type(devinfo, raw);
         inst->src[2].negate = brw_eu_inst_3src_src2_negate(devinfo, raw);
         inst->src[2].abs = brw_eu_inst_3src_src2_abs(devinfo, raw);
         if (inst->src[2].file != IMM) {
            inst->src[2].nr = brw_eu_inst_3src_src2_reg_nr(devinfo, raw);
            inst->src[2].subnr = brw_eu_inst_3src_a1_src2_subreg_nr(devinfo, raw);
            inst->src[2].hstride = STRIDE(brw_eu_inst_3src_a1_src2_hstride(devinfo, raw));
            inst->src[2].width = brw_implied_width_for_3src_a1(inst->src[2].vstride, inst->src[2].hstride);
         }

      } else {
         inst->dst.file = FIXED_GRF;
         inst->dst.type = brw_eu_inst_3src_a16_dst_type(devinfo, raw);
         inst->dst.nr = brw_eu_inst_3src_dst_reg_nr(devinfo, raw);
         inst->dst.subnr = brw_eu_inst_3src_a16_dst_subreg_nr(devinfo, raw) * 4;

         enum brw_reg_type src_type = brw_eu_inst_3src_a16_src_type(devinfo, raw);

         inst->src[0].file = FIXED_GRF;
         inst->src[0].type = src_type;
         inst->src[0].nr = brw_eu_inst_3src_src0_reg_nr(devinfo, raw);
         inst->src[0].subnr = brw_eu_inst_3src_a16_src0_subreg_nr(devinfo, raw) * 4;

         inst->src[1].file = FIXED_GRF;
         inst->src[1].type = src_type;
         inst->src[1].nr = brw_eu_inst_3src_src1_reg_nr(devinfo, raw);
         inst->src[1].subnr = brw_eu_inst_3src_a16_src1_subreg_nr(devinfo, raw) * 4;

         inst->src[2].file = FIXED_GRF;
         inst->src[2].type = src_type;
         inst->src[2].nr = brw_eu_inst_3src_src2_reg_nr(devinfo, raw);
         inst->src[2].subnr = brw_eu_inst_3src_a16_src2_subreg_nr(devinfo, raw) * 4;
      }
      break;
   }

   case FORMAT_DPAS_THREE_SRC: {
      assert(inst->num_sources == 3);
      assert(inst->has_dst);

      inst->dst.file = brw_eu_inst_dpas_3src_dst_reg_file(devinfo, raw);
      inst->dst.type = brw_eu_inst_dpas_3src_dst_type(devinfo, raw);
      inst->dst.nr = brw_eu_inst_dpas_3src_dst_reg_nr(devinfo, raw);
      inst->dst.subnr = brw_eu_inst_dpas_3src_dst_subreg_nr(devinfo, raw);

      inst->src[0].file = brw_eu_inst_dpas_3src_src0_reg_file(devinfo, raw);
      inst->src[0].type = brw_eu_inst_dpas_3src_src0_type(devinfo, raw);
      inst->src[0].nr = brw_eu_inst_dpas_3src_src0_reg_nr(devinfo, raw);
      inst->src[0].subnr = brw_eu_inst_dpas_3src_src0_subreg_nr(devinfo, raw);

      inst->src[1].file = brw_eu_inst_dpas_3src_src1_reg_file(devinfo, raw);
      inst->src[1].type = brw_eu_inst_dpas_3src_src1_type(devinfo, raw);
      inst->src[1].nr = brw_eu_inst_dpas_3src_src1_reg_nr(devinfo, raw);
      inst->src[1].subnr = brw_eu_inst_dpas_3src_src1_subreg_nr(devinfo, raw);

      inst->src[2].file = brw_eu_inst_dpas_3src_src2_reg_file(devinfo, raw);
      inst->src[2].type = brw_eu_inst_dpas_3src_src2_type(devinfo, raw);
      inst->src[2].nr = brw_eu_inst_dpas_3src_src2_reg_nr(devinfo, raw);
      inst->src[2].subnr = brw_eu_inst_dpas_3src_src2_subreg_nr(devinfo, raw);
      break;
   }

   case FORMAT_SEND: {
      if (inst->opcode == BRW_OPCODE_SENDS || inst->opcode == BRW_OPCODE_SENDSC) {
         assert(devinfo->ver < 12);

         inst->dst.file = brw_eu_inst_send_dst_reg_file(devinfo, raw);
         inst->dst.type = BRW_TYPE_D;
         inst->dst.nr = brw_eu_inst_dst_da_reg_nr(devinfo, raw);
         inst->dst.subnr = brw_eu_inst_dst_da16_subreg_nr(devinfo, raw) * 16;

         inst->src[0].file = FIXED_GRF;
         inst->src[0].type = BRW_TYPE_D;
         inst->src[0].nr = brw_eu_inst_src0_da_reg_nr(devinfo, raw);
         inst->src[0].subnr = brw_eu_inst_src0_da16_subreg_nr(devinfo, raw) * 16;

         if (inst->num_sources > 1) {
            inst->src[1].file = brw_eu_inst_send_src1_reg_file(devinfo, raw);
            inst->src[1].type = BRW_TYPE_D;
            inst->src[1].nr = brw_eu_inst_send_src1_reg_nr(devinfo, raw);
         }
      } else {
         assert(devinfo->ver >= 12);

         inst->dst.file = brw_eu_inst_dst_reg_file(devinfo, raw);
         inst->dst.type = BRW_TYPE_D;
         inst->dst.nr = brw_eu_inst_dst_da_reg_nr(devinfo, raw);

         inst->src[0].file = brw_eu_inst_send_src0_reg_file(devinfo, raw);
         inst->src[0].type = BRW_TYPE_D;
         inst->src[0].nr = brw_eu_inst_src0_da_reg_nr(devinfo, raw);

         if (inst->num_sources > 1) {
            inst->src[1].file = brw_eu_inst_send_src1_reg_file(devinfo, raw);
            inst->src[1].type = BRW_TYPE_D;
            inst->src[1].nr = brw_eu_inst_send_src1_reg_nr(devinfo, raw);
         }
      }
      break;
   }

   case FORMAT_BRANCH: {
      assert(!inst->has_dst);
      break;
   }

   case FORMAT_ILLEGAL:
   case FORMAT_NOP: {
      assert(!inst->has_dst);
      assert(inst->num_sources == 0);
      break;
   }
   }

   if (inst->has_dst) {
      ERROR_IF(inst->dst.type == BRW_TYPE_INVALID,
               "Invalid destination register type encoding.");
   }

   for (unsigned i = 0; i < inst->num_sources; i++) {
      ERROR_IF(inst->src[i].type == BRW_TYPE_INVALID,
               "Invalid source register type encoding.");
   }

   if ((inst->format == FORMAT_BASIC ||
        inst->format == FORMAT_BASIC_THREE_SRC ||
        inst->format == FORMAT_DPAS_THREE_SRC) &&
       !inst_is_send(inst)) {
      inst->saturate = brw_eu_inst_saturate(devinfo, raw);

      if (inst->num_sources > 1 ||
          devinfo->ver < 12 ||
          inst->src[0].file != IMM  ||
          brw_type_size_bytes(inst->src[0].type) < 8) {
         inst->cond_modifier = brw_eu_inst_cond_modifier(devinfo, raw);
      }
   }
}

bool
brw_validate_instruction(const struct brw_isa_info *isa,
                         const brw_eu_inst *inst, int offset,
                         unsigned inst_size,
                         struct disasm_info *disasm)
{
   struct error error = {};

   if (is_unsupported_inst(isa, inst)) {
      report_error(&error, "Instruction not supported on this Gfx version");
   } else {
      brw_hw_decoded_inst decoded = {};
      brw_hw_decode_inst(isa, &decoded, inst, &error);

#define CHECK(func, args...) func(isa, &decoded, &error, ##args);

      if (!error.msg)
         CHECK(invalid_values);

      if (!error.msg) {
         CHECK(sources_not_null);
         CHECK(send_restrictions);
         CHECK(general_restrictions_based_on_operand_types);
         CHECK(general_restrictions_on_region_parameters);
         CHECK(special_restrictions_for_mixed_float_mode);
         CHECK(region_alignment_rules);
         CHECK(vector_immediate_restrictions);
         CHECK(special_requirements_for_handling_double_precision_data_types);
         CHECK(instruction_restrictions);
         CHECK(send_descriptor_restrictions);
         CHECK(register_region_special_restrictions);
         CHECK(scalar_register_restrictions);
      }

#undef CHECK
   }

   if (error.msg) {
      if (disasm)
         disasm_insert_error(disasm, offset, inst_size, error.msg);
      ralloc_free(error.msg);
      return false;
   } else {
      return true;
   }
}

bool
brw_validate_instructions(const struct brw_isa_info *isa,
                          const void *assembly, int start_offset, int end_offset,
                          struct disasm_info *disasm)
{
   const struct intel_device_info *devinfo = isa->devinfo;
   bool valid = true;

   for (int src_offset = start_offset; src_offset < end_offset;) {
      const brw_eu_inst *inst = assembly + src_offset;
      bool is_compact = brw_eu_inst_cmpt_control(devinfo, inst);
      unsigned inst_size = is_compact ? sizeof(brw_eu_compact_inst)
                                      : sizeof(brw_eu_inst);
      brw_eu_inst uncompacted;

      if (is_compact) {
         brw_eu_compact_inst *compacted = (void *)inst;
         brw_uncompact_instruction(isa, &uncompacted, compacted);
         inst = &uncompacted;
      }

      bool v = brw_validate_instruction(isa, inst, src_offset,
                                        inst_size, disasm);
      valid = valid && v;

      src_offset += inst_size;
   }

   return valid;
}