src/transform/decompose_memory_access.cc

// Copyright 2021 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "src/transform/decompose_memory_access.h"

#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>

#include "src/ast/assignment_statement.h"
#include "src/ast/call_statement.h"
#include "src/ast/disable_validation_decoration.h"
#include "src/ast/type_name.h"
#include "src/ast/unary_op.h"
#include "src/block_allocator.h"
#include "src/program_builder.h"
#include "src/sem/array.h"
#include "src/sem/atomic_type.h"
#include "src/sem/call.h"
#include "src/sem/member_accessor_expression.h"
#include "src/sem/reference_type.h"
#include "src/sem/statement.h"
#include "src/sem/struct.h"
#include "src/sem/variable.h"
#include "src/utils/hash.h"
#include "src/utils/map.h"

TINT_INSTANTIATE_TYPEINFO(tint::transform::DecomposeMemoryAccess);
TINT_INSTANTIATE_TYPEINFO(tint::transform::DecomposeMemoryAccess::Intrinsic);

namespace tint {
namespace transform {

namespace {

/// Offset is a simple ast::Expression builder interface, used to build byte
/// offsets for storage and uniform buffer accesses.
struct Offset : Castable<Offset> {
  /// @returns builds and returns the ast::Expression in `ctx.dst`
  virtual const ast::Expression* Build(CloneContext& ctx) const = 0;
};

/// OffsetExpr is an implementation of Offset that clones and casts the given
/// expression to `u32`.
struct OffsetExpr : Offset {
  const ast::Expression* const expr = nullptr;

  explicit OffsetExpr(const ast::Expression* e) : expr(e) {}

  const ast::Expression* Build(CloneContext& ctx) const override {
    auto* type = ctx.src->Sem().Get(expr)->Type()->UnwrapRef();
    auto* res = ctx.Clone(expr);
    if (!type->Is<sem::U32>()) {
      res = ctx.dst->Construct<ProgramBuilder::u32>(res);
    }
    return res;
  }
};

/// OffsetLiteral is an implementation of Offset that constructs a u32 literal
/// value.
struct OffsetLiteral : Castable<OffsetLiteral, Offset> {
  uint32_t const literal = 0;

  explicit OffsetLiteral(uint32_t lit) : literal(lit) {}

  const ast::Expression* Build(CloneContext& ctx) const override {
    return ctx.dst->Expr(literal);
  }
};

/// OffsetBinOp is an implementation of Offset that constructs a binary-op of
/// two Offsets.
struct OffsetBinOp : Offset {
  ast::BinaryOp op;
  Offset const* lhs = nullptr;
  Offset const* rhs = nullptr;

  const ast::Expression* Build(CloneContext& ctx) const override {
    return ctx.dst->create<ast::BinaryExpression>(op, lhs->Build(ctx),
                                                  rhs->Build(ctx));
  }
};

/// LoadStoreKey is the unordered map key to a load or store intrinsic.
struct LoadStoreKey {
  ast::StorageClass const storage_class;  // buffer storage class
  sem::Type const* buf_ty = nullptr;      // buffer type
  sem::Type const* el_ty = nullptr;       // element type
  bool operator==(const LoadStoreKey& rhs) const {
    return storage_class == rhs.storage_class && buf_ty == rhs.buf_ty &&
           el_ty == rhs.el_ty;
  }
  struct Hasher {
    inline std::size_t operator()(const LoadStoreKey& u) const {
      return utils::Hash(u.storage_class, u.buf_ty, u.el_ty);
    }
  };
};

/// AtomicKey is the unordered map key to an atomic intrinsic.
struct AtomicKey {
  sem::Type const* buf_ty = nullptr;  // buffer type
  sem::Type const* el_ty = nullptr;   // element type
  sem::IntrinsicType const op;        // atomic op
  bool operator==(const AtomicKey& rhs) const {
    return buf_ty == rhs.buf_ty && el_ty == rhs.el_ty && op == rhs.op;
  }
  struct Hasher {
    inline std::size_t operator()(const AtomicKey& u) const {
      return utils::Hash(u.buf_ty, u.el_ty, u.op);
    }
  };
};

bool IntrinsicDataTypeFor(const sem::Type* ty,
                          DecomposeMemoryAccess::Intrinsic::DataType& out) {
  if (ty->Is<sem::I32>()) {
    out = DecomposeMemoryAccess::Intrinsic::DataType::kI32;
    return true;
  }
  if (ty->Is<sem::U32>()) {
    out = DecomposeMemoryAccess::Intrinsic::DataType::kU32;
    return true;
  }
  if (ty->Is<sem::F32>()) {
    out = DecomposeMemoryAccess::Intrinsic::DataType::kF32;
    return true;
  }
  if (auto* vec = ty->As<sem::Vector>()) {
    switch (vec->Width()) {
      case 2:
        if (vec->type()->Is<sem::I32>()) {
          out = DecomposeMemoryAccess::Intrinsic::DataType::kVec2I32;
          return true;
        }
        if (vec->type()->Is<sem::U32>()) {
          out = DecomposeMemoryAccess::Intrinsic::DataType::kVec2U32;
          return true;
        }
        if (vec->type()->Is<sem::F32>()) {
          out = DecomposeMemoryAccess::Intrinsic::DataType::kVec2F32;
          return true;
        }
        break;
      case 3:
        if (vec->type()->Is<sem::I32>()) {
          out = DecomposeMemoryAccess::Intrinsic::DataType::kVec3I32;
          return true;
        }
        if (vec->type()->Is<sem::U32>()) {
          out = DecomposeMemoryAccess::Intrinsic::DataType::kVec3U32;
          return true;
        }
        if (vec->type()->Is<sem::F32>()) {
          out = DecomposeMemoryAccess::Intrinsic::DataType::kVec3F32;
          return true;
        }
        break;
      case 4:
        if (vec->type()->Is<sem::I32>()) {
          out = DecomposeMemoryAccess::Intrinsic::DataType::kVec4I32;
          return true;
        }
        if (vec->type()->Is<sem::U32>()) {
          out = DecomposeMemoryAccess::Intrinsic::DataType::kVec4U32;
          return true;
        }
        if (vec->type()->Is<sem::F32>()) {
          out = DecomposeMemoryAccess::Intrinsic::DataType::kVec4F32;
          return true;
        }
        break;
    }
    return false;
  }

  return false;
}

/// @returns a DecomposeMemoryAccess::Intrinsic decoration that can be applied
/// to a stub function to load the type `ty`.
DecomposeMemoryAccess::Intrinsic* IntrinsicLoadFor(
    ProgramBuilder* builder,
    ast::StorageClass storage_class,
    const sem::Type* ty) {
  DecomposeMemoryAccess::Intrinsic::DataType type;
  if (!IntrinsicDataTypeFor(ty, type)) {
    return nullptr;
  }
  return builder->ASTNodes().Create<DecomposeMemoryAccess::Intrinsic>(
      builder->ID(), DecomposeMemoryAccess::Intrinsic::Op::kLoad, storage_class,
      type);
}

/// @returns a DecomposeMemoryAccess::Intrinsic decoration that can be applied
/// to a stub function to store the type `ty`.
DecomposeMemoryAccess::Intrinsic* IntrinsicStoreFor(
    ProgramBuilder* builder,
    ast::StorageClass storage_class,
    const sem::Type* ty) {
  DecomposeMemoryAccess::Intrinsic::DataType type;
  if (!IntrinsicDataTypeFor(ty, type)) {
    return nullptr;
  }
  return builder->ASTNodes().Create<DecomposeMemoryAccess::Intrinsic>(
      builder->ID(), DecomposeMemoryAccess::Intrinsic::Op::kStore,
      storage_class, type);
}

/// @returns a DecomposeMemoryAccess::Intrinsic decoration that can be applied
/// to a stub function for the atomic op and the type `ty`.
DecomposeMemoryAccess::Intrinsic* IntrinsicAtomicFor(ProgramBuilder* builder,
                                                     sem::IntrinsicType ity,
                                                     const sem::Type* ty) {
  auto op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicLoad;
  switch (ity) {
    case sem::IntrinsicType::kAtomicLoad:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicLoad;
      break;
    case sem::IntrinsicType::kAtomicStore:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicStore;
      break;
    case sem::IntrinsicType::kAtomicAdd:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicAdd;
      break;
    case sem::IntrinsicType::kAtomicSub:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicSub;
      break;
    case sem::IntrinsicType::kAtomicMax:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicMax;
      break;
    case sem::IntrinsicType::kAtomicMin:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicMin;
      break;
    case sem::IntrinsicType::kAtomicAnd:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicAnd;
      break;
    case sem::IntrinsicType::kAtomicOr:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicOr;
      break;
    case sem::IntrinsicType::kAtomicXor:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicXor;
      break;
    case sem::IntrinsicType::kAtomicExchange:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicExchange;
      break;
    case sem::IntrinsicType::kAtomicCompareExchangeWeak:
      op = DecomposeMemoryAccess::Intrinsic::Op::kAtomicCompareExchangeWeak;
      break;
    default:
      TINT_ICE(Transform, builder->Diagnostics())
          << "invalid IntrinsicType for DecomposeMemoryAccess::Intrinsic: "
          << ty->type_name();
      break;
  }

  DecomposeMemoryAccess::Intrinsic::DataType type;
  if (!IntrinsicDataTypeFor(ty, type)) {
    return nullptr;
  }
  return builder->ASTNodes().Create<DecomposeMemoryAccess::Intrinsic>(
      builder->ID(), op, ast::StorageClass::kStorage, type);
}

/// BufferAccess describes a single storage or uniform buffer access
struct BufferAccess {
  sem::Expression const* var = nullptr;  // Storage buffer variable
  Offset const* offset = nullptr;        // The byte offset on var
  sem::Type const* type = nullptr;       // The type of the access
  operator bool() const { return var; }  // Returns true if valid
};

/// Store describes a single storage or uniform buffer write
struct Store {
  const ast::AssignmentStatement* assignment;  // The AST assignment statement
  BufferAccess target;                         // The target for the write
};

}  // namespace

/// State holds the current transform state
struct DecomposeMemoryAccess::State {
  /// The clone context
  CloneContext& ctx;
  /// Alias to `*ctx.dst`
  ProgramBuilder& b;
  /// Map of AST expression to storage or uniform buffer access
  /// This map has entries added when encountered, and removed when outer
  /// expressions chain the access.
  /// Subset of #expression_order, as expressions are not removed from
  /// #expression_order.
  std::unordered_map<const ast::Expression*, BufferAccess> accesses;
  /// The visited order of AST expressions (superset of #accesses)
  std::vector<const ast::Expression*> expression_order;
  /// [buffer-type, element-type] -> load function name
  std::unordered_map<LoadStoreKey, Symbol, LoadStoreKey::Hasher> load_funcs;
  /// [buffer-type, element-type] -> store function name
  std::unordered_map<LoadStoreKey, Symbol, LoadStoreKey::Hasher> store_funcs;
  /// [buffer-type, element-type, atomic-op] -> load function name
  std::unordered_map<AtomicKey, Symbol, AtomicKey::Hasher> atomic_funcs;
  /// List of storage or uniform buffer writes
  std::vector<Store> stores;
  /// Allocations for offsets
  BlockAllocator<Offset> offsets_;

  /// Constructor
  /// @param context the CloneContext
  explicit State(CloneContext& context) : ctx(context), b(*ctx.dst) {}

  /// @param offset the offset value to wrap in an Offset
  /// @returns an Offset for the given literal value
  const Offset* ToOffset(uint32_t offset) {
    return offsets_.Create<OffsetLiteral>(offset);
  }

  /// @param expr the expression to convert to an Offset
  /// @returns an Offset for the given ast::Expression
  const Offset* ToOffset(const ast::Expression* expr) {
    if (auto* u32 = expr->As<ast::UintLiteralExpression>()) {
      return offsets_.Create<OffsetLiteral>(u32->value);
    } else if (auto* i32 = expr->As<ast::SintLiteralExpression>()) {
      if (i32->value > 0) {
        return offsets_.Create<OffsetLiteral>(i32->value);
      }
    }
    return offsets_.Create<OffsetExpr>(expr);
  }

  /// @param offset the Offset that is returned
  /// @returns the given offset (pass-through)
  const Offset* ToOffset(const Offset* offset) { return offset; }

  /// @param lhs_ the left-hand side of the add expression
  /// @param rhs_ the right-hand side of the add expression
  /// @return an Offset that is a sum of lhs and rhs, performing basic constant
  /// folding if possible
  template <typename LHS, typename RHS>
  const Offset* Add(LHS&& lhs_, RHS&& rhs_) {
    auto* lhs = ToOffset(std::forward<LHS>(lhs_));
    auto* rhs = ToOffset(std::forward<RHS>(rhs_));
    auto* lhs_lit = tint::As<OffsetLiteral>(lhs);
    auto* rhs_lit = tint::As<OffsetLiteral>(rhs);
    if (lhs_lit && lhs_lit->literal == 0) {
      return rhs;
    }
    if (rhs_lit && rhs_lit->literal == 0) {
      return lhs;
    }
    if (lhs_lit && rhs_lit) {
      if (static_cast<uint64_t>(lhs_lit->literal) +
              static_cast<uint64_t>(rhs_lit->literal) <=
          0xffffffff) {
        return offsets_.Create<OffsetLiteral>(lhs_lit->literal +
                                              rhs_lit->literal);
      }
    }
    auto* out = offsets_.Create<OffsetBinOp>();
    out->op = ast::BinaryOp::kAdd;
    out->lhs = lhs;
    out->rhs = rhs;
    return out;
  }

  /// @param lhs_ the left-hand side of the multiply expression
  /// @param rhs_ the right-hand side of the multiply expression
  /// @return an Offset that is the multiplication of lhs and rhs, performing
  /// basic constant folding if possible
  template <typename LHS, typename RHS>
  const Offset* Mul(LHS&& lhs_, RHS&& rhs_) {
    auto* lhs = ToOffset(std::forward<LHS>(lhs_));
    auto* rhs = ToOffset(std::forward<RHS>(rhs_));
    auto* lhs_lit = tint::As<OffsetLiteral>(lhs);
    auto* rhs_lit = tint::As<OffsetLiteral>(rhs);
    if (lhs_lit && lhs_lit->literal == 0) {
      return offsets_.Create<OffsetLiteral>(0);
    }
    if (rhs_lit && rhs_lit->literal == 0) {
      return offsets_.Create<OffsetLiteral>(0);
    }
    if (lhs_lit && lhs_lit->literal == 1) {
      return rhs;
    }
    if (rhs_lit && rhs_lit->literal == 1) {
      return lhs;
    }
    if (lhs_lit && rhs_lit) {
      return offsets_.Create<OffsetLiteral>(lhs_lit->literal *
                                            rhs_lit->literal);
    }
    auto* out = offsets_.Create<OffsetBinOp>();
    out->op = ast::BinaryOp::kMultiply;
    out->lhs = lhs;
    out->rhs = rhs;
    return out;
  }

  /// AddAccess() adds the `expr -> access` map item to #accesses, and `expr`
  /// to #expression_order.
  /// @param expr the expression that performs the access
  /// @param access the access
  void AddAccess(const ast::Expression* expr, const BufferAccess& access) {
    TINT_ASSERT(Transform, access.type);
    accesses.emplace(expr, access);
    expression_order.emplace_back(expr);
  }

  /// TakeAccess() removes the `node` item from #accesses (if it exists),
  /// returning the BufferAccess. If #accesses does not hold an item for
  /// `node`, an invalid BufferAccess is returned.
  /// @param node the expression that performed an access
  /// @return the BufferAccess for the given expression
  BufferAccess TakeAccess(const ast::Expression* node) {
    auto lhs_it = accesses.find(node);
    if (lhs_it == accesses.end()) {
      return {};
    }
    auto access = lhs_it->second;
    accesses.erase(node);
    return access;
  }

  /// LoadFunc() returns a symbol to an intrinsic function that loads an element
  /// of type `el_ty` from a storage or uniform buffer of type `buf_ty`.
  /// The emitted function has the signature:
  ///   `fn load(buf : buf_ty, offset : u32) -> el_ty`
  /// @param buf_ty the storage or uniform buffer type
  /// @param el_ty the storage or uniform buffer element type
  /// @param var_user the variable user
  /// @return the name of the function that performs the load
  Symbol LoadFunc(const sem::Type* buf_ty,
                  const sem::Type* el_ty,
                  const sem::VariableUser* var_user) {
    auto storage_class = var_user->Variable()->StorageClass();
    return utils::GetOrCreate(
        load_funcs, LoadStoreKey{storage_class, buf_ty, el_ty}, [&] {
          auto* buf_ast_ty = CreateASTTypeFor(ctx, buf_ty);
          auto* disable_validation = b.Disable(
              ast::DisabledValidation::kIgnoreConstructibleFunctionParameter);

          ast::VariableList params = {
              // Note: The buffer parameter requires the StorageClass in
              // order for HLSL to emit this as a ByteAddressBuffer or cbuffer
              // array.
              b.create<ast::Variable>(b.Sym("buffer"), storage_class,
                                      var_user->Variable()->Access(),
                                      buf_ast_ty, true, nullptr,
                                      ast::DecorationList{disable_validation}),
              b.Param("offset", b.ty.u32()),
          };

          auto name = b.Sym();

          if (auto* intrinsic =
                  IntrinsicLoadFor(ctx.dst, storage_class, el_ty)) {
            auto* el_ast_ty = CreateASTTypeFor(ctx, el_ty);
            auto* func = b.create<ast::Function>(
                name, params, el_ast_ty, nullptr,
                ast::DecorationList{
                    intrinsic,
                    b.Disable(ast::DisabledValidation::kFunctionHasNoBody),
                },
                ast::DecorationList{});
            b.AST().AddFunction(func);
          } else if (auto* arr_ty = el_ty->As<sem::Array>()) {
            // fn load_func(buf : buf_ty, offset : u32) -> array<T, N> {
            //   var arr : array<T, N>;
            //   for (var i = 0u; i < array_count; i = i + 1) {
            //     arr[i] = el_load_func(buf, offset + i * array_stride)
            //   }
            //   return arr;
            // }
            auto load =
                LoadFunc(buf_ty, arr_ty->ElemType()->UnwrapRef(), var_user);
            auto* arr =
                b.Var(b.Symbols().New("arr"), CreateASTTypeFor(ctx, arr_ty));
            auto* i = b.Var(b.Symbols().New("i"), nullptr, b.Expr(0u));
            auto* for_init = b.Decl(i);
            auto* for_cond = b.create<ast::BinaryExpression>(
                ast::BinaryOp::kLessThan, b.Expr(i), b.Expr(arr_ty->Count()));
            auto* for_cont = b.Assign(i, b.Add(i, 1u));
            auto* arr_el = b.IndexAccessor(arr, i);
            auto* el_offset =
                b.Add(b.Expr("offset"), b.Mul(i, arr_ty->Stride()));
            auto* el_val = b.Call(load, "buffer", el_offset);
            auto* for_loop = b.For(for_init, for_cond, for_cont,
                                   b.Block(b.Assign(arr_el, el_val)));

            b.Func(name, params, CreateASTTypeFor(ctx, arr_ty),
                   {
                       b.Decl(arr),
                       for_loop,
                       b.Return(arr),
                   });
          } else {
            ast::ExpressionList values;
            if (auto* mat_ty = el_ty->As<sem::Matrix>()) {
              auto* vec_ty = mat_ty->ColumnType();
              Symbol load = LoadFunc(buf_ty, vec_ty, var_user);
              for (uint32_t i = 0; i < mat_ty->columns(); i++) {
                auto* offset = b.Add("offset", i * mat_ty->ColumnStride());
                values.emplace_back(b.Call(load, "buffer", offset));
              }
            } else if (auto* str = el_ty->As<sem::Struct>()) {
              for (auto* member : str->Members()) {
                auto* offset = b.Add("offset", member->Offset());
                Symbol load =
                    LoadFunc(buf_ty, member->Type()->UnwrapRef(), var_user);
                values.emplace_back(b.Call(load, "buffer", offset));
              }
            }
            b.Func(
                name, params, CreateASTTypeFor(ctx, el_ty),
                {
                    b.Return(b.Construct(CreateASTTypeFor(ctx, el_ty), values)),
                });
          }
          return name;
        });
  }

  /// StoreFunc() returns a symbol to an intrinsic function that stores an
  /// element of type `el_ty` to a storage buffer of type `buf_ty`.
  /// The function has the signature:
  ///   `fn store(buf : buf_ty, offset : u32, value : el_ty)`
  /// @param buf_ty the storage buffer type
  /// @param el_ty the storage buffer element type
  /// @param var_user the variable user
  /// @return the name of the function that performs the store
  Symbol StoreFunc(const sem::Type* buf_ty,
                   const sem::Type* el_ty,
                   const sem::VariableUser* var_user) {
    auto storage_class = var_user->Variable()->StorageClass();
    return utils::GetOrCreate(
        store_funcs, LoadStoreKey{storage_class, buf_ty, el_ty}, [&] {
          auto* buf_ast_ty = CreateASTTypeFor(ctx, buf_ty);
          auto* el_ast_ty = CreateASTTypeFor(ctx, el_ty);
          auto* disable_validation = b.Disable(
              ast::DisabledValidation::kIgnoreConstructibleFunctionParameter);
          ast::VariableList params{
              // Note: The buffer parameter requires the StorageClass in
              // order for HLSL to emit this as a ByteAddressBuffer.

              b.create<ast::Variable>(b.Sym("buffer"), storage_class,
                                      var_user->Variable()->Access(),
                                      buf_ast_ty, true, nullptr,
                                      ast::DecorationList{disable_validation}),
              b.Param("offset", b.ty.u32()),
              b.Param("value", el_ast_ty),
          };

          auto name = b.Sym();

          if (auto* intrinsic =
                  IntrinsicStoreFor(ctx.dst, storage_class, el_ty)) {
            auto* func = b.create<ast::Function>(
                name, params, b.ty.void_(), nullptr,
                ast::DecorationList{
                    intrinsic,
                    b.Disable(ast::DisabledValidation::kFunctionHasNoBody),
                },
                ast::DecorationList{});
            b.AST().AddFunction(func);
          } else {
            ast::StatementList body;
            if (auto* arr_ty = el_ty->As<sem::Array>()) {
              // fn store_func(buf : buf_ty, offset : u32, value : el_ty) {
              //   var array = value; // No dynamic indexing on constant arrays
              //   for (var i = 0u; i < array_count; i = i + 1) {
              //     arr[i] = el_store_func(buf, offset + i * array_stride,
              //                            value[i])
              //   }
              //   return arr;
              // }
              auto* array =
                  b.Var(b.Symbols().New("array"), nullptr, b.Expr("value"));
              auto store =
                  StoreFunc(buf_ty, arr_ty->ElemType()->UnwrapRef(), var_user);
              auto* i = b.Var(b.Symbols().New("i"), nullptr, b.Expr(0u));
              auto* for_init = b.Decl(i);
              auto* for_cond = b.create<ast::BinaryExpression>(
                  ast::BinaryOp::kLessThan, b.Expr(i), b.Expr(arr_ty->Count()));
              auto* for_cont = b.Assign(i, b.Add(i, 1u));
              auto* arr_el = b.IndexAccessor(array, i);
              auto* el_offset =
                  b.Add(b.Expr("offset"), b.Mul(i, arr_ty->Stride()));
              auto* store_stmt =
                  b.CallStmt(b.Call(store, "buffer", el_offset, arr_el));
              auto* for_loop =
                  b.For(for_init, for_cond, for_cont, b.Block(store_stmt));

              body = {b.Decl(array), for_loop};
            } else if (auto* mat_ty = el_ty->As<sem::Matrix>()) {
              auto* vec_ty = mat_ty->ColumnType();
              Symbol store = StoreFunc(buf_ty, vec_ty, var_user);
              for (uint32_t i = 0; i < mat_ty->columns(); i++) {
                auto* offset = b.Add("offset", i * mat_ty->ColumnStride());
                auto* access = b.IndexAccessor("value", i);
                auto* call = b.Call(store, "buffer", offset, access);
                body.emplace_back(b.CallStmt(call));
              }
            } else if (auto* str = el_ty->As<sem::Struct>()) {
              for (auto* member : str->Members()) {
                auto* offset = b.Add("offset", member->Offset());
                auto* access = b.MemberAccessor(
                    "value", ctx.Clone(member->Declaration()->symbol));
                Symbol store =
                    StoreFunc(buf_ty, member->Type()->UnwrapRef(), var_user);
                auto* call = b.Call(store, "buffer", offset, access);
                body.emplace_back(b.CallStmt(call));
              }
            }
            b.Func(name, params, b.ty.void_(), body);
          }

          return name;
        });
  }

  /// AtomicFunc() returns a symbol to an intrinsic function that performs an
  /// atomic operation from a storage buffer of type `buf_ty`. The function has
  /// the signature:
  // `fn atomic_op(buf : buf_ty, offset : u32, ...) -> T`
  /// @param buf_ty the storage buffer type
  /// @param el_ty the storage buffer element type
  /// @param intrinsic the atomic intrinsic
  /// @param var_user the variable user
  /// @return the name of the function that performs the load
  Symbol AtomicFunc(const sem::Type* buf_ty,
                    const sem::Type* el_ty,
                    const sem::Intrinsic* intrinsic,
                    const sem::VariableUser* var_user) {
    auto op = intrinsic->Type();
    return utils::GetOrCreate(atomic_funcs, AtomicKey{buf_ty, el_ty, op}, [&] {
      auto* buf_ast_ty = CreateASTTypeFor(ctx, buf_ty);
      auto* disable_validation = b.Disable(
          ast::DisabledValidation::kIgnoreConstructibleFunctionParameter);
      // The first parameter to all WGSL atomics is the expression to the
      // atomic. This is replaced with two parameters: the buffer and offset.

      ast::VariableList params = {
          // Note: The buffer parameter requires the kStorage StorageClass in
          // order for HLSL to emit this as a ByteAddressBuffer.
          b.create<ast::Variable>(b.Sym("buffer"), ast::StorageClass::kStorage,
                                  var_user->Variable()->Access(), buf_ast_ty,
                                  true, nullptr,
                                  ast::DecorationList{disable_validation}),
          b.Param("offset", b.ty.u32()),
      };

      // Other parameters are copied as-is:
      for (size_t i = 1; i < intrinsic->Parameters().size(); i++) {
        auto* param = intrinsic->Parameters()[i];
        auto* ty = CreateASTTypeFor(ctx, param->Type());
        params.emplace_back(b.Param("param_" + std::to_string(i), ty));
      }

      auto* atomic = IntrinsicAtomicFor(ctx.dst, op, el_ty);
      if (atomic == nullptr) {
        TINT_ICE(Transform, b.Diagnostics())
            << "IntrinsicAtomicFor() returned nullptr for op " << op
            << " and type " << el_ty->type_name();
      }

      auto* ret_ty = CreateASTTypeFor(ctx, intrinsic->ReturnType());
      auto* func = b.create<ast::Function>(
          b.Sym(), params, ret_ty, nullptr,
          ast::DecorationList{
              atomic,
              b.Disable(ast::DisabledValidation::kFunctionHasNoBody),
          },
          ast::DecorationList{});

      b.AST().AddFunction(func);
      return func->symbol;
    });
  }
};

DecomposeMemoryAccess::Intrinsic::Intrinsic(ProgramID pid,
                                            Op o,
                                            ast::StorageClass sc,
                                            DataType ty)
    : Base(pid), op(o), storage_class(sc), type(ty) {}
DecomposeMemoryAccess::Intrinsic::~Intrinsic() = default;
std::string DecomposeMemoryAccess::Intrinsic::InternalName() const {
  std::stringstream ss;
  switch (op) {
    case Op::kLoad:
      ss << "intrinsic_load_";
      break;
    case Op::kStore:
      ss << "intrinsic_store_";
      break;
    case Op::kAtomicLoad:
      ss << "intrinsic_atomic_load_";
      break;
    case Op::kAtomicStore:
      ss << "intrinsic_atomic_store_";
      break;
    case Op::kAtomicAdd:
      ss << "intrinsic_atomic_add_";
      break;
    case Op::kAtomicSub:
      ss << "intrinsic_atomic_sub_";
      break;
    case Op::kAtomicMax:
      ss << "intrinsic_atomic_max_";
      break;
    case Op::kAtomicMin:
      ss << "intrinsic_atomic_min_";
      break;
    case Op::kAtomicAnd:
      ss << "intrinsic_atomic_and_";
      break;
    case Op::kAtomicOr:
      ss << "intrinsic_atomic_or_";
      break;
    case Op::kAtomicXor:
      ss << "intrinsic_atomic_xor_";
      break;
    case Op::kAtomicExchange:
      ss << "intrinsic_atomic_exchange_";
      break;
    case Op::kAtomicCompareExchangeWeak:
      ss << "intrinsic_atomic_compare_exchange_weak_";
      break;
  }
  ss << storage_class << "_";
  switch (type) {
    case DataType::kU32:
      ss << "u32";
      break;
    case DataType::kF32:
      ss << "f32";
      break;
    case DataType::kI32:
      ss << "i32";
      break;
    case DataType::kVec2U32:
      ss << "vec2_u32";
      break;
    case DataType::kVec2F32:
      ss << "vec2_f32";
      break;
    case DataType::kVec2I32:
      ss << "vec2_i32";
      break;
    case DataType::kVec3U32:
      ss << "vec3_u32";
      break;
    case DataType::kVec3F32:
      ss << "vec3_f32";
      break;
    case DataType::kVec3I32:
      ss << "vec3_i32";
      break;
    case DataType::kVec4U32:
      ss << "vec4_u32";
      break;
    case DataType::kVec4F32:
      ss << "vec4_f32";
      break;
    case DataType::kVec4I32:
      ss << "vec4_i32";
      break;
  }
  return ss.str();
}

const DecomposeMemoryAccess::Intrinsic* DecomposeMemoryAccess::Intrinsic::Clone(
    CloneContext* ctx) const {
  return ctx->dst->ASTNodes().Create<DecomposeMemoryAccess::Intrinsic>(
      ctx->dst->ID(), op, storage_class, type);
}

DecomposeMemoryAccess::DecomposeMemoryAccess() = default;
DecomposeMemoryAccess::~DecomposeMemoryAccess() = default;

bool DecomposeMemoryAccess::ShouldRun(const Program* program,
                                      const DataMap&) const {
  for (auto* decl : program->AST().GlobalDeclarations()) {
    if (auto* var = program->Sem().Get<sem::Variable>(decl)) {
      if (var->StorageClass() == ast::StorageClass::kStorage ||
          var->StorageClass() == ast::StorageClass::kUniform) {
        return true;
      }
    }
  }
  return false;
}

void DecomposeMemoryAccess::Run(CloneContext& ctx,
                                const DataMap&,
                                DataMap&) const {
  auto& sem = ctx.src->Sem();

  State state(ctx);

  // Scan the AST nodes for storage and uniform buffer accesses. Complex
  // expression chains (e.g. `storage_buffer.foo.bar[20].x`) are handled by
  // maintaining an offset chain via the `state.TakeAccess()`,
  // `state.AddAccess()` methods.
  //
  // Inner-most expression nodes are guaranteed to be visited first because AST
  // nodes are fully immutable and require their children to be constructed
  // first so their pointer can be passed to the parent's constructor.
  for (auto* node : ctx.src->ASTNodes().Objects()) {
    if (auto* ident = node->As<ast::IdentifierExpression>()) {
      // X
      if (auto* var = sem.Get<sem::VariableUser>(ident)) {
        if (var->Variable()->StorageClass() == ast::StorageClass::kStorage ||
            var->Variable()->StorageClass() == ast::StorageClass::kUniform) {
          // Variable to a storage or uniform buffer
          state.AddAccess(ident, {
                                     var,
                                     state.ToOffset(0u),
                                     var->Type()->UnwrapRef(),
                                 });
        }
      }
      continue;
    }

    if (auto* accessor = node->As<ast::MemberAccessorExpression>()) {
      // X.Y
      auto* accessor_sem = sem.Get(accessor);
      if (auto* swizzle = accessor_sem->As<sem::Swizzle>()) {
        if (swizzle->Indices().size() == 1) {
          if (auto access = state.TakeAccess(accessor->structure)) {
            auto* vec_ty = access.type->As<sem::Vector>();
            auto* offset =
                state.Mul(vec_ty->type()->Size(), swizzle->Indices()[0]);
            state.AddAccess(accessor, {
                                          access.var,
                                          state.Add(access.offset, offset),
                                          vec_ty->type()->UnwrapRef(),
                                      });
          }
        }
      } else {
        if (auto access = state.TakeAccess(accessor->structure)) {
          auto* str_ty = access.type->As<sem::Struct>();
          auto* member = str_ty->FindMember(accessor->member->symbol);
          auto offset = member->Offset();
          state.AddAccess(accessor, {
                                        access.var,
                                        state.Add(access.offset, offset),
                                        member->Type()->UnwrapRef(),
                                    });
        }
      }
      continue;
    }

    if (auto* accessor = node->As<ast::IndexAccessorExpression>()) {
      if (auto access = state.TakeAccess(accessor->object)) {
        // X[Y]
        if (auto* arr = access.type->As<sem::Array>()) {
          auto* offset = state.Mul(arr->Stride(), accessor->index);
          state.AddAccess(accessor, {
                                        access.var,
                                        state.Add(access.offset, offset),
                                        arr->ElemType()->UnwrapRef(),
                                    });
          continue;
        }
        if (auto* vec_ty = access.type->As<sem::Vector>()) {
          auto* offset = state.Mul(vec_ty->type()->Size(), accessor->index);
          state.AddAccess(accessor, {
                                        access.var,
                                        state.Add(access.offset, offset),
                                        vec_ty->type()->UnwrapRef(),
                                    });
          continue;
        }
        if (auto* mat_ty = access.type->As<sem::Matrix>()) {
          auto* offset = state.Mul(mat_ty->ColumnStride(), accessor->index);
          state.AddAccess(accessor, {
                                        access.var,
                                        state.Add(access.offset, offset),
                                        mat_ty->ColumnType(),
                                    });
          continue;
        }
      }
    }

    if (auto* op = node->As<ast::UnaryOpExpression>()) {
      if (op->op == ast::UnaryOp::kAddressOf) {
        // &X
        if (auto access = state.TakeAccess(op->expr)) {
          // HLSL does not support pointers, so just take the access from the
          // reference and place it on the pointer.
          state.AddAccess(op, access);
          continue;
        }
      }
    }

    if (auto* assign = node->As<ast::AssignmentStatement>()) {
      // X = Y
      // Move the LHS access to a store.
      if (auto lhs = state.TakeAccess(assign->lhs)) {
        state.stores.emplace_back(Store{assign, lhs});
      }
    }

    if (auto* call_expr = node->As<ast::CallExpression>()) {
      auto* call = sem.Get(call_expr);
      if (auto* intrinsic = call->Target()->As<sem::Intrinsic>()) {
        if (intrinsic->Type() == sem::IntrinsicType::kArrayLength) {
          // arrayLength(X)
          // Don't convert X into a load, this intrinsic actually requires the
          // real pointer.
          state.TakeAccess(call_expr->args[0]);
          continue;
        }
        if (intrinsic->IsAtomic()) {
          if (auto access = state.TakeAccess(call_expr->args[0])) {
            // atomic___(X)
            ctx.Replace(call_expr, [=, &ctx, &state] {
              auto* buf = access.var->Declaration();
              auto* offset = access.offset->Build(ctx);
              auto* buf_ty = access.var->Type()->UnwrapRef();
              auto* el_ty = access.type->UnwrapRef()->As<sem::Atomic>()->Type();
              Symbol func =
                  state.AtomicFunc(buf_ty, el_ty, intrinsic,
                                   access.var->As<sem::VariableUser>());

              ast::ExpressionList args{ctx.Clone(buf), offset};
              for (size_t i = 1; i < call_expr->args.size(); i++) {
                auto* arg = call_expr->args[i];
                args.emplace_back(ctx.Clone(arg));
              }
              return ctx.dst->Call(func, args);
            });
          }
        }
      }
    }
  }

  // All remaining accesses are loads, transform these into calls to the
  // corresponding load function
  for (auto* expr : state.expression_order) {
    auto access_it = state.accesses.find(expr);
    if (access_it == state.accesses.end()) {
      continue;
    }
    BufferAccess access = access_it->second;
    ctx.Replace(expr, [=, &ctx, &state] {
      auto* buf = access.var->Declaration();
      auto* offset = access.offset->Build(ctx);
      auto* buf_ty = access.var->Type()->UnwrapRef();
      auto* el_ty = access.type->UnwrapRef();
      Symbol func =
          state.LoadFunc(buf_ty, el_ty, access.var->As<sem::VariableUser>());
      return ctx.dst->Call(func, ctx.CloneWithoutTransform(buf), offset);
    });
  }

  // And replace all storage and uniform buffer assignments with stores
  for (auto store : state.stores) {
    ctx.Replace(store.assignment, [=, &ctx, &state] {
      auto* buf = store.target.var->Declaration();
      auto* offset = store.target.offset->Build(ctx);
      auto* buf_ty = store.target.var->Type()->UnwrapRef();
      auto* el_ty = store.target.type->UnwrapRef();
      auto* value = store.assignment->rhs;
      Symbol func = state.StoreFunc(buf_ty, el_ty,
                                    store.target.var->As<sem::VariableUser>());
      auto* call = ctx.dst->Call(func, ctx.CloneWithoutTransform(buf), offset,
                                 ctx.Clone(value));
      return ctx.dst->CallStmt(call);
    });
  }

  ctx.Clone();
}

}  // namespace transform
}  // namespace tint

TINT_INSTANTIATE_TYPEINFO(tint::transform::Offset);
TINT_INSTANTIATE_TYPEINFO(tint::transform::OffsetLiteral);