src/tint/transform/spirv_atomic.cc

// Copyright 2022 The Tint Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "src/tint/transform/spirv_atomic.h"

#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>

#include "src/tint/program_builder.h"
#include "src/tint/sem/block_statement.h"
#include "src/tint/sem/function.h"
#include "src/tint/sem/index_accessor_expression.h"
#include "src/tint/sem/member_accessor_expression.h"
#include "src/tint/sem/statement.h"
#include "src/tint/switch.h"
#include "src/tint/type/reference.h"
#include "src/tint/utils/map.h"
#include "src/tint/utils/unique_vector.h"

TINT_INSTANTIATE_TYPEINFO(tint::transform::SpirvAtomic);
TINT_INSTANTIATE_TYPEINFO(tint::transform::SpirvAtomic::Stub);

namespace tint::transform {

using namespace tint::number_suffixes;  // NOLINT

/// PIMPL state for the transform
struct SpirvAtomic::State {
  private:
    /// A struct that has been forked because a subset of members were made atomic.
    struct ForkedStruct {
        Symbol name;
        std::unordered_set<size_t> atomic_members;
    };

    /// The source program
    const Program* const src;
    /// The target program builder
    ProgramBuilder b;
    /// The clone context
    CloneContext ctx = {&b, src, /* auto_clone_symbols */ true};
    std::unordered_map<const ast::Struct*, ForkedStruct> forked_structs;
    std::unordered_set<const sem::Variable*> atomic_variables;
    utils::UniqueVector<const sem::ValueExpression*, 8> atomic_expressions;

  public:
    /// Constructor
    /// @param program the source program
    explicit State(const Program* program) : src(program) {}

    /// Runs the transform
    /// @returns the new program or SkipTransform if the transform is not required
    ApplyResult Run() {
        bool made_changes = false;

        // Look for stub functions generated by the SPIR-V reader, which are used as placeholders
        // for atomic builtin calls.
        for (auto* fn : ctx.src->AST().Functions()) {
            if (auto* stub = ast::GetAttribute<Stub>(fn->attributes)) {
                auto* sem = ctx.src->Sem().Get(fn);

                for (auto* call : sem->CallSites()) {
                    // The first argument is always the atomic.
                    // The stub passes this by value, whereas the builtin wants a pointer.
                    // Take the address of the atomic argument.
                    auto& args = call->Declaration()->args;
                    auto out_args = ctx.Clone(args);
                    out_args[0] = b.AddressOf(out_args[0]);

                    // Replace all callsites of this stub to a call to the real builtin
                    if (stub->builtin == builtin::Function::kAtomicCompareExchangeWeak) {
                        // atomicCompareExchangeWeak returns a struct, so insert a call to it above
                        // the current statement, and replace the current call with the struct's
                        // `old_value` member.
                        auto* block = call->Stmt()->Block()->Declaration();
                        auto old_value = b.Symbols().New("old_value");
                        auto old_value_decl = b.Decl(b.Let(
                            old_value, b.MemberAccessor(
                                           b.Call(builtin::str(stub->builtin), std::move(out_args)),
                                           "old_value")));
                        ctx.InsertBefore(block->statements, call->Stmt()->Declaration(),
                                         old_value_decl);
                        ctx.Replace(call->Declaration(), b.Expr(old_value));
                    } else {
                        ctx.Replace(call->Declaration(),
                                    b.Call(builtin::str(stub->builtin), std::move(out_args)));
                    }

                    // Keep track of this expression. We'll need to modify the root identifier /
                    // structure to be atomic.
                    atomic_expressions.Add(ctx.src->Sem().GetVal(args[0]));
                }

                // Remove the stub from the output program
                ctx.Remove(ctx.src->AST().GlobalDeclarations(), fn);
                made_changes = true;
            }
        }

        if (!made_changes) {
            return SkipTransform;
        }

        // Transform all variables and structure members that were used in atomic operations as
        // atomic types. This propagates up originating expression chains.
        ProcessAtomicExpressions();

        // If we need to change structure members, then fork them.
        if (!forked_structs.empty()) {
            ctx.ReplaceAll([&](const ast::Struct* str) {
                // Is `str` a structure we need to fork?
                if (auto it = forked_structs.find(str); it != forked_structs.end()) {
                    const auto& forked = it->second;

                    // Re-create the structure swapping in the atomic-flavoured members
                    utils::Vector<const ast::StructMember*, 8> members;
                    members.Reserve(str->members.Length());
                    for (size_t i = 0; i < str->members.Length(); i++) {
                        auto* member = str->members[i];
                        if (forked.atomic_members.count(i)) {
                            auto type = AtomicTypeFor(ctx.src->Sem().Get(member)->Type());
                            auto name = ctx.src->Symbols().NameFor(member->name->symbol);
                            members.Push(b.Member(name, type, ctx.Clone(member->attributes)));
                        } else {
                            members.Push(ctx.Clone(member));
                        }
                    }
                    b.Structure(forked.name, std::move(members));
                }
                return nullptr;
            });
        }

        // Replace assignments and decls from atomic variables with atomicLoads, and assignments to
        // atomic variables with atomicStores.
        ReplaceLoadsAndStores();

        ctx.Clone();
        return Program(std::move(b));
    }

  private:
    ForkedStruct& Fork(const ast::Struct* str) {
        auto& forked = forked_structs[str];
        if (!forked.name.IsValid()) {
            forked.name =
                b.Symbols().New(ctx.src->Symbols().NameFor(str->name->symbol) + "_atomic");
        }
        return forked;
    }

    void ProcessAtomicExpressions() {
        for (size_t i = 0; i < atomic_expressions.Length(); i++) {
            Switch(
                atomic_expressions[i]->UnwrapLoad(),  //
                [&](const sem::VariableUser* user) {
                    auto* v = user->Variable()->Declaration();
                    if (v->type && atomic_variables.emplace(user->Variable()).second) {
                        ctx.Replace(v->type.expr, b.Expr(AtomicTypeFor(user->Variable()->Type())));
                    }
                    if (auto* ctor = user->Variable()->Initializer()) {
                        atomic_expressions.Add(ctor);
                    }
                },
                [&](const sem::StructMemberAccess* access) {
                    // Fork the struct (the first time) and mark member(s) that need to be made
                    // atomic.
                    auto* member = access->Member();
                    Fork(member->Struct()->Declaration()).atomic_members.emplace(member->Index());
                    atomic_expressions.Add(access->Object());
                },
                [&](const sem::IndexAccessorExpression* index) {
                    atomic_expressions.Add(index->Object());
                },
                [&](const sem::ValueExpression* e) {
                    if (auto* unary = e->Declaration()->As<ast::UnaryOpExpression>()) {
                        atomic_expressions.Add(ctx.src->Sem().GetVal(unary->expr));
                    }
                });
        }
    }

    ast::Type AtomicTypeFor(const type::Type* ty) {
        return Switch(
            ty,  //
            [&](const type::I32*) { return b.ty.atomic(CreateASTTypeFor(ctx, ty)); },
            [&](const type::U32*) { return b.ty.atomic(CreateASTTypeFor(ctx, ty)); },
            [&](const sem::Struct* str) { return b.ty(Fork(str->Declaration()).name); },
            [&](const type::Array* arr) {
                if (arr->Count()->Is<type::RuntimeArrayCount>()) {
                    return b.ty.array(AtomicTypeFor(arr->ElemType()));
                }
                auto count = arr->ConstantCount();
                if (!count) {
                    ctx.dst->Diagnostics().add_error(
                        diag::System::Transform,
                        "the SpirvAtomic transform does not currently support array counts that "
                        "use override values");
                    count = 1;
                }
                return b.ty.array(AtomicTypeFor(arr->ElemType()), u32(count.value()));
            },
            [&](const type::Pointer* ptr) {
                return b.ty.pointer(AtomicTypeFor(ptr->StoreType()), ptr->AddressSpace(),
                                    ptr->Access());
            },
            [&](const type::Reference* ref) { return AtomicTypeFor(ref->StoreType()); },
            [&](Default) {
                TINT_ICE(Transform, b.Diagnostics())
                    << "unhandled type: " << ty->FriendlyName(ctx.src->Symbols());
                return ast::Type{};
            });
    }

    void ReplaceLoadsAndStores() {
        // Returns true if 'e' is a reference to an atomic variable or struct member
        auto is_ref_to_atomic_var = [&](const sem::ValueExpression* e) {
            if (tint::Is<type::Reference>(e->Type()) && e->RootIdentifier() &&
                (atomic_variables.count(e->RootIdentifier()) != 0)) {
                // If it's a struct member, make sure it's one we marked as atomic
                if (auto* ma = e->As<sem::StructMemberAccess>()) {
                    auto it = forked_structs.find(ma->Member()->Struct()->Declaration());
                    if (it != forked_structs.end()) {
                        auto& forked = it->second;
                        return forked.atomic_members.count(ma->Member()->Index()) != 0;
                    }
                }
                return true;
            }
            return false;
        };

        // Look for loads and stores via assignments and decls of atomic variables we've collected
        // so far, and replace them with atomicLoad and atomicStore.
        for (auto* atomic_var : atomic_variables) {
            for (auto* vu : atomic_var->Users()) {
                Switch(
                    vu->Stmt()->Declaration(),
                    [&](const ast::AssignmentStatement* assign) {
                        auto* sem_lhs = ctx.src->Sem().GetVal(assign->lhs);
                        if (is_ref_to_atomic_var(sem_lhs)) {
                            ctx.Replace(assign, [=] {
                                auto* lhs = ctx.CloneWithoutTransform(assign->lhs);
                                auto* rhs = ctx.CloneWithoutTransform(assign->rhs);
                                auto* call = b.Call(builtin::str(builtin::Function::kAtomicStore),
                                                    b.AddressOf(lhs), rhs);
                                return b.CallStmt(call);
                            });
                            return;
                        }

                        auto sem_rhs = ctx.src->Sem().GetVal(assign->rhs);
                        if (is_ref_to_atomic_var(sem_rhs->UnwrapLoad())) {
                            ctx.Replace(assign->rhs, [=] {
                                auto* rhs = ctx.CloneWithoutTransform(assign->rhs);
                                return b.Call(builtin::str(builtin::Function::kAtomicLoad),
                                              b.AddressOf(rhs));
                            });
                            return;
                        }
                    },
                    [&](const ast::VariableDeclStatement* decl) {
                        auto* var = decl->variable;
                        if (auto* sem_init = ctx.src->Sem().GetVal(var->initializer)) {
                            if (is_ref_to_atomic_var(sem_init->UnwrapLoad())) {
                                ctx.Replace(var->initializer, [=] {
                                    auto* rhs = ctx.CloneWithoutTransform(var->initializer);
                                    return b.Call(builtin::str(builtin::Function::kAtomicLoad),
                                                  b.AddressOf(rhs));
                                });
                                return;
                            }
                        }
                    });
            }
        }
    }
};

SpirvAtomic::SpirvAtomic() = default;
SpirvAtomic::~SpirvAtomic() = default;

SpirvAtomic::Stub::Stub(ProgramID pid, ast::NodeID nid, builtin::Function b)
    : Base(pid, nid, utils::Empty), builtin(b) {}
SpirvAtomic::Stub::~Stub() = default;
std::string SpirvAtomic::Stub::InternalName() const {
    return "@internal(spirv-atomic " + std::string(builtin::str(builtin)) + ")";
}

const SpirvAtomic::Stub* SpirvAtomic::Stub::Clone(CloneContext* ctx) const {
    return ctx->dst->ASTNodes().Create<SpirvAtomic::Stub>(ctx->dst->ID(),
                                                          ctx->dst->AllocateNodeID(), builtin);
}

Transform::ApplyResult SpirvAtomic::Apply(const Program* src, const DataMap&, DataMap&) const {
    return State{src}.Run();
}

}  // namespace tint::transform