| // Copyright 2022 The Dawn & Tint Authors |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are met: |
| // |
| // 1. Redistributions of source code must retain the above copyright notice, this |
| // list of conditions and the following disclaimer. |
| // |
| // 2. Redistributions in binary form must reproduce the above copyright notice, |
| // this list of conditions and the following disclaimer in the documentation |
| // and/or other materials provided with the distribution. |
| // |
| // 3. Neither the name of the copyright holder nor the names of its |
| // contributors may be used to endorse or promote products derived from |
| // this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
| // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE |
| // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
| // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
| // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
| // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
| #include "src/tint/lang/core/constant/eval.h" |
| |
| #include <algorithm> |
| #include <iomanip> |
| #include <limits> |
| #include <optional> |
| #include <string> |
| #include <type_traits> |
| #include <utility> |
| |
| #include "src/tint/lang/core/constant/composite.h" |
| #include "src/tint/lang/core/constant/scalar.h" |
| #include "src/tint/lang/core/constant/splat.h" |
| #include "src/tint/lang/core/constant/value.h" |
| #include "src/tint/lang/core/fluent_types.h" |
| #include "src/tint/lang/core/number.h" |
| #include "src/tint/lang/core/type/abstract_float.h" |
| #include "src/tint/lang/core/type/abstract_int.h" |
| #include "src/tint/lang/core/type/bool.h" |
| #include "src/tint/lang/core/type/f16.h" |
| #include "src/tint/lang/core/type/f32.h" |
| #include "src/tint/lang/core/type/i32.h" |
| #include "src/tint/lang/core/type/matrix.h" |
| #include "src/tint/lang/core/type/struct.h" |
| #include "src/tint/lang/core/type/u32.h" |
| #include "src/tint/lang/core/type/vector.h" |
| #include "src/tint/utils/containers/map.h" |
| #include "src/tint/utils/containers/transform.h" |
| #include "src/tint/utils/diagnostic/diagnostic.h" |
| #include "src/tint/utils/macros/compiler.h" |
| #include "src/tint/utils/memory/bitcast.h" |
| #include "src/tint/utils/rtti/switch.h" |
| #include "src/tint/utils/text/string_stream.h" |
| |
| using namespace tint::core::fluent_types; // NOLINT |
| using namespace tint::core::number_suffixes; // NOLINT |
| |
| namespace tint::core::constant { |
| namespace { |
| |
| /// Returns the first element of a parameter pack |
| template <typename T> |
| T First(T&& first, ...) { |
| return std::forward<T>(first); |
| } |
| |
| /// Helper that calls `f` passing in the value of all `cs`. |
| /// Calls `f` with all constants cast to the type of the first `cs` argument. |
| template <typename F, typename... CONSTANTS> |
| auto Dispatch_iu32(F&& f, CONSTANTS&&... cs) { |
| return Switch( |
| First(cs...)->Type(), // |
| [&](const core::type::I32*) { return f(cs->template ValueAs<i32>()...); }, |
| [&](const core::type::U32*) { return f(cs->template ValueAs<u32>()...); }); |
| } |
| |
| /// Helper that calls `f` passing in the value of all `cs`. |
| /// Calls `f` with all constants cast to the type of the first `cs` argument. |
| template <typename F, typename... CONSTANTS> |
| auto Dispatch_ia_iu32(F&& f, CONSTANTS&&... cs) { |
| return Switch( |
| First(cs...)->Type(), // |
| [&](const core::type::AbstractInt*) { return f(cs->template ValueAs<AInt>()...); }, |
| [&](const core::type::I32*) { return f(cs->template ValueAs<i32>()...); }, |
| [&](const core::type::U32*) { return f(cs->template ValueAs<u32>()...); }); |
| } |
| |
| /// Helper that calls `f` passing in the value of all `cs`. |
| /// Calls `f` with all constants cast to the type of the first `cs` argument. |
| template <typename F, typename... CONSTANTS> |
| auto Dispatch_ia_iu32_bool(F&& f, CONSTANTS&&... cs) { |
| return Switch( |
| First(cs...)->Type(), // |
| [&](const core::type::AbstractInt*) { return f(cs->template ValueAs<AInt>()...); }, |
| [&](const core::type::I32*) { return f(cs->template ValueAs<i32>()...); }, |
| [&](const core::type::U32*) { return f(cs->template ValueAs<u32>()...); }, |
| [&](const core::type::Bool*) { return f(cs->template ValueAs<bool>()...); }); |
| } |
| |
| /// Helper that calls `f` passing in the value of all `cs`. |
| /// Calls `f` with all constants cast to the type of the first `cs` argument. |
| template <typename F, typename... CONSTANTS> |
| auto Dispatch_fia_fi32_f16(F&& f, CONSTANTS&&... cs) { |
| return Switch( |
| First(cs...)->Type(), // |
| [&](const core::type::AbstractInt*) { return f(cs->template ValueAs<AInt>()...); }, |
| [&](const core::type::AbstractFloat*) { return f(cs->template ValueAs<AFloat>()...); }, |
| [&](const core::type::F32*) { return f(cs->template ValueAs<f32>()...); }, |
| [&](const core::type::I32*) { return f(cs->template ValueAs<i32>()...); }, |
| [&](const core::type::F16*) { return f(cs->template ValueAs<f16>()...); }); |
| } |
| |
| /// Helper that calls `f` passing in the value of all `cs`. |
| /// Calls `f` with all constants cast to the type of the first `cs` argument. |
| template <typename F, typename... CONSTANTS> |
| auto Dispatch_fia_fiu32_f16(F&& f, CONSTANTS&&... cs) { |
| return Switch( |
| First(cs...)->Type(), // |
| [&](const core::type::AbstractInt*) { return f(cs->template ValueAs<AInt>()...); }, |
| [&](const core::type::AbstractFloat*) { return f(cs->template ValueAs<AFloat>()...); }, |
| [&](const core::type::F32*) { return f(cs->template ValueAs<f32>()...); }, |
| [&](const core::type::I32*) { return f(cs->template ValueAs<i32>()...); }, |
| [&](const core::type::U32*) { return f(cs->template ValueAs<u32>()...); }, |
| [&](const core::type::F16*) { return f(cs->template ValueAs<f16>()...); }); |
| } |
| |
| /// Helper that calls `f` passing in the value of all `cs`. |
| /// Calls `f` with all constants cast to the type of the first `cs` argument. |
| template <typename F, typename... CONSTANTS> |
| auto Dispatch_fia_fiu32_f16_bool(F&& f, CONSTANTS&&... cs) { |
| return Switch( |
| First(cs...)->Type(), // |
| [&](const core::type::AbstractInt*) { return f(cs->template ValueAs<AInt>()...); }, |
| [&](const core::type::AbstractFloat*) { return f(cs->template ValueAs<AFloat>()...); }, |
| [&](const core::type::F32*) { return f(cs->template ValueAs<f32>()...); }, |
| [&](const core::type::I32*) { return f(cs->template ValueAs<i32>()...); }, |
| [&](const core::type::U32*) { return f(cs->template ValueAs<u32>()...); }, |
| [&](const core::type::F16*) { return f(cs->template ValueAs<f16>()...); }, |
| [&](const core::type::Bool*) { return f(cs->template ValueAs<bool>()...); }); |
| } |
| |
| /// Helper that calls `f` passing in the value of all `cs`. |
| /// Calls `f` with all constants cast to the type of the first `cs` argument. |
| template <typename F, typename... CONSTANTS> |
| auto Dispatch_fa_f32_f16(F&& f, CONSTANTS&&... cs) { |
| return Switch( |
| First(cs...)->Type(), // |
| [&](const core::type::AbstractFloat*) { return f(cs->template ValueAs<AFloat>()...); }, |
| [&](const core::type::F32*) { return f(cs->template ValueAs<f32>()...); }, |
| [&](const core::type::F16*) { return f(cs->template ValueAs<f16>()...); }); |
| } |
| |
| /// Helper that calls `f` passing in the value of all `cs`. |
| /// Calls `f` with all constants cast to the type of the first `cs` argument. |
| template <typename F, typename... CONSTANTS> |
| auto Dispatch_bool(F&& f, CONSTANTS&&... cs) { |
| return f(cs->template ValueAs<bool>()...); |
| } |
| |
| /// ZeroTypeDispatch is a helper for calling the function `f`, passing a single zero-value argument |
| /// of the C++ type that corresponds to the core::type::Type `type`. For example, calling |
| /// `ZeroTypeDispatch()` with a type of `type::I32*` will call the function f with a single argument |
| /// of `i32(0)`. |
| /// @returns the value returned by calling `f`. |
| /// @note `type` must be a scalar or abstract numeric type. Other types will not call `f`, and will |
| /// return the zero-initialized value of the return type for `f`. |
| template <typename F> |
| auto ZeroTypeDispatch(const core::type::Type* type, F&& f) { |
| return Switch( |
| type, // |
| [&](const core::type::AbstractInt*) { return f(AInt(0)); }, // |
| [&](const core::type::AbstractFloat*) { return f(AFloat(0)); }, // |
| [&](const core::type::I32*) { return f(i32(0)); }, // |
| [&](const core::type::U32*) { return f(u32(0)); }, // |
| [&](const core::type::F32*) { return f(f32(0)); }, // |
| [&](const core::type::F16*) { return f(f16(0)); }, // |
| [&](const core::type::Bool*) { return f(static_cast<bool>(0)); }); |
| } |
| |
| template <typename NumberT> |
| std::string OverflowErrorMessage(NumberT lhs, const char* op, NumberT rhs) { |
| StringStream ss; |
| ss << "'" << lhs.value << " " << op << " " << rhs.value << "' cannot be represented as '" |
| << FriendlyName<NumberT>() << "'"; |
| return ss.str(); |
| } |
| |
| template <typename VALUE_TY> |
| std::string OverflowErrorMessage(VALUE_TY value, std::string_view target_ty) { |
| StringStream ss; |
| ss << "value " << value << " cannot be represented as " << "'" << target_ty << "'"; |
| return ss.str(); |
| } |
| |
| template <typename NumberT> |
| std::string OverflowExpErrorMessage(std::string_view base, NumberT exp) { |
| StringStream ss; |
| ss << base << "^" << exp << " cannot be represented as " << "'" << FriendlyName<NumberT>() |
| << "'"; |
| return ss.str(); |
| } |
| |
| /// @returns the number of consecutive leading bits in `@p e` set to `@p bit_value_to_count`. |
| template <typename T> |
| std::make_unsigned_t<T> CountLeadingBits(T e, T bit_value_to_count) { |
| using UT = std::make_unsigned_t<T>; |
| constexpr UT kNumBits = sizeof(UT) * 8; |
| constexpr UT kLeftMost = UT{1} << (kNumBits - 1); |
| const UT b = bit_value_to_count == 0 ? UT{0} : kLeftMost; |
| |
| auto v = static_cast<UT>(e); |
| auto count = UT{0}; |
| while ((count < kNumBits) && ((v & kLeftMost) == b)) { |
| ++count; |
| v <<= 1; |
| } |
| return count; |
| } |
| |
| /// @returns the number of consecutive trailing bits set to `@p bit_value_to_count` in `@p e` |
| template <typename T> |
| std::make_unsigned_t<T> CountTrailingBits(T e, T bit_value_to_count) { |
| using UT = std::make_unsigned_t<T>; |
| constexpr UT kNumBits = sizeof(UT) * 8; |
| constexpr UT kRightMost = UT{1}; |
| const UT b = static_cast<UT>(bit_value_to_count); |
| |
| auto v = static_cast<UT>(e); |
| auto count = UT{0}; |
| while ((count < kNumBits) && ((v & kRightMost) == b)) { |
| ++count; |
| v >>= 1; |
| } |
| return count; |
| } |
| |
| /// Common data for constant conversion. |
| struct ConvertContext { |
| Manager& mgr; |
| diag::List& diags; |
| const Source& source; |
| bool use_runtime_semantics; |
| }; |
| |
| /// Converts the constant scalar value to the target type. |
| /// @returns the converted scalar, or nullptr on error. |
| template <typename T> |
| const ScalarBase* ScalarConvert(const Scalar<T>* scalar, |
| const core::type::Type* target_ty, |
| ConvertContext& ctx) { |
| TINT_BEGIN_DISABLE_WARNING(UNREACHABLE_CODE); |
| if (target_ty == scalar->type) { |
| // If the types are identical, then no conversion is needed. |
| return scalar; |
| } |
| return ZeroTypeDispatch(target_ty, [&](auto zero_to) -> const ScalarBase* { |
| // `value` is the source value. |
| // `FROM` is the source type. |
| // `TO` is the target type. |
| using TO = std::decay_t<decltype(zero_to)>; |
| using FROM = T; |
| if constexpr (std::is_same_v<TO, bool>) { |
| // [x -> bool] |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, !scalar->IsZero()); |
| } else if constexpr (std::is_same_v<FROM, bool>) { |
| // [bool -> x] |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, TO(scalar->value ? 1 : 0)); |
| } else if (auto conv = CheckedConvert<TO>(scalar->value); conv == Success) { |
| // Conversion success |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, conv.Get()); |
| // --- Below this point are the failure cases --- |
| } else if constexpr (IsAbstract<FROM>) { |
| // [abstract-numeric -> x] - materialization failure |
| auto msg = OverflowErrorMessage(scalar->value, target_ty->FriendlyName()); |
| if (ctx.use_runtime_semantics) { |
| ctx.diags.AddWarning(ctx.source) << msg; |
| switch (conv.Failure()) { |
| case ConversionFailure::kExceedsNegativeLimit: |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, TO::Lowest()); |
| case ConversionFailure::kExceedsPositiveLimit: |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, TO::Highest()); |
| } |
| } else { |
| ctx.diags.AddError(ctx.source) << msg; |
| return nullptr; |
| } |
| } else if constexpr (IsFloatingPoint<TO>) { |
| // [x -> floating-point] - number not exactly representable |
| // https://www.w3.org/TR/WGSL/#floating-point-conversion |
| auto msg = OverflowErrorMessage(scalar->value, target_ty->FriendlyName()); |
| if (ctx.use_runtime_semantics) { |
| ctx.diags.AddWarning(ctx.source) << msg; |
| switch (conv.Failure()) { |
| case ConversionFailure::kExceedsNegativeLimit: |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, TO::Lowest()); |
| case ConversionFailure::kExceedsPositiveLimit: |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, TO::Highest()); |
| } |
| } else { |
| ctx.diags.AddError(ctx.source) << msg; |
| return nullptr; |
| } |
| } else if constexpr (IsFloatingPoint<FROM>) { |
| // [floating-point -> integer] - number not exactly representable |
| // https://www.w3.org/TR/WGSL/#floating-point-conversion |
| switch (conv.Failure()) { |
| case ConversionFailure::kExceedsNegativeLimit: |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, TO::Lowest()); |
| case ConversionFailure::kExceedsPositiveLimit: |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, TO::Highest()); |
| } |
| } else if constexpr (IsIntegral<FROM>) { |
| // [integer -> integer] - number not exactly representable |
| // Static cast |
| return ctx.mgr.Get<Scalar<TO>>(target_ty, static_cast<TO>(scalar->value)); |
| } |
| TINT_UNREACHABLE() << "Expression is not constant"; |
| return nullptr; |
| }); |
| TINT_END_DISABLE_WARNING(UNREACHABLE_CODE); |
| } |
| |
| /// Converts the constant value to the target type. |
| /// @returns the converted value, or nullptr on error. |
| const Value* ConvertInternal(const Value* root_value, |
| const core::type::Type* root_target_ty, |
| ConvertContext& ctx) { |
| struct ActionConvert { |
| const Value* value = nullptr; |
| const core::type::Type* target_ty = nullptr; |
| }; |
| struct ActionBuildSplat { |
| size_t count = 0; |
| const core::type::Type* type = nullptr; |
| }; |
| struct ActionBuildComposite { |
| size_t count = 0; |
| const core::type::Type* type = nullptr; |
| }; |
| using Action = std::variant<ActionConvert, ActionBuildSplat, ActionBuildComposite>; |
| |
| Vector<Action, 8> pending{ |
| ActionConvert{root_value, root_target_ty}, |
| }; |
| |
| Vector<const Value*, 32> value_stack; |
| |
| while (!pending.IsEmpty()) { |
| auto next = pending.Pop(); |
| |
| if (auto* build = std::get_if<ActionBuildSplat>(&next)) { |
| TINT_ASSERT(value_stack.Length() >= 1); |
| auto* el = value_stack.Pop(); |
| value_stack.Push(ctx.mgr.Splat(build->type, el)); |
| continue; |
| } |
| |
| if (auto* build = std::get_if<ActionBuildComposite>(&next)) { |
| TINT_ASSERT(value_stack.Length() >= build->count); |
| // Take build->count elements off the top of value_stack |
| // Note: The values are ordered with the first composite value at the top of the stack. |
| Vector<const Value*, 32> elements; |
| elements.Reserve(build->count); |
| for (size_t i = 0; i < build->count; i++) { |
| elements.Push(value_stack.Pop()); |
| } |
| // Build the composite |
| value_stack.Push(ctx.mgr.Composite(build->type, std::move(elements))); |
| continue; |
| } |
| |
| auto* convert = std::get_if<ActionConvert>(&next); |
| |
| bool ok = Switch( |
| convert->value, |
| [&](const ScalarBase* scalar) { |
| auto* converted = Switch( |
| scalar, |
| [&](const Scalar<AFloat>* val) { |
| return ScalarConvert(val, convert->target_ty, ctx); |
| }, |
| [&](const Scalar<AInt>* val) { |
| return ScalarConvert(val, convert->target_ty, ctx); |
| }, |
| [&](const Scalar<u32>* val) { |
| return ScalarConvert(val, convert->target_ty, ctx); |
| }, |
| [&](const Scalar<i32>* val) { |
| return ScalarConvert(val, convert->target_ty, ctx); |
| }, |
| [&](const Scalar<f32>* val) { |
| return ScalarConvert(val, convert->target_ty, ctx); |
| }, |
| [&](const Scalar<f16>* val) { |
| return ScalarConvert(val, convert->target_ty, ctx); |
| }, |
| [&](const Scalar<bool>* val) { |
| return ScalarConvert(val, convert->target_ty, ctx); |
| }); |
| if (!converted) { |
| return false; |
| } |
| value_stack.Push(converted); |
| return true; |
| }, |
| [&](const Splat* splat) { |
| const core::type::Type* target_el_ty = nullptr; |
| if (auto* str = convert->target_ty->As<core::type::Struct>()) { |
| // Structure conversion. |
| auto members = str->Members(); |
| target_el_ty = members[0]->Type(); |
| |
| // Structures can only be converted during materialization. The user cannot |
| // declare the target structure type, so each member type must be the same |
| // default materialization type. |
| for (size_t i = 1; i < members.Length(); i++) { |
| if (members[i]->Type() != target_el_ty) { |
| TINT_ICE() |
| << "inconsistent target struct member types for SplatConvert"; |
| } |
| } |
| } else { |
| target_el_ty = convert->target_ty->Elements(convert->target_ty).type; |
| } |
| |
| // Convert the single splatted element type. |
| pending.Push(ActionBuildSplat{splat->count, convert->target_ty}); |
| pending.Push(ActionConvert{splat->el, target_el_ty}); |
| return true; |
| }, |
| [&](const Composite* composite) { |
| const size_t el_count = composite->NumElements(); |
| |
| // Build the new composite from the converted element types. |
| pending.Push(ActionBuildComposite{el_count, convert->target_ty}); |
| |
| if (auto* str = convert->target_ty->As<core::type::Struct>()) { |
| if (TINT_UNLIKELY(str->Members().Length() != el_count)) { |
| TINT_ICE() |
| << "const-eval conversion of structure has mismatched element counts"; |
| } |
| // Struct composites can have different types for each member. |
| auto members = str->Members(); |
| for (size_t i = 0; i < el_count; i++) { |
| pending.Push(ActionConvert{composite->Index(i), members[i]->Type()}); |
| } |
| } else { |
| // Non-struct composites have the same type for all elements. |
| auto* el_ty = convert->target_ty->Elements(convert->target_ty).type; |
| for (size_t i = 0; i < el_count; i++) { |
| auto* el = composite->Index(i); |
| pending.Push(ActionConvert{el, el_ty}); |
| } |
| } |
| |
| return true; |
| }); |
| if (!ok) { |
| return nullptr; |
| } |
| } |
| |
| TINT_ASSERT(value_stack.Length() == 1); |
| return value_stack.Pop(); |
| } |
| |
| /// TransformElements constructs a new constant of type `composite_ty` by applying the |
| /// transformation function `f` on each of the most deeply nested elements of 'cs'. Assumes that all |
| /// input constants `cs` are of the same arity (all scalars or all vectors of the same size). |
| /// If `f`'s last argument is a `size_t`, then the index of the most deeply nested element inside |
| /// the most deeply nested aggregate type will be passed in. |
| template <typename F, typename... CONSTANTS> |
| tint::traits::EnableIf<tint::traits::IsType<size_t, tint::traits::LastParameterType<F>>, |
| Eval::Result> |
| TransformElements(Manager& mgr, |
| const core::type::Type* composite_ty, |
| const F& f, |
| size_t index, |
| CONSTANTS&&... cs) { |
| auto [el_ty, n] = First(cs...)->Type()->Elements(); |
| if (!el_ty) { |
| return f(cs..., index); |
| } |
| |
| auto* composite_el_ty = composite_ty->Elements(composite_ty).type; |
| |
| Vector<const Value*, 8> els; |
| els.Reserve(n); |
| for (uint32_t i = 0; i < n; i++) { |
| if (auto el = TransformElements(mgr, composite_el_ty, f, index + i, cs->Index(i)...); |
| el == Success) { |
| els.Push(el.Get()); |
| } else { |
| return el.Failure(); |
| } |
| } |
| return mgr.Composite(composite_ty, std::move(els)); |
| } |
| |
| /// Signature of a unary transformation callback |
| using UnaryTransform = std::function<Eval::Result(const Value*)>; |
| |
| /// TransformUnaryElements constructs a new constant of type `composite_ty` by applying the |
| /// transformation function 'f' on each of the most deeply nested elements of `c0`. |
| Eval::Result TransformUnaryElements(Manager& mgr, |
| const core::type::Type* composite_ty, |
| const UnaryTransform& f, |
| const Value* c0) { |
| auto [el_ty, n] = c0->Type()->Elements(); |
| if (!el_ty) { |
| return f(c0); |
| } |
| |
| auto* composite_el_ty = composite_ty->Elements(composite_ty).type; |
| |
| Vector<const Value*, 8> els; |
| els.Reserve(n); |
| for (uint32_t i = 0; i < n; i++) { |
| if (auto el = TransformUnaryElements(mgr, composite_el_ty, f, c0->Index(i)); |
| el == Success) { |
| els.Push(el.Get()); |
| } else { |
| return el.Failure(); |
| } |
| } |
| return mgr.Composite(composite_ty, std::move(els)); |
| } |
| |
| /// Signature of a binary transformation callback. |
| using BinaryTransform = std::function<Eval::Result(const Value*, const Value*)>; |
| |
| /// TransformBinaryElements constructs a new constant of type `composite_ty` by applying the |
| /// transformation function 'f' on each of the most deeply nested elements of both `c0` and `c1`. |
| Eval::Result TransformBinaryElements(Manager& mgr, |
| const core::type::Type* composite_ty, |
| const BinaryTransform& f, |
| const Value* c0, |
| const Value* c1) { |
| auto [el_ty, n] = c0->Type()->Elements(); |
| if (!el_ty) { |
| return f(c0, c1); |
| } |
| |
| TINT_ASSERT(n == c1->Type()->Elements().count); |
| |
| auto* composite_el_ty = composite_ty->Elements(composite_ty).type; |
| |
| Vector<const Value*, 8> els; |
| els.Reserve(n); |
| for (uint32_t i = 0; i < n; i++) { |
| if (auto el = TransformBinaryElements(mgr, composite_el_ty, f, c0->Index(i), c1->Index(i)); |
| el == Success) { |
| els.Push(el.Get()); |
| } else { |
| return el.Failure(); |
| } |
| } |
| return mgr.Composite(composite_ty, std::move(els)); |
| } |
| |
| /// TransformBinaryDifferingArityElements constructs a new constant of type `composite_ty` by |
| /// applying the transformation function 'f' on each of the most deeply nested elements of both `c0` |
| /// and `c1`. Unlike TransformElements, this function handles the constants being of different |
| /// arity, e.g. vector-scalar, scalar-vector. |
| Eval::Result TransformBinaryDifferingArityElements(Manager& mgr, |
| const core::type::Type* composite_ty, |
| const BinaryTransform& f, |
| const Value* c0, |
| const Value* c1) { |
| uint32_t n0 = c0->Type()->Elements(nullptr, 1).count; |
| uint32_t n1 = c1->Type()->Elements(nullptr, 1).count; |
| uint32_t max_n = std::max(n0, n1); |
| // If arity of both constants is 1, invoke callback |
| if (max_n == 1u) { |
| return f(c0, c1); |
| } |
| |
| const auto* element_ty = composite_ty->Elements(composite_ty).type; |
| |
| Vector<const Value*, 8> els; |
| els.Reserve(max_n); |
| for (uint32_t i = 0; i < max_n; i++) { |
| auto nested_or_self = [&](auto* c, uint32_t num_elems) { |
| return (num_elems == 1) ? c : c->Index(i); |
| }; |
| if (auto el = TransformBinaryDifferingArityElements( |
| mgr, element_ty, f, nested_or_self(c0, n0), nested_or_self(c1, n1)); |
| el == Success) { |
| els.Push(el.Get()); |
| } else { |
| return el.Failure(); |
| } |
| } |
| return mgr.Composite(composite_ty, std::move(els)); |
| } |
| |
| /// Signature of a ternary transformation callback |
| using TernaryTransform = std::function<Eval::Result(const Value*, const Value*, const Value*)>; |
| |
| /// TransformTernaryElements constructs a new constant of type `composite_ty` by applying the |
| /// transformation function 'f' on each of the most deeply nested elements of both `c0`, `c1`, and |
| /// `c2`. |
| Eval::Result TransformTernaryElements(Manager& mgr, |
| const core::type::Type* composite_ty, |
| const TernaryTransform& f, |
| const Value* c0, |
| const Value* c1, |
| const Value* c2) { |
| auto [el_ty, n] = c0->Type()->Elements(); |
| if (!el_ty) { |
| return f(c0, c1, c2); |
| } |
| |
| auto* composite_el_ty = composite_ty->Elements(composite_ty).type; |
| |
| Vector<const Value*, 8> els; |
| els.Reserve(n); |
| for (uint32_t i = 0; i < n; i++) { |
| if (auto el = TransformTernaryElements(mgr, composite_el_ty, f, c0->Index(i), c1->Index(i), |
| c2->Index(i)); |
| el == Success) { |
| els.Push(el.Get()); |
| } else { |
| return el.Failure(); |
| } |
| } |
| return mgr.Composite(composite_ty, std::move(els)); |
| } |
| } // namespace |
| |
| Eval::Eval(Manager& manager, diag::List& diagnostics, bool use_runtime_semantics /* = false */) |
| : mgr(manager), diags(diagnostics), use_runtime_semantics_(use_runtime_semantics) {} |
| |
| template <typename T> |
| Eval::Result Eval::CreateScalar(const Source& source, const core::type::Type* t, T v) { |
| static_assert(IsNumber<T> || std::is_same_v<T, bool>, "T must be a Number or bool"); |
| TINT_ASSERT(t->Is<core::type::Scalar>()); |
| |
| if constexpr (IsFloatingPoint<T>) { |
| if (!std::isfinite(v.value)) { |
| AddError(source) << OverflowErrorMessage(v, t->FriendlyName()); |
| if (use_runtime_semantics_) { |
| return mgr.Zero(t); |
| } else { |
| return error; |
| } |
| } |
| } |
| return mgr.Get<Scalar<T>>(t, v); |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Add(const Source& source, NumberT a, NumberT b) { |
| NumberT result; |
| if constexpr (IsAbstract<NumberT> || IsFloatingPoint<NumberT>) { |
| if (auto r = CheckedAdd(a, b)) { |
| result = r->value; |
| } else { |
| AddError(source) << OverflowErrorMessage(a, "+", b); |
| if (use_runtime_semantics_) { |
| return NumberT{0}; |
| } else { |
| return error; |
| } |
| } |
| } else { |
| using T = UnwrapNumber<NumberT>; |
| auto add_values = [](T lhs, T rhs) { |
| if constexpr (std::is_integral_v<T> && std::is_signed_v<T>) { |
| // Ensure no UB for signed overflow |
| using UT = std::make_unsigned_t<T>; |
| return static_cast<T>(static_cast<UT>(lhs) + static_cast<UT>(rhs)); |
| } else { |
| return lhs + rhs; |
| } |
| }; |
| result = add_values(a.value, b.value); |
| } |
| return result; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Sub(const Source& source, NumberT a, NumberT b) { |
| NumberT result; |
| if constexpr (IsAbstract<NumberT> || IsFloatingPoint<NumberT>) { |
| if (auto r = CheckedSub(a, b)) { |
| result = r->value; |
| } else { |
| AddError(source) << OverflowErrorMessage(a, "-", b); |
| if (use_runtime_semantics_) { |
| return NumberT{0}; |
| } else { |
| return error; |
| } |
| } |
| } else { |
| using T = UnwrapNumber<NumberT>; |
| auto sub_values = [](T lhs, T rhs) { |
| if constexpr (std::is_integral_v<T> && std::is_signed_v<T>) { |
| // Ensure no UB for signed overflow |
| using UT = std::make_unsigned_t<T>; |
| return static_cast<T>(static_cast<UT>(lhs) - static_cast<UT>(rhs)); |
| } else { |
| return lhs - rhs; |
| } |
| }; |
| result = sub_values(a.value, b.value); |
| } |
| return result; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Mul(const Source& source, NumberT a, NumberT b) { |
| using T = UnwrapNumber<NumberT>; |
| NumberT result; |
| if constexpr (IsAbstract<NumberT> || IsFloatingPoint<NumberT>) { |
| if (auto r = CheckedMul(a, b)) { |
| result = r->value; |
| } else { |
| AddError(source) << OverflowErrorMessage(a, "*", b); |
| if (use_runtime_semantics_) { |
| return NumberT{0}; |
| } else { |
| return error; |
| } |
| } |
| } else { |
| auto mul_values = [](T lhs, T rhs) { |
| if constexpr (std::is_integral_v<T> && std::is_signed_v<T>) { |
| // For signed integrals, avoid C++ UB by multiplying as unsigned |
| using UT = std::make_unsigned_t<T>; |
| return static_cast<T>(static_cast<UT>(lhs) * static_cast<UT>(rhs)); |
| } else { |
| return lhs * rhs; |
| } |
| }; |
| result = mul_values(a.value, b.value); |
| } |
| return result; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Div(const Source& source, NumberT a, NumberT b) { |
| NumberT result; |
| if constexpr (IsAbstract<NumberT> || IsFloatingPoint<NumberT>) { |
| if (auto r = CheckedDiv(a, b)) { |
| result = r->value; |
| } else { |
| AddError(source) << OverflowErrorMessage(a, "/", b); |
| if (use_runtime_semantics_) { |
| return a; |
| } else { |
| return error; |
| } |
| } |
| } else { |
| using T = UnwrapNumber<NumberT>; |
| auto lhs = a.value; |
| auto rhs = b.value; |
| if (rhs == 0) { |
| // For integers (as for floats), lhs / 0 is an error |
| AddError(source) << OverflowErrorMessage(a, "/", b); |
| if (use_runtime_semantics_) { |
| return a; |
| } else { |
| return error; |
| } |
| } |
| if constexpr (std::is_signed_v<T>) { |
| // For signed integers, lhs / -1 where lhs is the |
| // most negative value is an error |
| if (rhs == -1 && lhs == std::numeric_limits<T>::min()) { |
| AddError(source) << OverflowErrorMessage(a, "/", b); |
| if (use_runtime_semantics_) { |
| return a; |
| } else { |
| return error; |
| } |
| } |
| } |
| result = lhs / rhs; |
| } |
| return result; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Mod(const Source& source, NumberT a, NumberT b) { |
| NumberT result; |
| if constexpr (IsAbstract<NumberT> || IsFloatingPoint<NumberT>) { |
| if (auto r = CheckedMod(a, b)) { |
| result = r->value; |
| } else { |
| AddError(source) << OverflowErrorMessage(a, "%", b); |
| if (use_runtime_semantics_) { |
| return NumberT{0}; |
| } else { |
| return error; |
| } |
| } |
| } else { |
| using T = UnwrapNumber<NumberT>; |
| auto lhs = a.value; |
| auto rhs = b.value; |
| if (rhs == 0) { |
| // lhs % 0 is an error |
| AddError(source) << OverflowErrorMessage(a, "%", b); |
| if (use_runtime_semantics_) { |
| return NumberT{0}; |
| } else { |
| return error; |
| } |
| } |
| if constexpr (std::is_signed_v<T>) { |
| // For signed integers, lhs % -1 where lhs is the |
| // most negative value is an error |
| if (rhs == -1 && lhs == std::numeric_limits<T>::min()) { |
| AddError(source) << OverflowErrorMessage(a, "%", b); |
| if (use_runtime_semantics_) { |
| return NumberT{0}; |
| } else { |
| return error; |
| } |
| } |
| } |
| result = lhs % rhs; |
| } |
| return result; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Dot2(const Source& source, |
| NumberT a1, |
| NumberT a2, |
| NumberT b1, |
| NumberT b2) { |
| auto r1 = Mul(source, a1, b1); |
| if (r1 != Success) { |
| return error; |
| } |
| auto r2 = Mul(source, a2, b2); |
| if (r2 != Success) { |
| return error; |
| } |
| auto r = Add(source, r1.Get(), r2.Get()); |
| if (r != Success) { |
| return error; |
| } |
| return r; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Dot3(const Source& source, |
| NumberT a1, |
| NumberT a2, |
| NumberT a3, |
| NumberT b1, |
| NumberT b2, |
| NumberT b3) { |
| auto r1 = Mul(source, a1, b1); |
| if (r1 != Success) { |
| return error; |
| } |
| auto r2 = Mul(source, a2, b2); |
| if (r2 != Success) { |
| return error; |
| } |
| auto r3 = Mul(source, a3, b3); |
| if (r3 != Success) { |
| return error; |
| } |
| auto r = Add(source, r1.Get(), r2.Get()); |
| if (r != Success) { |
| return error; |
| } |
| r = Add(source, r.Get(), r3.Get()); |
| if (r != Success) { |
| return error; |
| } |
| return r; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Dot4(const Source& source, |
| NumberT a1, |
| NumberT a2, |
| NumberT a3, |
| NumberT a4, |
| NumberT b1, |
| NumberT b2, |
| NumberT b3, |
| NumberT b4) { |
| auto r1 = Mul(source, a1, b1); |
| if (r1 != Success) { |
| return error; |
| } |
| auto r2 = Mul(source, a2, b2); |
| if (r2 != Success) { |
| return error; |
| } |
| auto r3 = Mul(source, a3, b3); |
| if (r3 != Success) { |
| return error; |
| } |
| auto r4 = Mul(source, a4, b4); |
| if (r4 != Success) { |
| return error; |
| } |
| auto r = Add(source, r1.Get(), r2.Get()); |
| if (r != Success) { |
| return error; |
| } |
| r = Add(source, r.Get(), r3.Get()); |
| if (r != Success) { |
| return error; |
| } |
| r = Add(source, r.Get(), r4.Get()); |
| if (r != Success) { |
| return error; |
| } |
| return r; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Det2(const Source& source, |
| NumberT a, |
| NumberT b, |
| NumberT c, |
| NumberT d) { |
| // | a c | |
| // | b d | |
| // |
| // = |
| // |
| // a * d - c * b |
| |
| auto r1 = Mul(source, a, d); |
| if (r1 != Success) { |
| return error; |
| } |
| auto r2 = Mul(source, c, b); |
| if (r2 != Success) { |
| return error; |
| } |
| auto r = Sub(source, r1.Get(), r2.Get()); |
| if (r != Success) { |
| return error; |
| } |
| return r; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Det3(const Source& source, |
| NumberT a, |
| NumberT b, |
| NumberT c, |
| NumberT d, |
| NumberT e, |
| NumberT f, |
| NumberT g, |
| NumberT h, |
| NumberT i) { |
| // | a d g | |
| // | b e h | |
| // | c f i | |
| // |
| // = |
| // |
| // a | e h | - d | b h | + g | b e | |
| // | f i | | c i | | c f | |
| |
| auto det1 = Det2(source, e, f, h, i); |
| if (det1 != Success) { |
| return error; |
| } |
| auto a_det1 = Mul(source, a, det1.Get()); |
| if (a_det1 != Success) { |
| return error; |
| } |
| auto det2 = Det2(source, b, c, h, i); |
| if (det2 != Success) { |
| return error; |
| } |
| auto d_det2 = Mul(source, d, det2.Get()); |
| if (d_det2 != Success) { |
| return error; |
| } |
| auto det3 = Det2(source, b, c, e, f); |
| if (det3 != Success) { |
| return error; |
| } |
| auto g_det3 = Mul(source, g, det3.Get()); |
| if (g_det3 != Success) { |
| return error; |
| } |
| auto r = Sub(source, a_det1.Get(), d_det2.Get()); |
| if (r != Success) { |
| return error; |
| } |
| return Add(source, r.Get(), g_det3.Get()); |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Det4(const Source& source, |
| NumberT a, |
| NumberT b, |
| NumberT c, |
| NumberT d, |
| NumberT e, |
| NumberT f, |
| NumberT g, |
| NumberT h, |
| NumberT i, |
| NumberT j, |
| NumberT k, |
| NumberT l, |
| NumberT m, |
| NumberT n, |
| NumberT o, |
| NumberT p) { |
| // | a e i m | |
| // | b f j n | |
| // | c g k o | |
| // | d h l p | |
| // |
| // = |
| // |
| // a | f j n | - e | b j n | + i | b f n | - m | b f j | |
| // | g k o | | c k o | | c g o | | c g k | |
| // | h l p | | d l p | | d h p | | d h l | |
| |
| auto det1 = Det3(source, f, g, h, j, k, l, n, o, p); |
| if (det1 != Success) { |
| return error; |
| } |
| auto a_det1 = Mul(source, a, det1.Get()); |
| if (a_det1 != Success) { |
| return error; |
| } |
| auto det2 = Det3(source, b, c, d, j, k, l, n, o, p); |
| if (det2 != Success) { |
| return error; |
| } |
| auto e_det2 = Mul(source, e, det2.Get()); |
| if (e_det2 != Success) { |
| return error; |
| } |
| auto det3 = Det3(source, b, c, d, f, g, h, n, o, p); |
| if (det3 != Success) { |
| return error; |
| } |
| auto i_det3 = Mul(source, i, det3.Get()); |
| if (i_det3 != Success) { |
| return error; |
| } |
| auto det4 = Det3(source, b, c, d, f, g, h, j, k, l); |
| if (det4 != Success) { |
| return error; |
| } |
| auto m_det4 = Mul(source, m, det4.Get()); |
| if (m_det4 != Success) { |
| return error; |
| } |
| auto r = Sub(source, a_det1.Get(), e_det2.Get()); |
| if (r != Success) { |
| return error; |
| } |
| r = Add(source, r.Get(), i_det3.Get()); |
| if (r != Success) { |
| return error; |
| } |
| return Sub(source, r.Get(), m_det4.Get()); |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Sqrt(const Source& source, NumberT v) { |
| if (v < NumberT(0)) { |
| AddError(source) << "sqrt must be called with a value >= 0"; |
| if (use_runtime_semantics_) { |
| return NumberT{0}; |
| } else { |
| return error; |
| } |
| } |
| return NumberT{std::sqrt(v)}; |
| } |
| |
| auto Eval::SqrtFunc(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto v) -> Eval::Result { |
| if (auto r = Sqrt(source, v); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| template <typename NumberT> |
| tint::Result<NumberT, Eval::Error> Eval::Clamp(const Source& source, |
| NumberT e, |
| NumberT low, |
| NumberT high) { |
| if (low > high) { |
| AddError(source) << "clamp called with 'low' (" << low << ") greater than 'high' (" << high |
| << ")"; |
| if (!use_runtime_semantics_) { |
| return error; |
| } |
| } |
| return NumberT{std::min(std::max(e, low), high)}; |
| } |
| |
| auto Eval::ClampFunc(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto e, auto low, auto high) -> Eval::Result { |
| if (auto r = Clamp(source, e, low, high); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::AddFunc(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a1, auto a2) -> Eval::Result { |
| if (auto r = Add(source, a1, a2); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::SubFunc(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a1, auto a2) -> Eval::Result { |
| if (auto r = Sub(source, a1, a2); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::MulFunc(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a1, auto a2) -> Eval::Result { |
| if (auto r = Mul(source, a1, a2); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::DivFunc(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a1, auto a2) -> Eval::Result { |
| if (auto r = Div(source, a1, a2); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::ModFunc(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a1, auto a2) -> Eval::Result { |
| if (auto r = Mod(source, a1, a2); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::Dot2Func(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a1, auto a2, auto b1, auto b2) -> Eval::Result { |
| if (auto r = Dot2(source, a1, a2, b1, b2); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::Dot3Func(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a1, auto a2, auto a3, auto b1, auto b2, auto b3) -> Eval::Result { |
| if (auto r = Dot3(source, a1, a2, a3, b1, b2, b3); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::Dot4Func(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a1, auto a2, auto a3, auto a4, auto b1, auto b2, auto b3, |
| auto b4) -> Eval::Result { |
| if (auto r = Dot4(source, a1, a2, a3, a4, b1, b2, b3, b4); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| Eval::Result Eval::Dot(const Source& source, const Value* v1, const Value* v2) { |
| auto* vec_ty = v1->Type()->As<core::type::Vector>(); |
| TINT_ASSERT(vec_ty); |
| auto* elem_ty = vec_ty->type(); |
| switch (vec_ty->Width()) { |
| case 2: |
| return Dispatch_fia_fiu32_f16( // |
| Dot2Func(source, elem_ty), // |
| v1->Index(0), v1->Index(1), // |
| v2->Index(0), v2->Index(1)); |
| case 3: |
| return Dispatch_fia_fiu32_f16( // |
| Dot3Func(source, elem_ty), // |
| v1->Index(0), v1->Index(1), v1->Index(2), // |
| v2->Index(0), v2->Index(1), v2->Index(2)); |
| case 4: |
| return Dispatch_fia_fiu32_f16( // |
| Dot4Func(source, elem_ty), // |
| v1->Index(0), v1->Index(1), v1->Index(2), v1->Index(3), // |
| v2->Index(0), v2->Index(1), v2->Index(2), v2->Index(3)); |
| } |
| TINT_ICE() << "Expected vector"; |
| } |
| |
| Eval::Result Eval::Length(const Source& source, const core::type::Type* ty, const Value* c0) { |
| auto* vec_ty = c0->Type()->As<core::type::Vector>(); |
| // Evaluates to the absolute value of e if T is scalar. |
| if (vec_ty == nullptr) { |
| auto create = [&](auto e) { |
| using NumberT = decltype(e); |
| return CreateScalar(source, ty, NumberT{std::abs(e)}); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| } |
| |
| // Evaluates to sqrt(e[0]^2 + e[1]^2 + ...) if T is a vector type. |
| auto d = Dot(source, c0, c0); |
| if (d != Success) { |
| return error; |
| } |
| return Dispatch_fa_f32_f16(SqrtFunc(source, ty), d.Get()); |
| } |
| |
| Eval::Result Eval::Mul(const Source& source, |
| const core::type::Type* ty, |
| const Value* v1, |
| const Value* v2) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| return Dispatch_fia_fiu32_f16(MulFunc(source, c0->Type()), c0, c1); |
| }; |
| return TransformBinaryDifferingArityElements(mgr, ty, transform, v1, v2); |
| } |
| |
| Eval::Result Eval::Sub(const Source& source, |
| const core::type::Type* ty, |
| const Value* v1, |
| const Value* v2) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| return Dispatch_fia_fiu32_f16(SubFunc(source, c0->Type()), c0, c1); |
| }; |
| return TransformBinaryDifferingArityElements(mgr, ty, transform, v1, v2); |
| } |
| |
| auto Eval::Det2Func(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a, auto b, auto c, auto d) -> Eval::Result { |
| if (auto r = Det2(source, a, b, c, d); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::Det3Func(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a, auto b, auto c, auto d, auto e, auto f, auto g, auto h, |
| auto i) -> Eval::Result { |
| if (auto r = Det3(source, a, b, c, d, e, f, g, h, i); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| auto Eval::Det4Func(const Source& source, const core::type::Type* elem_ty) { |
| return [=](auto a, auto b, auto c, auto d, auto e, auto f, auto g, auto h, auto i, auto j, |
| auto k, auto l, auto m, auto n, auto o, auto p) -> Eval::Result { |
| if (auto r = Det4(source, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p); r == Success) { |
| return CreateScalar(source, elem_ty, r.Get()); |
| } |
| return error; |
| }; |
| } |
| |
| Eval::Result Eval::ArrayOrStructCtor(const core::type::Type* ty, VectorRef<const Value*> args) { |
| if (args.IsEmpty()) { |
| return mgr.Zero(ty); |
| } |
| |
| if (args.Length() == 1 && args[0]->Type() == ty) { |
| // Identity constructor. |
| return args[0]; |
| } |
| |
| // Multiple arguments. Must be a value constructor. |
| return mgr.Composite(ty, std::move(args)); |
| } |
| |
| Eval::Result Eval::Conv(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto* el_ty = ty->Elements(ty).type; |
| if (!el_ty) { |
| return nullptr; |
| } |
| |
| if (!args[0]) { |
| return nullptr; // Single argument is not constant. |
| } |
| |
| return Convert(ty, args[0], source); |
| } |
| |
| Eval::Result Eval::Zero(const core::type::Type* ty, VectorRef<const Value*>, const Source&) { |
| return mgr.Zero(ty); |
| } |
| |
| Eval::Result Eval::Identity(const core::type::Type*, VectorRef<const Value*> args, const Source&) { |
| return args[0]; |
| } |
| |
| Eval::Result Eval::VecSplat(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source&) { |
| if (auto* arg = args[0]) { |
| return mgr.Splat(ty, arg); |
| } |
| return nullptr; |
| } |
| |
| Eval::Result Eval::VecInitS(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source&) { |
| return mgr.Composite(ty, args); |
| } |
| |
| Eval::Result Eval::VecInitM(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source&) { |
| Vector<const Value*, 4> els; |
| for (auto* arg : args) { |
| auto* val = arg; |
| if (!val) { |
| return nullptr; |
| } |
| auto* arg_ty = arg->Type(); |
| if (auto* arg_vec = arg_ty->As<core::type::Vector>()) { |
| // Extract out vector elements. |
| for (uint32_t j = 0; j < arg_vec->Width(); j++) { |
| auto* el = val->Index(j); |
| if (!el) { |
| return nullptr; |
| } |
| els.Push(el); |
| } |
| } else { |
| els.Push(val); |
| } |
| } |
| return mgr.Composite(ty, std::move(els)); |
| } |
| |
| Eval::Result Eval::MatInitS(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source&) { |
| auto* m = static_cast<const core::type::Matrix*>(ty); |
| |
| Vector<const Value*, 4> els; |
| for (uint32_t c = 0; c < m->columns(); c++) { |
| Vector<const Value*, 4> column; |
| for (uint32_t r = 0; r < m->rows(); r++) { |
| auto i = r + c * m->rows(); |
| column.Push(args[i]); |
| } |
| els.Push(mgr.Composite(m->ColumnType(), std::move(column))); |
| } |
| return mgr.Composite(ty, std::move(els)); |
| } |
| |
| Eval::Result Eval::MatInitV(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source&) { |
| return mgr.Composite(ty, args); |
| } |
| |
| Eval::Result Eval::Index(const Value* obj_val, |
| const core::type::Type* obj_ty, |
| const Value* idx_val, |
| const Source& idx_source) { |
| auto el = obj_ty->UnwrapPtrOrRef()->Elements(); |
| |
| AInt idx = idx_val->ValueAs<AInt>(); |
| if (idx < 0 || (el.count > 0 && idx >= el.count)) { |
| auto& err = AddError(idx_source) << "index " << idx << " out of bounds"; |
| if (el.count > 0) { |
| err << " [0.." + std::to_string(el.count - 1) + "]"; |
| } |
| |
| if (use_runtime_semantics_) { |
| return mgr.Zero(el.type); |
| } else { |
| return error; |
| } |
| } |
| |
| return obj_val ? obj_val->Index(static_cast<size_t>(idx)) : nullptr; |
| } |
| |
| Eval::Result Eval::Swizzle(const core::type::Type* ty, |
| const Value* object, |
| VectorRef<uint32_t> indices) { |
| if (indices.Length() == 1) { |
| return object->Index(static_cast<size_t>(indices[0])); |
| } |
| auto values = tint::Transform<4>( |
| indices, [&](uint32_t i) { return object->Index(static_cast<size_t>(i)); }); |
| return mgr.Composite(ty, std::move(values)); |
| } |
| |
| Eval::Result Eval::bitcast(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto* value = args[0]; |
| bool is_abstract = value->Type()->is_abstract_integer_scalar_or_vector(); |
| |
| // Target type |
| auto dst_elements = ty->Elements(ty->DeepestElement(), 1u); |
| auto dst_el_ty = dst_elements.type; |
| auto dst_count = dst_elements.count; |
| // Source type |
| auto src_elements = value->Type()->Elements(value->Type()->DeepestElement(), 1u); |
| auto src_el_ty = src_elements.type; |
| auto src_count = src_elements.count; |
| |
| TINT_ASSERT(is_abstract || (dst_count * dst_el_ty->Size() == src_count * src_el_ty->Size())); |
| uint32_t total_bitwidth = dst_count * dst_el_ty->Size(); |
| // Buffer holding the bits from source value, result value reinterpreted from it. |
| Vector<std::byte, 16> buffer; |
| buffer.Reserve(total_bitwidth); |
| |
| // Pushes bits from source value into the buffer. |
| auto push_src_element_bits = [&](const Value* element) { |
| auto push_32_bits = [&](uint32_t v) { |
| buffer.Push(std::byte(v & 0xffu)); |
| buffer.Push(std::byte((v >> 8) & 0xffu)); |
| buffer.Push(std::byte((v >> 16) & 0xffu)); |
| buffer.Push(std::byte((v >> 24) & 0xffu)); |
| return Success; |
| }; |
| auto push_16_bits = [&](uint16_t v) { |
| buffer.Push(std::byte(v & 0xffu)); |
| buffer.Push(std::byte((v >> 8) & 0xffu)); |
| return Success; |
| }; |
| return Switch( |
| src_el_ty, |
| [&](const core::type::AbstractInt*) -> tint::Result<SuccessType, Error> { |
| if (element->ValueAs<AInt>() < 0) { |
| auto res = Conv(mgr.types.i32(), Vector{element}, source); |
| if (res != Success) { |
| return res.Failure(); |
| } |
| return push_32_bits(tint::Bitcast<u32>(res.Get()->ValueAs<i32>())); |
| } else { |
| auto res = Conv(mgr.types.u32(), Vector{element}, source); |
| if (res != Success) { |
| return res.Failure(); |
| } |
| return push_32_bits(res.Get()->ValueAs<u32>()); |
| } |
| }, |
| [&](const core::type::U32*) -> tint::Result<SuccessType, Error> { |
| return push_32_bits(element->ValueAs<u32>()); |
| }, |
| [&](const core::type::I32*) -> tint::Result<SuccessType, Error> { |
| return push_32_bits(tint::Bitcast<u32>(element->ValueAs<i32>())); |
| }, |
| [&](const core::type::F32*) -> tint::Result<SuccessType, Error> { |
| return push_32_bits(tint::Bitcast<u32>(element->ValueAs<f32>())); |
| }, |
| [&](const core::type::F16*) -> tint::Result<SuccessType, Error> { |
| return push_16_bits(element->ValueAs<f16>().BitsRepresentation()); |
| }, |
| TINT_ICE_ON_NO_MATCH); |
| }; |
| if (src_count == 1) { |
| if (auto res = push_src_element_bits(value); res != Success) { |
| return res.Failure(); |
| } |
| } else { |
| for (size_t i = 0; i < src_count; i++) { |
| if (auto res = push_src_element_bits(value->Index(i)); res != Success) { |
| return res.Failure(); |
| } |
| } |
| } |
| |
| // Vector holding elements of return value |
| Vector<const Value*, 4> els; |
| |
| // Reinterprets the buffer bits as destination element and push the result into the vector. |
| // Return false if an error occurred, otherwise return true. |
| auto push_dst_element = [&](size_t offset) -> bool { |
| uint32_t v; |
| if (dst_el_ty->Size() == 4) { |
| v = (std::to_integer<uint32_t>(buffer[offset])) | |
| (std::to_integer<uint32_t>(buffer[offset + 1]) << 8) | |
| (std::to_integer<uint32_t>(buffer[offset + 2]) << 16) | |
| (std::to_integer<uint32_t>(buffer[offset + 3]) << 24); |
| } else { |
| v = (std::to_integer<uint32_t>(buffer[offset])) | |
| (std::to_integer<uint32_t>(buffer[offset + 1]) << 8); |
| } |
| |
| return Switch( |
| dst_el_ty, |
| [&](const core::type::U32*) { // |
| auto r = CreateScalar(source, dst_el_ty, u32(v)); |
| if (r != Success) { |
| return false; |
| } |
| els.Push(r.Get()); |
| return true; |
| }, |
| [&](const core::type::I32*) { // |
| auto r = CreateScalar(source, dst_el_ty, tint::Bitcast<i32>(v)); |
| if (r != Success) { |
| return false; |
| } |
| els.Push(r.Get()); |
| return true; |
| }, |
| [&](const core::type::F32*) { // |
| auto r = CreateScalar(source, dst_el_ty, tint::Bitcast<f32>(v)); |
| if (r != Success) { |
| return false; |
| } |
| els.Push(r.Get()); |
| return true; |
| }, |
| [&](const core::type::F16*) { // |
| auto r = CreateScalar(source, dst_el_ty, f16::FromBits(static_cast<uint16_t>(v))); |
| if (r != Success) { |
| return false; |
| } |
| els.Push(r.Get()); |
| return true; |
| }, |
| TINT_ICE_ON_NO_MATCH); |
| }; |
| |
| TINT_ASSERT((buffer.Length() == total_bitwidth)); |
| for (size_t i = 0; i < dst_count; i++) { |
| if (!push_dst_element(i * dst_el_ty->Size())) { |
| return error; |
| } |
| } |
| |
| if (dst_count == 1) { |
| return std::move(els[0]); |
| } |
| return mgr.Composite(ty, std::move(els)); |
| } |
| |
| Eval::Result Eval::Complement(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c) { |
| auto create = [&](auto i) { |
| return CreateScalar(source, c->Type(), decltype(i)(~i.value)); |
| }; |
| return Dispatch_ia_iu32(create, c); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::UnaryMinus(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c) { |
| auto create = [&](auto i) { |
| // For signed integrals, avoid C++ UB by not negating the |
| // smallest negative number. In WGSL, this operation is well |
| // defined to return the same value, see: |
| // https://gpuweb.github.io/gpuweb/wgsl/#arithmetic-expr. |
| using T = UnwrapNumber<decltype(i)>; |
| if constexpr (std::is_integral_v<T>) { |
| auto v = i.value; |
| if (v != std::numeric_limits<T>::min()) { |
| v = -v; |
| } |
| return CreateScalar(source, c->Type(), decltype(i)(v)); |
| } else { |
| return CreateScalar(source, c->Type(), decltype(i)(-i.value)); |
| } |
| }; |
| return Dispatch_fia_fi32_f16(create, c); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::Not(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c) { |
| auto create = [&](auto i) { return CreateScalar(source, c->Type(), decltype(i)(!i)); }; |
| return Dispatch_bool(create, c); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::Plus(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| return Dispatch_fia_fiu32_f16(AddFunc(source, c0->Type()), c0, c1); |
| }; |
| |
| return TransformBinaryDifferingArityElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::Minus(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| return Sub(source, ty, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::Multiply(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| return Mul(source, ty, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::MultiplyMatVec(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto* mat_ty = args[0]->Type()->As<core::type::Matrix>(); |
| auto* vec_ty = args[1]->Type()->As<core::type::Vector>(); |
| auto* elem_ty = vec_ty->type(); |
| |
| auto dot = [&](const Value* m, size_t row, const Value* v) { |
| Eval::Result result; |
| switch (mat_ty->columns()) { |
| case 2: |
| result = Dispatch_fa_f32_f16(Dot2Func(source, elem_ty), // |
| m->Index(0)->Index(row), // |
| m->Index(1)->Index(row), // |
| v->Index(0), // |
| v->Index(1)); |
| break; |
| case 3: |
| result = Dispatch_fa_f32_f16(Dot3Func(source, elem_ty), // |
| m->Index(0)->Index(row), // |
| m->Index(1)->Index(row), // |
| m->Index(2)->Index(row), // |
| v->Index(0), // |
| v->Index(1), v->Index(2)); |
| break; |
| case 4: |
| result = Dispatch_fa_f32_f16(Dot4Func(source, elem_ty), // |
| m->Index(0)->Index(row), // |
| m->Index(1)->Index(row), // |
| m->Index(2)->Index(row), // |
| m->Index(3)->Index(row), // |
| v->Index(0), // |
| v->Index(1), // |
| v->Index(2), // |
| v->Index(3)); |
| break; |
| } |
| return result; |
| }; |
| |
| Vector<const Value*, 4> result; |
| for (size_t i = 0; i < mat_ty->rows(); ++i) { |
| auto r = dot(args[0], i, args[1]); // matrix row i * vector |
| if (r != Success) { |
| return error; |
| } |
| result.Push(r.Get()); |
| } |
| return mgr.Composite(ty, result); |
| } |
| Eval::Result Eval::MultiplyVecMat(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto* vec_ty = args[0]->Type()->As<core::type::Vector>(); |
| auto* mat_ty = args[1]->Type()->As<core::type::Matrix>(); |
| auto* elem_ty = vec_ty->type(); |
| |
| auto dot = [&](const Value* v, const Value* m, size_t col) { |
| Eval::Result result; |
| switch (mat_ty->rows()) { |
| case 2: |
| result = Dispatch_fa_f32_f16(Dot2Func(source, elem_ty), // |
| m->Index(col)->Index(0), // |
| m->Index(col)->Index(1), // |
| v->Index(0), // |
| v->Index(1)); |
| break; |
| case 3: |
| result = Dispatch_fa_f32_f16(Dot3Func(source, elem_ty), // |
| m->Index(col)->Index(0), // |
| m->Index(col)->Index(1), // |
| m->Index(col)->Index(2), |
| v->Index(0), // |
| v->Index(1), // |
| v->Index(2)); |
| break; |
| case 4: |
| result = Dispatch_fa_f32_f16(Dot4Func(source, elem_ty), // |
| m->Index(col)->Index(0), // |
| m->Index(col)->Index(1), // |
| m->Index(col)->Index(2), // |
| m->Index(col)->Index(3), // |
| v->Index(0), // |
| v->Index(1), // |
| v->Index(2), // |
| v->Index(3)); |
| } |
| return result; |
| }; |
| |
| Vector<const Value*, 4> result; |
| for (size_t i = 0; i < mat_ty->columns(); ++i) { |
| auto r = dot(args[0], args[1], i); // vector * matrix col i |
| if (r != Success) { |
| return error; |
| } |
| result.Push(r.Get()); |
| } |
| return mgr.Composite(ty, result); |
| } |
| |
| Eval::Result Eval::MultiplyMatMat(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto* mat1 = args[0]; |
| auto* mat2 = args[1]; |
| auto* mat1_ty = mat1->Type()->As<core::type::Matrix>(); |
| auto* mat2_ty = mat2->Type()->As<core::type::Matrix>(); |
| auto* elem_ty = mat1_ty->type(); |
| |
| auto dot = [&](const Value* m1, size_t row, const Value* m2, size_t col) { |
| auto m1e = [&](size_t r, size_t c) { return m1->Index(c)->Index(r); }; |
| auto m2e = [&](size_t r, size_t c) { return m2->Index(c)->Index(r); }; |
| |
| Eval::Result result; |
| switch (mat1_ty->columns()) { |
| case 2: |
| result = Dispatch_fa_f32_f16(Dot2Func(source, elem_ty), // |
| m1e(row, 0), // |
| m1e(row, 1), // |
| m2e(0, col), // |
| m2e(1, col)); |
| break; |
| case 3: |
| result = Dispatch_fa_f32_f16(Dot3Func(source, elem_ty), // |
| m1e(row, 0), // |
| m1e(row, 1), // |
| m1e(row, 2), // |
| m2e(0, col), // |
| m2e(1, col), // |
| m2e(2, col)); |
| break; |
| case 4: |
| result = Dispatch_fa_f32_f16(Dot4Func(source, elem_ty), // |
| m1e(row, 0), // |
| m1e(row, 1), // |
| m1e(row, 2), // |
| m1e(row, 3), // |
| m2e(0, col), // |
| m2e(1, col), // |
| m2e(2, col), // |
| m2e(3, col)); |
| break; |
| } |
| return result; |
| }; |
| |
| Vector<const Value*, 4> result_mat; |
| for (size_t c = 0; c < mat2_ty->columns(); ++c) { |
| Vector<const Value*, 4> col_vec; |
| for (size_t r = 0; r < mat1_ty->rows(); ++r) { |
| auto v = dot(mat1, r, mat2, c); // mat1 row r * mat2 col c |
| if (v != Success) { |
| return error; |
| } |
| col_vec.Push(v.Get()); // mat1 row r * mat2 col c |
| } |
| |
| // Add column vector to matrix |
| auto* col_vec_ty = ty->As<core::type::Matrix>()->ColumnType(); |
| result_mat.Push(mgr.Composite(col_vec_ty, col_vec)); |
| } |
| return mgr.Composite(ty, result_mat); |
| } |
| |
| Eval::Result Eval::Divide(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| return Dispatch_fia_fiu32_f16(DivFunc(source, c0->Type()), c0, c1); |
| }; |
| |
| return TransformBinaryDifferingArityElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::Modulo(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| return Dispatch_fia_fiu32_f16(ModFunc(source, c0->Type()), c0, c1); |
| }; |
| |
| return TransformBinaryDifferingArityElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::Equal(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) -> Eval::Result { |
| return CreateScalar(source, ty->DeepestElement(), i == j); |
| }; |
| return Dispatch_fia_fiu32_f16_bool(create, c0, c1); |
| }; |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::NotEqual(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) -> Eval::Result { |
| return CreateScalar(source, ty->DeepestElement(), i != j); |
| }; |
| return Dispatch_fia_fiu32_f16_bool(create, c0, c1); |
| }; |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::LessThan(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) -> Eval::Result { |
| return CreateScalar(source, ty->DeepestElement(), i < j); |
| }; |
| return Dispatch_fia_fiu32_f16(create, c0, c1); |
| }; |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::GreaterThan(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) -> Eval::Result { |
| return CreateScalar(source, ty->DeepestElement(), i > j); |
| }; |
| return Dispatch_fia_fiu32_f16(create, c0, c1); |
| }; |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::LessThanEqual(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) -> Eval::Result { |
| return CreateScalar(source, ty->DeepestElement(), i <= j); |
| }; |
| return Dispatch_fia_fiu32_f16(create, c0, c1); |
| }; |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::GreaterThanEqual(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) -> Eval::Result { |
| return CreateScalar(source, ty->DeepestElement(), i >= j); |
| }; |
| return Dispatch_fia_fiu32_f16(create, c0, c1); |
| }; |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::LogicalAnd(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| // Due to short-circuiting, this function is only called if lhs is true, so we only return the |
| // value of the rhs. |
| TINT_ASSERT(args[0]->ValueAs<bool>()); |
| return CreateScalar(source, ty, args[1]->ValueAs<bool>()); |
| } |
| |
| Eval::Result Eval::LogicalOr(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| // Due to short-circuiting, this function is only called if lhs is false, so we only only return |
| // the value of the rhs. |
| TINT_ASSERT(!args[0]->ValueAs<bool>()); |
| return CreateScalar(source, ty, args[1]->ValueAs<bool>()); |
| } |
| |
| Eval::Result Eval::And(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) -> Eval::Result { |
| using T = decltype(i); |
| T result; |
| if constexpr (std::is_same_v<T, bool>) { |
| result = i && j; |
| } else { // integral |
| result = i & j; |
| } |
| return CreateScalar(source, ty->DeepestElement(), result); |
| }; |
| return Dispatch_ia_iu32_bool(create, c0, c1); |
| }; |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::Or(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) -> Eval::Result { |
| using T = decltype(i); |
| T result; |
| if constexpr (std::is_same_v<T, bool>) { |
| result = i || j; |
| } else { // integral |
| result = i | j; |
| } |
| return CreateScalar(source, ty->DeepestElement(), result); |
| }; |
| return Dispatch_ia_iu32_bool(create, c0, c1); |
| }; |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::Xor(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) -> Eval::Result { |
| return CreateScalar(source, ty->DeepestElement(), decltype(i){i ^ j}); |
| }; |
| return Dispatch_ia_iu32(create, c0, c1); |
| }; |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::ShiftLeft(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto e1, auto e2) -> Eval::Result { |
| using NumberT = decltype(e1); |
| using T = UnwrapNumber<NumberT>; |
| using UT = std::make_unsigned_t<T>; |
| constexpr size_t bit_width = BitWidth<NumberT>; |
| UT e1u = static_cast<UT>(e1); |
| UT e2u = static_cast<UT>(e2); |
| |
| if constexpr (IsAbstract<NumberT>) { |
| // The e2 + 1 most significant bits of e1 must have the same bit value, otherwise |
| // sign change (overflow) would occur. |
| // Check sign change only if e2 is less than bit width of e1. If e1 is larger |
| // than bit width, we check for non-representable value below. |
| if (e2u < bit_width) { |
| UT must_match_msb = e2u + 1; |
| UT mask = ~UT{0} << (bit_width - must_match_msb); |
| if ((e1u & mask) != 0 && (e1u & mask) != mask) { |
| AddError(source) << "shift left operation results in sign change"; |
| if (!use_runtime_semantics_) { |
| return error; |
| } |
| } |
| } else { |
| // If shift value >= bit_width, then any non-zero value would overflow |
| if (e1 != 0) { |
| AddError(source) << OverflowErrorMessage(e1, "<<", e2); |
| if (!use_runtime_semantics_) { |
| return error; |
| } |
| } |
| |
| // It's UB in C++ to shift by greater or equal to the bit width (even if the lhs |
| // is 0), so we make sure to avoid this by setting the shift value to 0. |
| e2u = 0; |
| } |
| } else { |
| if (static_cast<size_t>(e2) >= bit_width) { |
| // At shader/pipeline-creation time, it is an error to shift by the bit width of |
| // the lhs or greater. |
| // NOTE: At runtime, we shift by e2 % (bit width of e1). |
| AddError(source) |
| << "shift left value must be less than the bit width of the lhs, which is " |
| << bit_width; |
| if (use_runtime_semantics_) { |
| e2u = e2u % bit_width; |
| } else { |
| return error; |
| } |
| } |
| |
| if constexpr (std::is_signed_v<T>) { |
| // If T is a signed integer type, and the e2+1 most significant bits of e1 do |
| // not have the same bit value, then error. |
| size_t must_match_msb = e2u + 1; |
| UT mask = ~UT{0} << (bit_width - must_match_msb); |
| if ((e1u & mask) != 0 && (e1u & mask) != mask) { |
| AddError(source) << "shift left operation results in sign change"; |
| if (!use_runtime_semantics_) { |
| return error; |
| } |
| } |
| } else { |
| // If T is an unsigned integer type, and any of the e2 most significant bits of |
| // e1 are 1, then error. |
| if (e2u > 0) { |
| size_t must_be_zero_msb = e2u; |
| UT mask = ~UT{0} << (bit_width - must_be_zero_msb); |
| if ((e1u & mask) != 0) { |
| AddError(source) << OverflowErrorMessage(e1, "<<", e2); |
| if (!use_runtime_semantics_) { |
| return error; |
| } |
| } |
| } |
| } |
| } |
| |
| // Avoid UB by left shifting as unsigned value |
| auto result = static_cast<T>(static_cast<UT>(e1) << e2u); |
| return CreateScalar(source, ty->DeepestElement(), NumberT{result}); |
| }; |
| return Dispatch_ia_iu32(create, c0, c1); |
| }; |
| |
| if (TINT_UNLIKELY(!args[1]->Type()->DeepestElement()->Is<core::type::U32>())) { |
| TINT_ICE() << "Element type of rhs of ShiftLeft must be a u32"; |
| } |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::ShiftRight(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto e1, auto e2) -> Eval::Result { |
| using NumberT = decltype(e1); |
| using T = UnwrapNumber<NumberT>; |
| using UT = std::make_unsigned_t<T>; |
| const size_t bit_width = BitWidth<NumberT>; |
| UT e1u = static_cast<UT>(e1); |
| UT e2u = static_cast<UT>(e2); |
| |
| [[maybe_unused]] auto signed_shift_right = [&] { |
| // In C++, right shift of a signed negative number is implementation-defined. |
| // Although most implementations sign-extend, we do it manually to ensure it works |
| // correctly on all implementations. |
| const UT msb = UT{1} << (bit_width - 1); |
| UT sign_ext = 0; |
| if (e1u & msb) { |
| // Set e2 + 1 bits to 1 |
| UT num_shift_bits_mask = ((UT{1} << e2u) - UT{1}); |
| sign_ext = (num_shift_bits_mask << (bit_width - e2u - UT{1})) | msb; |
| } |
| return static_cast<T>((e1u >> e2u) | sign_ext); |
| }; |
| |
| T result = 0; |
| if constexpr (IsAbstract<NumberT>) { |
| if (static_cast<size_t>(e2) >= bit_width) { |
| result = T{0}; |
| } else { |
| result = signed_shift_right(); |
| } |
| } else { |
| if (static_cast<size_t>(e2) >= bit_width) { |
| // At shader/pipeline-creation time, it is an error to shift by the bit width of |
| // the lhs or greater. NOTE: At runtime, we shift by e2 % (bit width of e1). |
| AddError(source) |
| << "shift right value must be less than the bit width of the lhs, which is " |
| << bit_width; |
| if (use_runtime_semantics_) { |
| e2u = e2u % bit_width; |
| } else { |
| return error; |
| } |
| } |
| |
| if constexpr (std::is_signed_v<T>) { |
| result = signed_shift_right(); |
| } else { |
| result = e1 >> e2u; |
| } |
| } |
| return CreateScalar(source, ty->DeepestElement(), NumberT{result}); |
| }; |
| return Dispatch_ia_iu32(create, c0, c1); |
| }; |
| |
| if (TINT_UNLIKELY(!args[1]->Type()->DeepestElement()->Is<core::type::U32>())) { |
| TINT_ICE() << "Element type of rhs of ShiftLeft must be a u32"; |
| } |
| |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::abs(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto e) { |
| using NumberT = decltype(e); |
| NumberT result; |
| if constexpr (IsUnsignedIntegral<NumberT>) { |
| result = e; |
| } else if constexpr (IsSignedIntegral<NumberT>) { |
| if (e == NumberT::Lowest()) { |
| result = e; |
| } else { |
| result = NumberT{std::abs(e)}; |
| } |
| } else { |
| result = NumberT{std::abs(e)}; |
| } |
| return CreateScalar(source, c0->Type(), result); |
| }; |
| return Dispatch_fia_fiu32_f16(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::acos(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = |
| [&](const Value* c0) { |
| auto create = [&](auto i) -> Eval::Result { |
| using NumberT = decltype(i); |
| if (i < NumberT(-1.0) || i > NumberT(1.0)) { |
| AddError(source) |
| << "acos must be called with a value in the range [-1 .. 1] (inclusive)"; |
| if (use_runtime_semantics_) { |
| return mgr.Zero(c0->Type()); |
| } else { |
| return error; |
| } |
| } |
| return CreateScalar(source, c0->Type(), NumberT(std::acos(i.value))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::acosh(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto i) -> Eval::Result { |
| using NumberT = decltype(i); |
| if (i < NumberT(1.0)) { |
| AddError(source) << "acosh must be called with a value >= 1.0"; |
| if (use_runtime_semantics_) { |
| return mgr.Zero(c0->Type()); |
| } else { |
| return error; |
| } |
| } |
| return CreateScalar(source, c0->Type(), NumberT(std::acosh(i.value))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| }; |
| |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::all(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| return CreateScalar(source, ty, !args[0]->AnyZero()); |
| } |
| |
| Eval::Result Eval::any(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| return CreateScalar(source, ty, !args[0]->AllZero()); |
| } |
| |
| Eval::Result Eval::asin(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = |
| [&](const Value* c0) { |
| auto create = [&](auto i) -> Eval::Result { |
| using NumberT = decltype(i); |
| if (i < NumberT(-1.0) || i > NumberT(1.0)) { |
| AddError(source) |
| << "asin must be called with a value in the range [-1 .. 1] (inclusive)"; |
| if (use_runtime_semantics_) { |
| return mgr.Zero(c0->Type()); |
| } else { |
| return error; |
| } |
| } |
| return CreateScalar(source, c0->Type(), NumberT(std::asin(i.value))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::asinh(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto i) { |
| return CreateScalar(source, c0->Type(), decltype(i)(std::asinh(i.value))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| }; |
| |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::atan(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto i) { |
| return CreateScalar(source, c0->Type(), decltype(i)(std::atan(i.value))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::atanh(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = |
| [&](const Value* c0) { |
| auto create = [&](auto i) -> Eval::Result { |
| using NumberT = decltype(i); |
| if (i <= NumberT(-1.0) || i >= NumberT(1.0)) { |
| AddError(source) |
| << "atanh must be called with a value in the range (-1 .. 1) (exclusive)"; |
| if (use_runtime_semantics_) { |
| return mgr.Zero(c0->Type()); |
| } else { |
| return error; |
| } |
| } |
| return CreateScalar(source, c0->Type(), NumberT(std::atanh(i.value))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| }; |
| |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::atan2(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1) { |
| auto create = [&](auto i, auto j) { |
| return CreateScalar(source, c0->Type(), decltype(i)(std::atan2(i.value, j.value))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0, c1); |
| }; |
| return TransformBinaryElements(mgr, ty, transform, args[0], args[1]); |
| } |
| |
| Eval::Result Eval::ceil(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto e) { |
| return CreateScalar(source, c0->Type(), decltype(e)(std::ceil(e))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::clamp(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0, const Value* c1, const Value* c2) { |
| return Dispatch_fia_fiu32_f16(ClampFunc(source, c0->Type()), c0, c1, c2); |
| }; |
| return TransformTernaryElements(mgr, ty, transform, args[0], args[1], args[2]); |
| } |
| |
| Eval::Result Eval::cos(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto i) -> Eval::Result { |
| using NumberT = decltype(i); |
| return CreateScalar(source, c0->Type(), NumberT(std::cos(i.value))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::cosh(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto i) -> Eval::Result { |
| using NumberT = decltype(i); |
| return CreateScalar(source, c0->Type(), NumberT(std::cosh(i.value))); |
| }; |
| return Dispatch_fa_f32_f16(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::countLeadingZeros(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto e) { |
| using NumberT = decltype(e); |
| using T = UnwrapNumber<NumberT>; |
| auto count = CountLeadingBits(T{e}, T{0}); |
| return CreateScalar(source, c0->Type(), NumberT(count)); |
| }; |
| return Dispatch_iu32(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::countOneBits(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto e) { |
| using NumberT = decltype(e); |
| using T = UnwrapNumber<NumberT>; |
| using UT = std::make_unsigned_t<T>; |
| constexpr UT kRightMost = UT{1}; |
| |
| auto count = UT{0}; |
| for (auto v = static_cast<UT>(e); v != UT{0}; v >>= 1) { |
| if ((v & kRightMost) == 1) { |
| ++count; |
| } |
| } |
| |
| return CreateScalar(source, c0->Type(), NumberT(count)); |
| }; |
| return Dispatch_iu32(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::countTrailingZeros(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) { |
| auto transform = [&](const Value* c0) { |
| auto create = [&](auto e) { |
| using NumberT = decltype(e); |
| using T = UnwrapNumber<NumberT>; |
| auto count = CountTrailingBits(T{e}, T{0}); |
| return CreateScalar(source, c0->Type(), NumberT(count)); |
| }; |
| return Dispatch_iu32(create, c0); |
| }; |
| return TransformUnaryElements(mgr, ty, transform, args[0]); |
| } |
| |
| Eval::Result Eval::cross(const core::type::Type* ty, |
| VectorRef<const Value*> args, |
| const Source& source) |