[tools]: Add remote-compile
A tool for compiling shaders on a remote machine.
Helpful for combining with `test-all`, so that a single non-windows
machine can validate SPIR-V, MSL and HLSL.
Change-Id: I3a0f70e6e4edd13952eb5dc72fbbed7c495036ee
Reviewed-on: https://dawn-review.googlesource.com/c/tint/+/56940
Auto-Submit: Ben Clayton <bclayton@google.com>
Kokoro: Kokoro <noreply+kokoro@google.com>
Reviewed-by: Antonio Maiorano <amaiorano@google.com>
Commit-Queue: Ben Clayton <bclayton@google.com>
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 47a130a..2359e4d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -53,6 +53,7 @@
option(TINT_BUILD_SPIRV_TOOLS_FUZZER "Build SPIRV-Tools fuzzer" OFF)
option(TINT_BUILD_TESTS "Build tests" ${TINT_BUILD_TESTS_DEFAULT})
option(TINT_BUILD_AS_OTHER_OS "Override OS detection to force building of *_other.cc files" OFF)
+option(TINT_BUILD_REMOTE_COMPILE "Build the remote-compile tool for validating shaders on a remote machine" OFF)
option(TINT_ENABLE_MSAN "Enable memory sanitizer" OFF)
option(TINT_ENABLE_ASAN "Enable address sanitizer" OFF)
@@ -77,6 +78,7 @@
message(STATUS "Tint build with MSAN: ${TINT_ENABLE_MSAN}")
message(STATUS "Tint build with UBSAN: ${TINT_ENABLE_UBSAN}")
message(STATUS "Tint build checking [chromium-style]: ${TINT_CHECK_CHROMIUM_STYLE}")
+message(STATUS "Tint build remote-compile tool: ${TINT_BUILD_REMOTE_COMPILE}")
message(STATUS "Using python3")
find_package(PythonInterp 3 REQUIRED)
@@ -99,6 +101,8 @@
set(TINT_BUILD_SPV_WRITER ON)
endif()
+set(TINT_ROOT_SOURCE_DIR ${PROJECT_SOURCE_DIR})
+
# CMake < 3.15 sets /W3 in CMAKE_CXX_FLAGS. Remove it if it's there.
# See https://gitlab.kitware.com/cmake/cmake/-/issues/18317
if (MSVC)
@@ -112,7 +116,7 @@
endif()
if (${TINT_BUILD_SPV_READER})
- include_directories("${PROJECT_SOURCE_DIR}/third_party/spirv-tools/include")
+ include_directories("${TINT_ROOT_SOURCE_DIR}/third_party/spirv-tools/include")
endif()
if((CMAKE_CXX_COMPILER_ID STREQUAL "Clang") AND (CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC"))
@@ -171,12 +175,12 @@
endif()
function(tint_default_compile_options TARGET)
- target_include_directories(${TARGET} PUBLIC "${PROJECT_SOURCE_DIR}")
- target_include_directories(${TARGET} PUBLIC "${PROJECT_SOURCE_DIR}/include")
+ target_include_directories(${TARGET} PUBLIC "${TINT_ROOT_SOURCE_DIR}")
+ target_include_directories(${TARGET} PUBLIC "${TINT_ROOT_SOURCE_DIR}/include")
if (${TINT_BUILD_SPV_READER} OR ${TINT_BUILD_SPV_WRITER})
target_include_directories(${TARGET} PUBLIC
- "${PROJECT_SOURCE_DIR}/third_party/spirv-headers/include")
+ "${TINT_ROOT_SOURCE_DIR}/third_party/spirv-headers/include")
endif()
target_compile_definitions(${TARGET} PUBLIC
@@ -186,7 +190,7 @@
target_compile_definitions(${TARGET} PUBLIC
-DTINT_BUILD_HLSL_WRITER=$<BOOL:${TINT_BUILD_HLSL_WRITER}>)
target_compile_definitions(${TARGET} PUBLIC
- -DTINT_BUILD_MSL_WRITER=$<BOOL:${TINT_BUILD_MSL_WRITER}>)
+ -DTINT_BUILD_MSL_WRITER=$<BOOL:${TINT_BUILD_MSL_WRITER}>)
target_compile_definitions(${TARGET} PUBLIC
-DTINT_BUILD_SPV_WRITER=$<BOOL:${TINT_BUILD_SPV_WRITER}>)
target_compile_definitions(${TARGET} PUBLIC
@@ -304,13 +308,13 @@
add_custom_target(tint-lint
COMMAND ./tools/lint
- WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+ WORKING_DIRECTORY ${TINT_ROOT_SOURCE_DIR}
COMMENT "Running linter"
VERBATIM)
add_custom_target(tint-format
COMMAND ./tools/format
- WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+ WORKING_DIRECTORY ${TINT_ROOT_SOURCE_DIR}
COMMENT "Running formatter"
VERBATIM)
@@ -324,7 +328,11 @@
add_custom_target(tint-generate-coverage
COMMAND ${CMAKE_COMMAND} -E env PATH=${PATH_WITH_CLANG} ./tools/tint-generate-coverage $<TARGET_FILE:tint_unittests>
DEPENDS tint_unittests
- WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+ WORKING_DIRECTORY ${TINT_ROOT_SOURCE_DIR}
COMMENT "Generating tint coverage data"
VERBATIM)
endif()
+
+if (${TINT_BUILD_REMOTE_COMPILE})
+ add_subdirectory("${TINT_ROOT_SOURCE_DIR}/tools/src/cmd/remote-compile")
+endif()
diff --git a/Doxyfile b/Doxyfile
index 63bbc67..8cf5eee 100644
--- a/Doxyfile
+++ b/Doxyfile
@@ -786,8 +786,9 @@
# Note: If this tag is empty the current directory is searched.
INPUT = CODE_OF_CONDUCT.md \
+ fuzzers/tint_spirv_tools_fuzzer \
src \
- fuzzers/tint_spirv_tools_fuzzer
+ tools/src \
# This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
diff --git a/tools/lint b/tools/lint
index 1b20be8..1e6dfd9 100755
--- a/tools/lint
+++ b/tools/lint
@@ -25,6 +25,7 @@
FILTER="-runtime/references"
FILES="`find src -type f` `find samples -type f`"
+FILES+="`find tools/src -type f` `find samples -type f`"
if command -v go &> /dev/null; then
# Go is installed. Run cpplint in parallel for speed wins
diff --git a/tools/src/cmd/remote-compile/CMakeLists.txt b/tools/src/cmd/remote-compile/CMakeLists.txt
new file mode 100644
index 0000000..9a28f62
--- /dev/null
+++ b/tools/src/cmd/remote-compile/CMakeLists.txt
@@ -0,0 +1,37 @@
+# Copyright 2021 The Tint Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(SRC
+ main.cc
+ rwmutex.h
+ socket.cc
+ socket.h
+)
+
+add_executable(tint-remote-compile ${SRC})
+
+target_include_directories(tint-remote-compile PRIVATE "${TINT_ROOT_SOURCE_DIR}")
+
+# If we're building on mac / ios and we have CoreGraphics, then we can use the
+# metal API to validate our shaders. This is roughly 4x faster than invoking
+# the metal shader compiler executable.
+if(APPLE)
+ find_library(LIB_CORE_GRAPHICS CoreGraphics)
+ if(LIB_CORE_GRAPHICS)
+ target_sources(tint-remote-compile PRIVATE "msl_metal.mm")
+ target_compile_definitions(tint-remote-compile PRIVATE "-DTINT_ENABLE_MSL_COMPILATION_USING_METAL_API=1")
+ target_compile_options(tint-remote-compile PRIVATE "-fmodules" "-fcxx-modules")
+ target_link_options(tint-remote-compile PRIVATE "-framework" "CoreGraphics")
+ endif()
+endif()
diff --git a/tools/src/cmd/remote-compile/compile.h b/tools/src/cmd/remote-compile/compile.h
new file mode 100644
index 0000000..8fe1841
--- /dev/null
+++ b/tools/src/cmd/remote-compile/compile.h
@@ -0,0 +1,30 @@
+// Copyright 2021 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_
+#define TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_
+
+#include <string>
+
+/// The return structure of a compile function
+struct CompileResult {
+ /// True if shader compiled
+ bool success = false;
+ /// Output of the compiler
+ std::string output;
+};
+
+CompileResult CompileMslUsingMetalAPI(const std::string& src);
+
+#endif // TOOLS_SRC_CMD_REMOTE_COMPILE_COMPILE_H_
diff --git a/tools/src/cmd/remote-compile/main.cc b/tools/src/cmd/remote-compile/main.cc
new file mode 100644
index 0000000..20c1005
--- /dev/null
+++ b/tools/src/cmd/remote-compile/main.cc
@@ -0,0 +1,440 @@
+// Copyright 2021 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include "tools/src/cmd/remote-compile/compile.h"
+#include "tools/src/cmd/remote-compile/socket.h"
+
+namespace {
+
+#if 0
+#define DEBUG(msg, ...) printf(msg "\n", ##__VA_ARGS__)
+#else
+#define DEBUG(...)
+#endif
+
+/// Print the tool usage, and exit with 1.
+void ShowUsage() {
+ const char* name = "tint-remote-compile";
+ printf(R"(%s is a tool for compiling a shader on a remote machine
+
+usage as server:
+ %s -s [-p port-number]
+
+usage as client:
+ %s [-p port-number] [server-address] shader-file-path
+
+ [server-address] can be omitted if the TINT_REMOTE_COMPILE_ADDRESS environment
+ variable is set.
+ Alternatively, you can pass xcrun arguments so %s can be used as a
+ drop-in replacement.
+)",
+ name, name, name, name);
+ exit(1);
+}
+
+/// The protocol version code. Bump each time the protocol changes
+constexpr uint32_t kProtocolVersion = 1;
+
+/// Supported shader source languages
+enum SourceLanguage {
+ MSL,
+};
+
+/// Stream is a serialization wrapper around a socket
+struct Stream {
+ /// The underlying socket
+ Socket* const socket;
+ /// Error state
+ std::string error;
+
+ /// Writes a uint32_t to the socket
+ Stream operator<<(uint32_t v) {
+ if (error.empty()) {
+ Write(&v, sizeof(v));
+ }
+ return *this;
+ }
+
+ /// Reads a uint32_t from the socket
+ Stream operator>>(uint32_t& v) {
+ if (error.empty()) {
+ Read(&v, sizeof(v));
+ }
+ return *this;
+ }
+
+ /// Writes a std::string to the socket
+ Stream operator<<(const std::string& v) {
+ if (error.empty()) {
+ uint32_t count = static_cast<uint32_t>(v.size());
+ *this << count;
+ if (count) {
+ Write(v.data(), count);
+ }
+ }
+ return *this;
+ }
+
+ /// Reads a std::string from the socket
+ Stream operator>>(std::string& v) {
+ uint32_t count = 0;
+ *this >> count;
+ if (count) {
+ std::vector<char> buf(count);
+ if (Read(buf.data(), count)) {
+ v = std::string(buf.data(), buf.size());
+ }
+ } else {
+ v.clear();
+ }
+ return *this;
+ }
+
+ /// Writes an enum value to the socket
+ template <typename T>
+ std::enable_if_t<std::is_enum<T>::value, Stream> operator<<(T e) {
+ return *this << static_cast<uint32_t>(e);
+ }
+
+ /// Reads an enum value from the socket
+ template <typename T>
+ std::enable_if_t<std::is_enum<T>::value, Stream> operator>>(T& e) {
+ uint32_t v;
+ *this >> v;
+ e = static_cast<T>(v);
+ return *this;
+ }
+
+ private:
+ bool Write(const void* data, size_t size) {
+ if (error.empty()) {
+ if (!socket->Write(data, size)) {
+ error = "Socket::Write() failed";
+ }
+ }
+ return error.empty();
+ }
+
+ bool Read(void* data, size_t size) {
+ auto buf = reinterpret_cast<uint8_t*>(data);
+ while (size > 0 && error.empty()) {
+ if (auto n = socket->Read(buf, size)) {
+ if (n > size) {
+ error = "Socket::Read() returned more bytes than requested";
+ return false;
+ }
+ size -= n;
+ buf += n;
+ }
+ }
+ return error.empty();
+ }
+};
+
+////////////////////////////////////////////////////////////////////////////////
+// Messages
+////////////////////////////////////////////////////////////////////////////////
+
+/// Base class for all messages
+struct Message {
+ /// The type of the message
+ enum class Type {
+ ConnectionRequest,
+ ConnectionResponse,
+ CompileRequest,
+ CompileResponse,
+ };
+
+ explicit Message(Type ty) : type(ty) {}
+
+ Type const type;
+};
+
+struct ConnectionResponse : Message { // Server -> Client
+ ConnectionResponse() : Message(Type::ConnectionResponse) {}
+
+ template <typename T>
+ void Serialize(T&& f) {
+ f(error);
+ }
+
+ std::string error;
+};
+
+struct ConnectionRequest : Message { // Client -> Server
+ using Response = ConnectionResponse;
+
+ explicit ConnectionRequest(uint32_t proto_ver = kProtocolVersion)
+ : Message(Type::ConnectionRequest), protocol_version(proto_ver) {}
+
+ template <typename T>
+ void Serialize(T&& f) {
+ f(protocol_version);
+ }
+
+ uint32_t protocol_version;
+};
+
+struct CompileResponse : Message { // Server -> Client
+ CompileResponse() : Message(Type::CompileResponse) {}
+
+ template <typename T>
+ void Serialize(T&& f) {
+ f(error);
+ }
+
+ std::string error;
+};
+
+struct CompileRequest : Message { // Client -> Server
+ using Response = CompileResponse;
+
+ CompileRequest() : Message(Type::CompileRequest) {}
+ CompileRequest(SourceLanguage lang, std::string src)
+ : Message(Type::CompileRequest), language(lang), source(src) {}
+
+ template <typename T>
+ void Serialize(T&& f) {
+ f(language);
+ f(source);
+ }
+
+ SourceLanguage language;
+ std::string source;
+};
+
+/// Writes the message `m` to the stream `s`
+template <typename MESSAGE>
+std::enable_if_t<std::is_base_of<Message, MESSAGE>::value, Stream>& operator<<(
+ Stream& s,
+ const MESSAGE& m) {
+ s << m.type;
+ const_cast<MESSAGE&>(m).Serialize([&s](const auto& value) { s << value; });
+ return s;
+}
+
+/// Reads the message `m` from the stream `s`
+template <typename MESSAGE>
+std::enable_if_t<std::is_base_of<Message, MESSAGE>::value, Stream>& operator>>(
+ Stream& s,
+ MESSAGE& m) {
+ Message::Type ty;
+ s >> ty;
+ if (ty == m.type) {
+ m.Serialize([&s](auto& value) { s >> value; });
+ } else {
+ std::stringstream ss;
+ ss << "expected message type " << static_cast<int>(m.type) << ", got "
+ << static_cast<int>(ty);
+ s.error = ss.str();
+ }
+ return s;
+}
+
+/// Writes the request message `req` to the stream `s`, then reads and returns
+/// the response message from the same stream.
+template <typename REQUEST, typename RESPONSE = typename REQUEST::Response>
+RESPONSE Send(Stream& s, const REQUEST& req) {
+ s << req;
+ if (s.error.empty()) {
+ RESPONSE resp;
+ s >> resp;
+ if (s.error.empty()) {
+ return resp;
+ }
+ }
+ return {};
+}
+
+} // namespace
+
+bool RunServer(std::string port);
+bool RunClient(std::string address, std::string port, std::string file);
+
+int main(int argc, char* argv[]) {
+ bool run_server = false;
+ std::string port = "19000";
+
+ std::vector<std::string> args;
+ for (int i = 1; i < argc; i++) {
+ std::string arg = argv[i];
+ if (arg == "-s" || arg == "--server") {
+ run_server = true;
+ continue;
+ }
+ if (arg == "-p" || arg == "--port") {
+ if (i < argc - 1) {
+ i++;
+ port = argv[i];
+ } else {
+ printf("expected port number");
+ exit(1);
+ }
+ continue;
+ }
+
+ // xcrun flags are ignored so this executable can be used as a replacement
+ // for xcrun.
+ if ((arg == "-x" || arg == "-sdk") && (i < argc - 1)) {
+ i++;
+ continue;
+ }
+ if (arg == "metal") {
+ for (; i < argc; i++) {
+ if (std::string(argv[i]) == "-c") {
+ break;
+ }
+ }
+ continue;
+ }
+
+ args.emplace_back(arg);
+ }
+
+ bool success = false;
+
+ if (run_server) {
+ success = RunServer(port);
+ } else {
+ std::string address;
+ std::string file;
+ switch (args.size()) {
+ case 1:
+ if (auto* addr = getenv("TINT_REMOTE_COMPILE_ADDRESS")) {
+ address = addr;
+ }
+ file = args[0];
+ break;
+ case 2:
+ address = args[0];
+ file = args[1];
+ break;
+ }
+ if (address.empty() || file.empty()) {
+ ShowUsage();
+ }
+ success = RunClient(address, port, file);
+ }
+
+ if (!success) {
+ exit(1);
+ }
+
+ return 0;
+}
+
+bool RunServer(std::string port) {
+ auto server_socket = Socket::Listen("", port.c_str());
+ if (!server_socket) {
+ printf("Failed to listen on port %s\n", port.c_str());
+ return false;
+ }
+ printf("Listening on port %s...\n", port.c_str());
+ while (auto conn = server_socket->Accept()) {
+ DEBUG("Client connected...");
+ Stream stream{conn.get()};
+
+ {
+ ConnectionRequest req;
+ stream >> req;
+ if (!stream.error.empty()) {
+ printf("%s\n", stream.error.c_str());
+ continue;
+ }
+ ConnectionResponse resp;
+ if (req.protocol_version != kProtocolVersion) {
+ DEBUG("Protocol version mismatch");
+ resp.error = "Protocol version mismatch";
+ stream << resp;
+ continue;
+ }
+ stream << resp;
+ }
+ DEBUG("Connection established");
+ {
+ CompileRequest req;
+ stream >> req;
+ if (!stream.error.empty()) {
+ printf("%s\n", stream.error.c_str());
+ continue;
+ }
+#ifdef TINT_ENABLE_MSL_COMPILATION_USING_METAL_API
+ if (req.language == SourceLanguage::MSL) {
+ auto result = CompileMslUsingMetalAPI(req.source);
+ CompileResponse resp;
+ if (!result.success) {
+ resp.error = result.output;
+ }
+ stream << resp;
+ continue;
+ }
+#endif
+ CompileResponse resp;
+ resp.error = "server cannot compile this type of shader";
+ stream << resp;
+ }
+ }
+ return true;
+}
+
+bool RunClient(std::string address, std::string port, std::string file) {
+ // Read the file
+ std::ifstream input(file, std::ios::binary);
+ if (!input) {
+ printf("Couldn't open '%s'\n", file.c_str());
+ return false;
+ }
+ std::string source((std::istreambuf_iterator<char>(input)),
+ std::istreambuf_iterator<char>());
+
+ constexpr const int timeout_ms = 10000;
+ DEBUG("Connecting to %s:%s...", address.c_str(), port.c_str());
+ auto conn = Socket::Connect(address.c_str(), port.c_str(), timeout_ms);
+ if (!conn) {
+ printf("Connection failed\n");
+ return false;
+ }
+
+ Stream stream{conn.get()};
+
+ DEBUG("Sending connection request...");
+ auto conn_resp = Send(stream, ConnectionRequest{kProtocolVersion});
+ if (!stream.error.empty()) {
+ printf("%s\n", stream.error.c_str());
+ return false;
+ }
+ if (!conn_resp.error.empty()) {
+ printf("%s\n", conn_resp.error.c_str());
+ return false;
+ }
+ DEBUG("Connection established. Requesting compile...");
+ auto comp_resp = Send(stream, CompileRequest{SourceLanguage::MSL, source});
+ if (!stream.error.empty()) {
+ printf("%s\n", stream.error.c_str());
+ return false;
+ }
+ if (!comp_resp.error.empty()) {
+ printf("%s\n", comp_resp.error.c_str());
+ return false;
+ }
+ DEBUG("Compilation successful");
+ return true;
+}
diff --git a/tools/src/cmd/remote-compile/msl_metal.mm b/tools/src/cmd/remote-compile/msl_metal.mm
new file mode 100644
index 0000000..a0b0b5a
--- /dev/null
+++ b/tools/src/cmd/remote-compile/msl_metal.mm
@@ -0,0 +1,53 @@
+// Copyright 2021 The Tint Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef TINT_ENABLE_MSL_COMPILATION_USING_METAL_API
+
+@import Metal;
+
+// Disable: error: treating #include as an import of module 'std.string'
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wauto-import"
+#include "compile.h"
+#pragma clang diagnostic pop
+
+CompileResult CompileMslUsingMetalAPI(const std::string& src) {
+ CompileResult result;
+ result.success = false;
+
+ NSError* error = nil;
+
+ id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+ if (!device) {
+ result.output = "MTLCreateSystemDefaultDevice returned null";
+ result.success = false;
+ return result;
+ }
+
+ NSString* source = [NSString stringWithCString:src.c_str()
+ encoding:NSUTF8StringEncoding];
+
+ id<MTLLibrary> library = [device newLibraryWithSource:source
+ options:nil
+ error:&error];
+ if (!library) {
+ NSString* output = [error localizedDescription];
+ result.output = [output UTF8String];
+ result.success = false;
+ }
+
+ return result;
+}
+
+#endif
diff --git a/tools/src/cmd/remote-compile/rwmutex.h b/tools/src/cmd/remote-compile/rwmutex.h
new file mode 100644
index 0000000..ea55209
--- /dev/null
+++ b/tools/src/cmd/remote-compile/rwmutex.h
@@ -0,0 +1,190 @@
+// Copyright 2020 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_
+#define TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_
+
+#include <condition_variable> // NOLINT
+#include <mutex> // NOLINT
+
+////////////////////////////////////////////////////////////////////////////////
+// RWMutex
+////////////////////////////////////////////////////////////////////////////////
+
+/// A RWMutex is a reader/writer mutual exclusion lock.
+/// The lock can be held by an arbitrary number of readers or a single writer.
+/// Also known as a shared mutex.
+class RWMutex {
+ public:
+ inline RWMutex() = default;
+
+ /// lockReader() locks the mutex for reading.
+ /// Multiple read locks can be held while there are no writer locks.
+ inline void lockReader();
+
+ /// unlockReader() unlocks the mutex for reading.
+ inline void unlockReader();
+
+ /// lockWriter() locks the mutex for writing.
+ /// If the lock is already locked for reading or writing, lockWriter blocks
+ /// until the lock is available.
+ inline void lockWriter();
+
+ /// unlockWriter() unlocks the mutex for writing.
+ inline void unlockWriter();
+
+ private:
+ RWMutex(const RWMutex&) = delete;
+ RWMutex& operator=(const RWMutex&) = delete;
+
+ int readLocks = 0;
+ int pendingWriteLocks = 0;
+ std::mutex mutex;
+ std::condition_variable cv;
+};
+
+void RWMutex::lockReader() {
+ std::unique_lock<std::mutex> lock(mutex);
+ readLocks++;
+}
+
+void RWMutex::unlockReader() {
+ std::unique_lock<std::mutex> lock(mutex);
+ readLocks--;
+ if (readLocks == 0 && pendingWriteLocks > 0) {
+ cv.notify_one();
+ }
+}
+
+void RWMutex::lockWriter() {
+ std::unique_lock<std::mutex> lock(mutex);
+ if (readLocks > 0) {
+ pendingWriteLocks++;
+ cv.wait(lock, [&] { return readLocks == 0; });
+ pendingWriteLocks--;
+ }
+ lock.release(); // Keep lock held
+}
+
+void RWMutex::unlockWriter() {
+ if (pendingWriteLocks > 0) {
+ cv.notify_one();
+ }
+ mutex.unlock();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// RLock
+////////////////////////////////////////////////////////////////////////////////
+
+/// RLock is a RAII read lock helper for a RWMutex.
+class RLock {
+ public:
+ /// Constructor.
+ /// Locks `mutex` with a read-lock for the lifetime of the WLock.
+ /// @param mutex the mutex
+ explicit inline RLock(RWMutex& mutex);
+ /// Destructor.
+ /// Unlocks the RWMutex.
+ inline ~RLock();
+
+ /// Move constructor
+ /// @param other the other RLock to move into this RLock.
+ inline RLock(RLock&& other);
+ /// Move assignment operator
+ /// @param other the other RLock to move into this RLock.
+ /// @returns this RLock so calls can be chained
+ inline RLock& operator=(RLock&& other);
+
+ private:
+ RLock(const RLock&) = delete;
+ RLock& operator=(const RLock&) = delete;
+
+ RWMutex* m;
+};
+
+RLock::RLock(RWMutex& mutex) : m(&mutex) {
+ m->lockReader();
+}
+
+RLock::~RLock() {
+ if (m != nullptr) {
+ m->unlockReader();
+ }
+}
+
+RLock::RLock(RLock&& other) {
+ m = other.m;
+ other.m = nullptr;
+}
+
+RLock& RLock::operator=(RLock&& other) {
+ m = other.m;
+ other.m = nullptr;
+ return *this;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// WLock
+////////////////////////////////////////////////////////////////////////////////
+
+/// WLock is a RAII write lock helper for a RWMutex.
+class WLock {
+ public:
+ /// Constructor.
+ /// Locks `mutex` with a write-lock for the lifetime of the WLock.
+ /// @param mutex the mutex
+ explicit inline WLock(RWMutex& mutex);
+
+ /// Destructor.
+ /// Unlocks the RWMutex.
+ inline ~WLock();
+
+ /// Move constructor
+ /// @param other the other WLock to move into this WLock.
+ inline WLock(WLock&& other);
+ /// Move assignment operator
+ /// @param other the other WLock to move into this WLock.
+ /// @returns this WLock so calls can be chained
+ inline WLock& operator=(WLock&& other);
+
+ private:
+ WLock(const WLock&) = delete;
+ WLock& operator=(const WLock&) = delete;
+
+ RWMutex* m;
+};
+
+WLock::WLock(RWMutex& mutex) : m(&mutex) {
+ m->lockWriter();
+}
+
+WLock::~WLock() {
+ if (m != nullptr) {
+ m->unlockWriter();
+ }
+}
+
+WLock::WLock(WLock&& other) {
+ m = other.m;
+ other.m = nullptr;
+}
+
+WLock& WLock::operator=(WLock&& other) {
+ m = other.m;
+ other.m = nullptr;
+ return *this;
+}
+
+#endif // TOOLS_SRC_CMD_REMOTE_COMPILE_RWMUTEX_H_
diff --git a/tools/src/cmd/remote-compile/socket.cc b/tools/src/cmd/remote-compile/socket.cc
new file mode 100644
index 0000000..6c21311
--- /dev/null
+++ b/tools/src/cmd/remote-compile/socket.cc
@@ -0,0 +1,310 @@
+// Copyright 2021 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/src/cmd/remote-compile/socket.h"
+
+#include "tools/src/cmd/remote-compile/rwmutex.h"
+
+#if defined(_WIN32)
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#else
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <sys/select.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <unistd.h>
+#endif
+
+#if defined(_WIN32)
+#include <atomic>
+namespace {
+std::atomic<int> wsaInitCount = {0};
+} // anonymous namespace
+#else
+#include <fcntl.h>
+namespace {
+using SOCKET = int;
+} // anonymous namespace
+#endif
+
+namespace {
+constexpr SOCKET InvalidSocket = static_cast<SOCKET>(-1);
+void init() {
+#if defined(_WIN32)
+ if (wsaInitCount++ == 0) {
+ WSADATA winsockData;
+ (void)WSAStartup(MAKEWORD(2, 2), &winsockData);
+ }
+#endif
+}
+
+void term() {
+#if defined(_WIN32)
+ if (--wsaInitCount == 0) {
+ WSACleanup();
+ }
+#endif
+}
+
+bool setBlocking(SOCKET s, bool blocking) {
+#if defined(_WIN32)
+ u_long mode = blocking ? 0 : 1;
+ return ioctlsocket(s, FIONBIO, &mode) == NO_ERROR;
+#else
+ auto arg = fcntl(s, F_GETFL, nullptr);
+ if (arg < 0) {
+ return false;
+ }
+ arg = blocking ? (arg & ~O_NONBLOCK) : (arg | O_NONBLOCK);
+ return fcntl(s, F_SETFL, arg) >= 0;
+#endif
+}
+
+bool errored(SOCKET s) {
+ if (s == InvalidSocket) {
+ return true;
+ }
+ char error = 0;
+ socklen_t len = sizeof(error);
+ getsockopt(s, SOL_SOCKET, SO_ERROR, &error, &len);
+ return error != 0;
+}
+
+class Impl : public Socket {
+ public:
+ static std::shared_ptr<Impl> create(const char* address, const char* port) {
+ init();
+
+ addrinfo hints = {};
+ hints.ai_family = AF_INET;
+ hints.ai_socktype = SOCK_STREAM;
+ hints.ai_protocol = IPPROTO_TCP;
+ hints.ai_flags = AI_PASSIVE;
+
+ addrinfo* info = nullptr;
+ auto err = getaddrinfo(address, port, &hints, &info);
+#if !defined(_WIN32)
+ if (err) {
+ printf("getaddrinfo(%s, %s) error: %s\n", address, port,
+ gai_strerror(err));
+ }
+#endif
+
+ if (info) {
+ auto socket =
+ ::socket(info->ai_family, info->ai_socktype, info->ai_protocol);
+ auto out = std::make_shared<Impl>(info, socket);
+ out->setOptions();
+ return out;
+ }
+
+ freeaddrinfo(info);
+ term();
+ return nullptr;
+ }
+
+ explicit Impl(SOCKET socket) : info(nullptr), s(socket) {}
+ Impl(addrinfo* info, SOCKET socket) : info(info), s(socket) {}
+
+ ~Impl() {
+ freeaddrinfo(info);
+ Close();
+ term();
+ }
+
+ template <typename FUNCTION>
+ void lock(FUNCTION&& f) {
+ RLock l(mutex);
+ f(s, info);
+ }
+
+ void setOptions() {
+ RLock l(mutex);
+ if (s == InvalidSocket) {
+ return;
+ }
+
+ int enable = 1;
+
+#if !defined(_WIN32)
+ // Prevent sockets lingering after process termination, causing
+ // reconnection issues on the same port.
+ setsockopt(s, SOL_SOCKET, SO_REUSEADDR, reinterpret_cast<char*>(&enable),
+ sizeof(enable));
+
+ struct {
+ int l_onoff; /* linger active */
+ int l_linger; /* how many seconds to linger for */
+ } linger = {false, 0};
+ setsockopt(s, SOL_SOCKET, SO_LINGER, reinterpret_cast<char*>(&linger),
+ sizeof(linger));
+#endif // !defined(_WIN32)
+
+ // Enable TCP_NODELAY.
+ setsockopt(s, IPPROTO_TCP, TCP_NODELAY, reinterpret_cast<char*>(&enable),
+ sizeof(enable));
+ }
+
+ bool IsOpen() override {
+ {
+ RLock l(mutex);
+ if ((s != InvalidSocket) && !errored(s)) {
+ return true;
+ }
+ }
+ WLock lock(mutex);
+ s = InvalidSocket;
+ return false;
+ }
+
+ void Close() override {
+ {
+ RLock l(mutex);
+ if (s != InvalidSocket) {
+#if defined(_WIN32)
+ closesocket(s);
+#else
+ ::shutdown(s, SHUT_RDWR);
+#endif
+ }
+ }
+
+ WLock l(mutex);
+ if (s != InvalidSocket) {
+#if !defined(_WIN32)
+ ::close(s);
+#endif
+ s = InvalidSocket;
+ }
+ }
+
+ size_t Read(void* buffer, size_t bytes) override {
+ RLock lock(mutex);
+ if (s == InvalidSocket) {
+ return 0;
+ }
+ auto len =
+ recv(s, reinterpret_cast<char*>(buffer), static_cast<int>(bytes), 0);
+ return (len < 0) ? 0 : len;
+ }
+
+ bool Write(const void* buffer, size_t bytes) override {
+ RLock lock(mutex);
+ if (s == InvalidSocket) {
+ return false;
+ }
+ if (bytes == 0) {
+ return true;
+ }
+ return ::send(s, reinterpret_cast<const char*>(buffer),
+ static_cast<int>(bytes), 0) > 0;
+ }
+
+ std::shared_ptr<Socket> Accept() override {
+ std::shared_ptr<Impl> out;
+ lock([&](SOCKET socket, const addrinfo*) {
+ if (socket != InvalidSocket) {
+ init();
+ out = std::make_shared<Impl>(::accept(socket, 0, 0));
+ out->setOptions();
+ }
+ });
+ return out;
+ }
+
+ private:
+ addrinfo* const info;
+ SOCKET s = InvalidSocket;
+ RWMutex mutex;
+};
+
+} // anonymous namespace
+
+std::shared_ptr<Socket> Socket::Listen(const char* address, const char* port) {
+ auto impl = Impl::create(address, port);
+ if (!impl) {
+ return nullptr;
+ }
+ impl->lock([&](SOCKET socket, const addrinfo* info) {
+ if (bind(socket, info->ai_addr, static_cast<int>(info->ai_addrlen)) != 0) {
+ impl.reset();
+ return;
+ }
+
+ if (listen(socket, 0) != 0) {
+ impl.reset();
+ return;
+ }
+ });
+ return impl;
+}
+
+std::shared_ptr<Socket> Socket::Connect(const char* address,
+ const char* port,
+ uint32_t timeoutMillis) {
+ auto impl = Impl::create(address, port);
+ if (!impl) {
+ return nullptr;
+ }
+
+ std::shared_ptr<Socket> out;
+ impl->lock([&](SOCKET socket, const addrinfo* info) {
+ if (socket == InvalidSocket) {
+ return;
+ }
+
+ if (timeoutMillis == 0) {
+ if (::connect(socket, info->ai_addr,
+ static_cast<int>(info->ai_addrlen)) == 0) {
+ out = impl;
+ }
+ return;
+ }
+
+ if (!setBlocking(socket, false)) {
+ return;
+ }
+
+ auto res =
+ ::connect(socket, info->ai_addr, static_cast<int>(info->ai_addrlen));
+ if (res == 0) {
+ if (setBlocking(socket, true)) {
+ out = impl;
+ }
+ } else {
+ const auto microseconds = timeoutMillis * 1000;
+
+ fd_set fdset;
+ FD_ZERO(&fdset);
+ FD_SET(socket, &fdset);
+
+ timeval tv;
+ tv.tv_sec = microseconds / 1000000;
+ tv.tv_usec = microseconds - static_cast<uint32_t>(tv.tv_sec * 1000000);
+ res = select(static_cast<int>(socket + 1), nullptr, &fdset, nullptr, &tv);
+ if (res > 0 && !errored(socket) && setBlocking(socket, true)) {
+ out = impl;
+ }
+ }
+ });
+
+ if (!out) {
+ return nullptr;
+ }
+
+ return out->IsOpen() ? out : nullptr;
+}
diff --git a/tools/src/cmd/remote-compile/socket.h b/tools/src/cmd/remote-compile/socket.h
new file mode 100644
index 0000000..06e93dd
--- /dev/null
+++ b/tools/src/cmd/remote-compile/socket.h
@@ -0,0 +1,71 @@
+// Copyright 2021 The Tint Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_
+#define TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_
+
+#include <atomic>
+#include <memory>
+
+/// Socket provides an OS abstraction to a TCP socket.
+class Socket {
+ public:
+ /// Connects to the given TCP address and port.
+ /// @param address the target socket address
+ /// @param port the target socket port
+ /// @param timeoutMillis the timeout for the connection attempt.
+ /// If timeoutMillis is non-zero and no connection was made before
+ /// timeoutMillis milliseconds, then nullptr is returned.
+ /// @returns the connected Socket, or nullptr on failure
+ static std::shared_ptr<Socket> Connect(const char* address,
+ const char* port,
+ uint32_t timeoutMillis);
+
+ /// Begins listening for connections on the given TCP address and port.
+ /// Call Accept() on the returned Socket to block and wait for a connection.
+ /// @param address the socket address to listen on. Use "localhost" for
+ /// connections from only this machine, or an empty string to allow
+ /// connections from any incoming address.
+ /// @param port the socket port to listen on
+ /// @returns the Socket that listens for connections
+ static std::shared_ptr<Socket> Listen(const char* address, const char* port);
+
+ /// Attempts to read at most `n` bytes into buffer, returning the actual
+ /// number of bytes read.
+ /// read() will block until the socket is closed or at least one byte is read.
+ /// @param buffer the output buffer. Must be at least `n` bytes in size.
+ /// @param n the maximum number of bytes to read
+ /// @return the number of bytes read, or 0 if the socket was closed
+ virtual size_t Read(void* buffer, size_t n) = 0;
+
+ /// Writes `n` bytes from buffer into the socket.
+ /// @param buffer the source data buffer. Must be at least `n` bytes in size.
+ /// @param n the number of bytes to read from `buffer`
+ /// @returns true on success, or false if there was an error or the socket was
+ /// closed.
+ virtual bool Write(const void* buffer, size_t n) = 0;
+
+ /// @returns true if the socket has not been closed.
+ virtual bool IsOpen() = 0;
+
+ /// Closes the socket.
+ virtual void Close() = 0;
+
+ /// Blocks for a connection to be made to the listening port, or for the
+ /// Socket to be closed.
+ /// @returns a pointer to the next established incoming connection
+ virtual std::shared_ptr<Socket> Accept() = 0;
+};
+
+#endif // TOOLS_SRC_CMD_REMOTE_COMPILE_SOCKET_H_
