From 0ec5a93c4ea62177eda2f438005bd4b6464669dd Mon Sep 17 00:00:00 2001 From: Pablo Marquez Tello Date: Tue, 8 Sep 2026 10:42:20 +0100 Subject: [PATCH] fix: Avoid SVE codegen in SME wrapper The arm_gemm SME interleave wrappers are instantiated from the SVE source bucket so explicit SME and Streaming-SVE assembly can be assembled. That also allowed the compiler to auto-vectorize ordinary C++ wrapper code into non-streaming SVE instructions, which is invalid on CPUs that expose SME/SME2 without normal SVE. Split the affected wrapper into a protected SVE bucket. It keeps the existing SVE assembler target but disables compiler vectorization for the ordinary wrapper code. Normal SVE and SVE2 buckets stay unchanged, so multi-ISA builds still carry their runtime-dispatched implementations. Resolves MLCE-2015 Signed-off-by: Pablo Marquez Tello Change-Id: Idea17eef5a0cd422c2510ba4dbe6882f33120591 --- BUILD.bazel | 59 +++++++++++++++++++++++++++++++++ CMakeLists.txt | 28 ++++++++++++++++ SConscript | 13 ++++++++ filedefs.json | 5 +++ scripts/generate_build_files.py | 34 +++++++++++++++---- src/BUILD.bazel | 10 +++++- src/CMakeLists.txt | 7 +++- 7 files changed, 148 insertions(+), 8 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 8be867e8140..4096de08e9b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -363,6 +363,64 @@ cc_library( alwayslink = True, ) +#--------------------------------------------------------------------- +# SVE library sources that must not be compiler-vectorized +# +# SME does not imply non-streaming SVE. These sources still need the SVE target +# for explicit Streaming-SVE/SME code, but ordinary C++ must not gain an +# unintended non-streaming SVE dependency. + +cc_library( + name = "arm_compute_sve_no_vectorize", + srcs = ["//src:arm_compute_sve_no_vectorize_srcs"], + copts = [ + "-march=armv8.2-a+sve+fp16+dotprod", + ] + select({ + "//:debug_flag": [ + "-O0", + "-g", + "-gdwarf-2", + ], + "//conditions:default": ["-O3"], + }) + + [ + "-fno-tree-vectorize", + "-fno-tree-slp-vectorize", + ] + + select({ + "//:openmp_flag": ["-fopenmp"], + "//conditions:default": [], + }) + + select({ + "//:Werror_flag": ["-Werror"], + "//conditions:default": [], + }), + includes = [ + "src/core/NEON/kernels/arm_conv", + "src/core/NEON/kernels/arm_gemm", + "src/core/NEON/kernels/assembly", + "src/core/cpu/kernels/assembly", + "src/cpu/kernels/assembly", + ], + linkopts = select({ + "//:openmp_flag": ["-fopenmp"], + "//conditions:default": [], + }), + local_defines = [ + "ENABLE_SVE", + "ARM_COMPUTE_ENABLE_SVE", + "ARM_COMPUTE_ENABLE_BF16", + ], + deps = [ + "//:common_defines", + "//arm_compute:core_headers", + "//arm_compute:runtime_headers", + "//include", + "//support", + ], + alwayslink = True, +) + #--------------------------------------------------------------------- # Core and Runtime library @@ -426,6 +484,7 @@ cc_library( "@@kleidiai//:common", "@@kleidiai//kai/ukernels/matmul:matmul", "//:arm_compute_sve", + "//:arm_compute_sve_no_vectorize", "//:arm_compute_sve2" ], alwayslink = True, diff --git a/CMakeLists.txt b/CMakeLists.txt index 7280352f8c0..6c9d0ea6110 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,6 +136,11 @@ else() message(STATUS "Using arch: ${ARM_COMPUTE_ARCH}") endif() +# SME does not imply non-streaming SVE. SME-only wrappers built in an +# SVE-capable target must disable compiler vectorization so ordinary C++ cannot +# acquire an unintended non-streaming SVE dependency. +set(ARM_COMPUTE_SME_WRAPPER_NO_VECTORIZE_FLAGS "-fno-tree-vectorize;-fno-tree-slp-vectorize") + if(ARM_COMPUTE_ENABLE_OPENMP) find_package(OpenMP REQUIRED) endif() @@ -204,6 +209,20 @@ else() add_library(arm_compute_sve OBJECT EXCLUDE_FROM_ALL) endif() +if(ACL_MULTI_ISA OR ACL_BUILD_SVE) + add_library(arm_compute_sve_no_vectorize OBJECT) + set_target_properties( + arm_compute_sve_no_vectorize + PROPERTIES + COMPILE_OPTIONS "${ARM_COMPUTE_SVE_ARCH};${ARM_COMPUTE_COMMON_CCXX_FLAGS};${ARM_COMPUTE_SME_WRAPPER_NO_VECTORIZE_FLAGS}" + COMPILE_DEFINITIONS "${ARM_COMPUTE_DEFINES}" + INCLUDE_DIRECTORIES "${ARM_COMPUTE_SVE_COMMON_INCLUDE}" + LINK_LIBRARIES "${ARM_COMPUTE_LINK_LIBS}" + ) +else() + add_library(arm_compute_sve_no_vectorize OBJECT EXCLUDE_FROM_ALL) +endif() + if(ACL_MULTI_ISA OR ACL_BUILD_SVE2) add_library(arm_compute_sve2 OBJECT) set_target_properties( @@ -257,6 +276,7 @@ if(ACL_MULTI_ISA) $ $ $ + $ $ ) else() @@ -269,6 +289,7 @@ else() list(APPEND lib_objs $) if(ACL_BUILD_SVE) list(APPEND lib_objs $) + list(APPEND lib_objs $) endif() if(ACL_BUILD_SVE2) list(APPEND lib_objs $) @@ -278,6 +299,7 @@ else() list(APPEND lib_objs $) if(ACL_BUILD_SVE) list(APPEND lib_objs $) + list(APPEND lib_objs $) endif() if(ACL_BUILD_SVE2) list(APPEND lib_objs $) @@ -296,6 +318,9 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") if(TARGET arm_compute_core_fp16) target_compile_options(arm_compute_core_fp16 PRIVATE -arch arm64) endif() + if(TARGET arm_compute_sve_no_vectorize) + target_compile_options(arm_compute_sve_no_vectorize PRIVATE -arch arm64) + endif() endif() @@ -319,6 +344,9 @@ endif() if(TARGET arm_compute_sve) list(APPEND ARM_COMPUTE_TARGETS arm_compute_sve) endif() +if(TARGET arm_compute_sve_no_vectorize) + list(APPEND ARM_COMPUTE_TARGETS arm_compute_sve_no_vectorize) +endif() if(TARGET arm_compute_sve2) list(APPEND ARM_COMPUTE_TARGETS arm_compute_sve2) endif() diff --git a/SConscript b/SConscript index 53839406258..f1c6a851066 100644 --- a/SConscript +++ b/SConscript @@ -118,6 +118,8 @@ def build_multiisa_lib_objects(): # Build the SVE specific files lib_static_objs += build_obj_list(filedefs["armv8.2-a-sve"], misa_lib_files_sve, static=True) lib_shared_objs += build_obj_list(filedefs["armv8.2-a-sve"], misa_lib_files_sve, static=False) + lib_static_objs += build_obj_list(filedefs["armv8.2-a-sve-no-vectorize"], misa_lib_files_sve_no_vectorize, static=True) + lib_shared_objs += build_obj_list(filedefs["armv8.2-a-sve-no-vectorize"], misa_lib_files_sve_no_vectorize, static=False) lib_static_objs += build_obj_list(filedefs["armv8.2-a-sve"], misa_lib_files_sve_fp16, static=True) lib_shared_objs += build_obj_list(filedefs["armv8.2-a-sve"], misa_lib_files_sve_fp16, static=False) @@ -645,11 +647,16 @@ lib_files_sve2 = [] misa_lib_files = lib_files misa_lib_files_sve = [] +misa_lib_files_sve_no_vectorize = [] misa_lib_files_sve2 = [] misa_lib_files_neon_fp16 = [] misa_lib_files_sve_fp16 = [] misa_lib_files_sve2_fp16 = [] +sve_no_vectorize_files = [ + "src/core/NEON/kernels/arm_gemm/interleave_indirect-sve.cpp", +] + arm_compute_env.Append(CPPPATH = ["src/cpu/kernels/assembly/"]) if env['neon']: @@ -708,6 +715,12 @@ if env['neon']: # SVE files only minus FP16 misa_lib_files_sve = cpu_files.get('sve', []) + # SME does not imply non-streaming SVE. This SME-only wrapper still + # needs the SVE assembler target for explicit Streaming-SVE/SME code, + # but ordinary compiler-generated C++ must not gain an SVE dependency. + sve_no_vectorize_set = set(sve_no_vectorize_files) + misa_lib_files_sve_no_vectorize = [f for f in misa_lib_files_sve if f in sve_no_vectorize_set] + misa_lib_files_sve = [f for f in misa_lib_files_sve if f not in sve_no_vectorize_set] # SVE2 files only minus FP16 misa_lib_files_sve2 = cpu_files.get('sve2', []) diff --git a/filedefs.json b/filedefs.json index 885f958d3ad..c04257dff33 100644 --- a/filedefs.json +++ b/filedefs.json @@ -13,6 +13,11 @@ "cppdefines": ["ARM_COMPUTE_ENABLE_FP16", "ARM_COMPUTE_ENABLE_BF16", "ARM_COMPUTE_ENABLE_I8MM", "ARM_COMPUTE_ENABLE_SVEF32MM"] }, + "armv8.2-a-sve-no-vectorize": { + "ccflags": ["-march=armv8.2-a+sve+fp16+dotprod", "-fno-tree-vectorize", "-fno-tree-slp-vectorize"], + "cppdefines": ["ARM_COMPUTE_ENABLE_FP16", "ARM_COMPUTE_ENABLE_BF16", + "ARM_COMPUTE_ENABLE_I8MM", "ARM_COMPUTE_ENABLE_SVEF32MM"] + }, "armv8.2-a-sve2": { "ccflags": ["-march=armv8.2-a+sve2+fp16+dotprod"], "cppdefines": ["ARM_COMPUTE_ENABLE_FP16", "ARM_COMPUTE_ENABLE_BF16", diff --git a/scripts/generate_build_files.py b/scripts/generate_build_files.py index b5d18390494..65517dda32a 100644 --- a/scripts/generate_build_files.py +++ b/scripts/generate_build_files.py @@ -36,6 +36,10 @@ import json import glob +SVE_NO_VECTORIZE_FILES = [ + "src/core/NEON/kernels/arm_gemm/interleave_indirect-sve.cpp", +] + def get_operator_backend_files(filelist, operators, backend='', techs=[], attrs=[], include_common=True): files = {"common": []} @@ -117,7 +121,7 @@ def get_template_header(): # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE.""" -def build_from_template_bazel(srcs_graph, srcs_sve, srcs_sve2, _srcs_core): +def build_from_template_bazel(srcs_graph, srcs_sve, srcs_sve_no_vectorize, srcs_sve2, _srcs_core): # Bazel does not support targets referencing upper-levels. srcs_core = [path for path in _srcs_core if not path.startswith("../")] @@ -152,6 +156,15 @@ def build_from_template_bazel(srcs_graph, srcs_sve, srcs_sve2, _srcs_core): visibility = ["//visibility:public"] ) +filegroup( + name = "arm_compute_sve_no_vectorize_srcs", + srcs = ["{line_separator.join(srcs_sve_no_vectorize)}"] + + glob(["**/*.h", + "**/*.hpp", + "**/*.inl"]), + visibility = ["//visibility:public"] +) + filegroup( name = "arm_compute_srcs", srcs = ["{line_separator.join(srcs_core)}"] + @@ -165,7 +178,7 @@ def build_from_template_bazel(srcs_graph, srcs_sve, srcs_sve2, _srcs_core): return template -def build_from_template_cmake(srcs_graph, srcs_sve, srcs_sve2, srcs_core, srcs_core_fp16): +def build_from_template_cmake(srcs_graph, srcs_sve, srcs_sve_no_vectorize, srcs_sve2, srcs_core, srcs_core_fp16): line_separator = '\n\t' @@ -183,6 +196,12 @@ def build_from_template_cmake(srcs_graph, srcs_sve, srcs_sve2, srcs_core, srcs_c {line_separator.join(srcs_sve)} ) +target_sources( + arm_compute_sve_no_vectorize + PRIVATE + {line_separator.join(srcs_sve_no_vectorize)} +) + target_sources( arm_compute_sve2 PRIVATE @@ -266,6 +285,8 @@ def gather_sources(): # SVE files only lib_files_sve = cpu_files.get('sve', []) lib_files_sve += fp16_cpu_files.get('sve', []) + lib_files_sve_no_vectorize = [path for path in lib_files_sve if path in SVE_NO_VECTORIZE_FILES] + lib_files_sve = [path for path in lib_files_sve if path not in SVE_NO_VECTORIZE_FILES] # SVE2 files only lib_files_sve2 = cpu_files.get('sve2', []) @@ -280,11 +301,12 @@ def strip_prefix(filename, prefix = "src/"): graph_files = sorted([strip_prefix(path, "src/") for path in graph_files]) lib_files_sve = sorted([strip_prefix(path, "src/") for path in lib_files_sve]) + lib_files_sve_no_vectorize = sorted([strip_prefix(path, "src/") for path in lib_files_sve_no_vectorize]) lib_files_sve2 = sorted([strip_prefix(path, "src/") for path in lib_files_sve2]) lib_files = sorted([strip_prefix(path, "src/") for path in lib_files]) lib_files_neon_fp16 = sorted([strip_prefix(path, "src/") for path in lib_files_neon_fp16]) - return (graph_files, lib_files_sve, lib_files_sve2, lib_files, lib_files_neon_fp16) + return (graph_files, lib_files_sve, lib_files_sve_no_vectorize, lib_files_sve2, lib_files, lib_files_neon_fp16) if "__main__" in __name__: @@ -294,20 +316,20 @@ def strip_prefix(filename, prefix = "src/"): parser.add_argument("--cmake", action="store_true") args = parser.parse_args() - (graph_files, lib_files_sve, lib_files_sve2, lib_files, lib_files_neon_fp16) = gather_sources() + (graph_files, lib_files_sve, lib_files_sve_no_vectorize, lib_files_sve2, lib_files, lib_files_neon_fp16) = gather_sources() if args.bazel: # 8562a4ec: Remove CommonGraphOptions from Utils target and warnings graph_files += ["//utils:CommonGraphOptions.cpp"] bazel_build_string = build_from_template_bazel( - graph_files, lib_files_sve, lib_files_sve2, lib_files + lib_files_neon_fp16) + graph_files, lib_files_sve, lib_files_sve_no_vectorize, lib_files_sve2, lib_files + lib_files_neon_fp16) with open("src/BUILD.bazel", "w") as fp: fp.write(bazel_build_string) if args.cmake: cmake_build_string = build_from_template_cmake( - graph_files, lib_files_sve, lib_files_sve2, lib_files, lib_files_neon_fp16) + graph_files, lib_files_sve, lib_files_sve_no_vectorize, lib_files_sve2, lib_files, lib_files_neon_fp16) with open("src/CMakeLists.txt", "w") as fp: fp.write(cmake_build_string) diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 9124569303b..b7f7f68ce09 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -250,7 +250,6 @@ filegroup( "core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp", "core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp", "core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp", - "core/NEON/kernels/arm_gemm/interleave_indirect-sve.cpp", "core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp", "core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp16fp32fp16_dot_16VL/generic.cpp", "core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp", @@ -407,6 +406,15 @@ filegroup( visibility = ["//visibility:public"] ) +filegroup( + name = "arm_compute_sve_no_vectorize_srcs", + srcs = ["core/NEON/kernels/arm_gemm/interleave_indirect-sve.cpp"] + + glob(["**/*.h", + "**/*.hpp", + "**/*.inl"]), + visibility = ["//visibility:public"] +) + filegroup( name = "arm_compute_srcs", srcs = ["c/AclContext.cpp", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 27fca46643e..f814e59e01c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -219,7 +219,6 @@ target_sources( core/NEON/kernels/arm_conv/pooling/kernels/sve_u8_nhwc_max_generic_depthfirst/generic.cpp core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_avg_generic_depthfirst/generic.cpp core/NEON/kernels/arm_conv/pooling/kernels/sve_u8q_nhwc_max_generic_depthfirst/generic.cpp - core/NEON/kernels/arm_gemm/interleave_indirect-sve.cpp core/NEON/kernels/arm_gemm/kernels/sme2_gemv_bf16fp32_dot_16VL/generic.cpp core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp16fp32fp16_dot_16VL/generic.cpp core/NEON/kernels/arm_gemm/kernels/sme2_gemv_fp32_mla_16VL/generic.cpp @@ -372,6 +371,12 @@ target_sources( cpu/kernels/topkv/generic/sve/qasymm8_signed.cpp ) +target_sources( + arm_compute_sve_no_vectorize + PRIVATE + core/NEON/kernels/arm_gemm/interleave_indirect-sve.cpp +) + target_sources( arm_compute_sve2 PRIVATE