Skip to content

Fix ARM64 SME direct-kernel build and dispatch #5898

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
moluopro wants to merge 6 commits into
base: develop
Choose a base branch
from
+1,160 −1,142
Open

Fix ARM64 SME direct-kernel build and dispatch #5898

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8dcf13a
cmake: clean up ARMV9SME Windows Clang flags
moluopro Jul 9, 2026
a33e238
cmake: define ARM64 SME direct kernel variants
moluopro Jul 9, 2026
18b41f3
interface: gate ARM64 STRMM direct fast path
moluopro Jul 9, 2026
95cae64
arm64: harden SME SGEMM direct heuristic
moluopro Jul 9, 2026
6fdc8db
arm64: tighten SME STRMM direct variants
moluopro Jul 9, 2026
06d553d
arm64: normalize SME direct source line endings
moluopro Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions cmake/cc.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,13 +313,13 @@ endif ()
if (${CORE} STREQUAL ARMV9SME)
if (NOT DYNAMIC_ARCH)
if (${CMAKE_C_COMPILER_ID} STREQUAL "NVHPC" AND NOT NO_SVE)
set (CCOMMON_OPT "${CCOMMON_OPT} -tp=host")
set (CCOMMON_OPT "${CCOMMON_OPT} -tp=host")
else ()
set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv9-a+sme")
if (${OSNAME} STREQUAL Windows AND ${CMAKE_C_COMPILER_ID} MATCHES "Clang" )
set (CCOMMON_OPT "${CCOMMON_OPT} --aarch64-stack-hazard-size=0")
set (CCOMMON_OPT "${CCOMMON_OPT} -march=armv9-a+sme")
if (CMAKE_SYSTEM_NAME STREQUAL "Windows" AND CMAKE_C_COMPILER_ID MATCHES "Clang")
set (CCOMMON_OPT "${CCOMMON_OPT} --aarch64-stack-hazard-size=0")
endif ()
endif ()
endif ()
endif ()
endif ()

Expand Down
9 changes: 6 additions & 3 deletions interface/trsm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,7 @@ void CNAME(enum CBLAS_ORDER order,

if (args.m == 0 || args.n == 0) return;

#if defined(TRMM)
#if !defined(COMPLEX) && !defined(DOUBLE) && !defined(BFLOAT16) && !defined(HFLOAT16)
#if defined(ARCH_ARM64) && (defined(USE_STRMM_KERNEL_DIRECT)||defined(DYNAMIC_ARCH))
#if defined(DYNAMIC_ARCH)
Expand All @@ -371,15 +372,17 @@ if (strcmp(gotoblas_corename(), "armv9sme") == 0
)
#endif
if (order == CblasRowMajor && Diag == CblasNonUnit && Side == CblasLeft && m == lda && n == ldb) {
if (Trans == CblasNoTrans) {
if (Trans == CblasNoTrans || Trans == CblasConjNoTrans) {
(Uplo == CblasUpper ? STRMM_DIRECT_LNUN : STRMM_DIRECT_LNLN)(m, n, alpha, a, lda, b, ldb);
} else if (Trans == CblasTrans) {
return;
} else if (Trans == CblasTrans || Trans == CblasConjTrans) {
(Uplo == CblasUpper ? STRMM_DIRECT_LTUN : STRMM_DIRECT_LTLN)(m, n, alpha, a, lda, b, ldb);
return;
}
return;
}
#endif
#endif
#endif



Expand Down
23 changes: 11 additions & 12 deletions kernel/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,38 +292,38 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
if (USE_DIRECT_SSYMM)
if (ARM64)
set (SSYMMDIRECTKERNEL_ALPHA_BETA ssymm_direct_alpha_beta_arm64_sme1.c)
GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "" "symm_direct_alpha_betaLU" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "" "symm_direct_alpha_betaLL" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "LEFT;UPPER" "symm_direct_alpha_betaLU" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYMMDIRECTKERNEL_ALPHA_BETA}" "LEFT;LOWER" "symm_direct_alpha_betaLL" false "" "" false SINGLE)
endif ()
endif()

if (USE_DIRECT_STRMM)
if (ARM64)
set (STRMMDIRECTKERNEL strmm_direct_arm64_sme1.c)
GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LNUN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "UPPER" "trmm_direct_LNUN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LNLN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LTUN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "" "trmm_direct_LTLN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "TRANSA;UPPER" "trmm_direct_LTUN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${STRMMDIRECTKERNEL}" "TRANSA" "trmm_direct_LTLN" false "" "" false SINGLE)
endif ()
endif ()

if (USE_DIRECT_SSYRK)
if (ARM64)
set (SSYRKDIRECTKERNEL_ALPHA_BETA ssyrk_direct_alpha_beta_arm64_sme1.c)
GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaUN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaUT" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "UPPER" "syrk_direct_alpha_betaUN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "TRANSA;UPPER" "syrk_direct_alpha_betaUT" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaLN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "" "syrk_direct_alpha_betaLT" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYRKDIRECTKERNEL_ALPHA_BETA}" "TRANSA" "syrk_direct_alpha_betaLT" false "" "" false SINGLE)
endif ()
endif()

if (USE_DIRECT_SSYR2K)
if (ARM64)
set (SSYR2KDIRECTKERNEL_ALPHA_BETA ssyr2k_direct_alpha_beta_arm64_sme1.c)
GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "" "syr2k_direct_alpha_betaUN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "" "syr2k_direct_alpha_betaUT" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "UPPER" "syr2k_direct_alpha_betaUN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "TRANSA;UPPER" "syr2k_direct_alpha_betaUT" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "" "syr2k_direct_alpha_betaLN" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "" "syr2k_direct_alpha_betaLT" false "" "" false SINGLE)
GenerateNamedObjects("${KERNELDIR}/${SSYR2KDIRECTKERNEL_ALPHA_BETA}" "TRANSA" "syr2k_direct_alpha_betaLT" false "" "" false SINGLE)
endif ()
endif()

Expand Down Expand Up @@ -1555,4 +1555,3 @@ else ()
set(KERNEL_DEFINITIONS "")
build_core("${TARGET_CORE}" "${KDIR}" "${TSUFFIX}" "${KERNEL_DEFINITIONS}")
endif ()

Loading
Loading