Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CPU] migrate onednn3.6 #27957

Merged
merged 18 commits into from
Jan 2, 2025
Merged

[CPU] migrate onednn3.6 #27957

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
6f85245
try to migrate onednn3.6
xczhai Dec 6, 2024
b4eabff
fix test
xczhai Dec 9, 2024
7c6843c
fix brgemm f16 on spr
xczhai Dec 12, 2024
8634b4b
rebase and fix conflicts
xczhai Dec 13, 2024
e7b2711
fix coding format
xczhai Dec 13, 2024
402802b
comment out rnn test case
xczhai Dec 17, 2024
c576825
merge arm patch
xczhai Dec 20, 2024
6720e7f
apply and fix
xczhai Dec 24, 2024
7cea12c
Merge branch 'master' into xc/dnn3.6_based_on_master
xczhai Dec 24, 2024
9a903e2
update onednn cpu
xczhai Dec 26, 2024
ce58e91
Merge branch 'master' into xc/dnn3.6_based_on_master
xczhai Dec 26, 2024
c868510
Merge branch 'master' into xc/dnn3.6_based_on_master
xczhai Dec 26, 2024
6abd030
Merge branch 'master' into xc/dnn3.6_based_on_master
xczhai Dec 27, 2024
7ac8aa4
skip win test for known bug
xczhai Dec 28, 2024
6526b43
Merge branch 'master' into xc/dnn3.6_based_on_master
xczhai Dec 28, 2024
4bc2b92
update onednn
xczhai Dec 31, 2024
ac512e1
Merge branch 'master' into xc/dnn3.6_based_on_master
xczhai Jan 1, 2025
fb13afa
revert a commit of onednn
xczhai Jan 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions src/plugins/intel_cpu/src/emitters/plugin/x64/jit_dnnl_emitters.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,18 @@ jit_dnnl_emitter::jit_dnnl_emitter(jit_generator* host,
void jit_dnnl_emitter::set_injector() {
if (host_isa_ == cpu::x64::sse41) {
eltwise_injector_sse42 =
std::make_shared<jit_uni_eltwise_injector_f32<cpu::x64::sse41>>(h, kind, alpha, beta, 1.f);
std::make_shared<jit_uni_eltwise_injector<cpu::x64::sse41>>(h, kind, alpha, beta, 1.f, data_type::f32);
} else if (host_isa_ == cpu::x64::avx2) {
eltwise_injector_avx2 =
std::make_shared<jit_uni_eltwise_injector_f32<cpu::x64::avx2>>(h, kind, alpha, beta, 1.f);
std::make_shared<jit_uni_eltwise_injector<cpu::x64::avx2>>(h, kind, alpha, beta, 1.f, data_type::f32);
} else if (host_isa_ == cpu::x64::avx512_core) {
eltwise_injector_avx512_core =
std::make_shared<jit_uni_eltwise_injector_f32<cpu::x64::avx512_core>>(h, kind, alpha, beta, 1.f);
std::make_shared<jit_uni_eltwise_injector<cpu::x64::avx512_core>>(h,
kind,
alpha,
beta,
1.f,
data_type::f32);
} else {
OV_CPU_JIT_EMITTER_THROW("Unsupported ISA ", host_isa_);
}
Expand Down
8 changes: 3 additions & 5 deletions src/plugins/intel_cpu/src/emitters/plugin/x64/jit_dnnl_emitters.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,9 @@ class jit_dnnl_emitter : public jit_emitter {
float alpha{0.f};
float beta{0.f};

std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector_f32<dnnl::impl::cpu::x64::sse41>>
eltwise_injector_sse42;
std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector_f32<dnnl::impl::cpu::x64::avx2>>
eltwise_injector_avx2;
std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector_f32<dnnl::impl::cpu::x64::avx512_core>>
std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector<dnnl::impl::cpu::x64::sse41>> eltwise_injector_sse42;
std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector<dnnl::impl::cpu::x64::avx2>> eltwise_injector_avx2;
std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector<dnnl::impl::cpu::x64::avx512_core>>
eltwise_injector_avx512_core;

private:
Expand Down
13 changes: 7 additions & 6 deletions src/plugins/intel_cpu/src/nodes/bin_conv.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,12 @@ struct jit_uni_bin_conv_kernel_f32 : public jit_uni_bin_conv_kernel, public jit_
for (int i = 0; i < end_idx; i++) {
auto& post_op = p.entry_[i];
if (post_op.is_eltwise()) {
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this,
post_op.eltwise,
true,
eltwise_reserved,
mask_post_op_reserved));
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector<isa>>(this,
post_op.eltwise,
data_type::f32,
true,
eltwise_reserved,
mask_post_op_reserved));
} else if (post_op.is_depthwise()) {
depthwise_injectors.push_back(
std::make_shared<jit_uni_depthwise_injector_f32<isa>>(this, post_op, mask_post_op_reserved));
Expand Down Expand Up @@ -217,7 +218,7 @@ struct jit_uni_bin_conv_kernel_f32 : public jit_uni_bin_conv_kernel, public jit_

Xbyak::Label l_table;

nstl::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
nstl::vector<std::shared_ptr<jit_uni_eltwise_injector<isa>>> eltwise_injectors;
nstl::vector<std::shared_ptr<jit_uni_depthwise_injector_f32<isa>>> depthwise_injectors;

void cvt2ps(dnnl::memory::data_type type_in, Vmm vmm_in, const Xbyak::Operand& op, bool scalar_load) {
Expand Down
5 changes: 3 additions & 2 deletions src/plugins/intel_cpu/src/nodes/common/softmax.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "utils/bfloat16.hpp"

using namespace dnnl;
using namespace dnnl::impl;
using namespace dnnl::impl::cpu;
using namespace dnnl::impl::cpu::x64;
using namespace dnnl::impl::utils;
Expand Down Expand Up @@ -68,7 +69,7 @@ struct jit_uni_softmax_kernel_f32 : public jit_uni_softmax_kernel, public jit_ge

void generate() override {
exp_injector.reset(
new jit_uni_eltwise_injector_f32<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.0f));
new jit_uni_eltwise_injector<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.0f, data_type::f32));

if (mayiuse(avx512_core))
uni_vcvtneps2bf16.reset(new jit_uni_vcvtneps2bf16(this, isa));
Expand Down Expand Up @@ -200,7 +201,7 @@ struct jit_uni_softmax_kernel_f32 : public jit_uni_softmax_kernel, public jit_ge

std::unique_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16;

std::shared_ptr<jit_uni_eltwise_injector_f32<isa>> exp_injector;
std::shared_ptr<jit_uni_eltwise_injector<isa>> exp_injector;

jit_softmax_config_params jcp_;

Expand Down
13 changes: 7 additions & 6 deletions src/plugins/intel_cpu/src/nodes/interpolate.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,12 @@ struct jit_uni_interpolate_kernel_f32 : public jit_uni_interpolate_kernel, publi
for (int i = 0; i < p.len(); i++) {
auto& post_op = p.entry_[i];
if (post_op.is_eltwise()) {
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this,
post_op.eltwise.alg,
post_op.eltwise.alpha,
post_op.eltwise.beta,
1.f));
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector<isa>>(this,
post_op.eltwise.alg,
post_op.eltwise.alpha,
post_op.eltwise.beta,
1.f,
data_type::f32));
} else if (post_op.is_depthwise()) {
depthwise_injectors.push_back(std::make_shared<jit_uni_depthwise_injector_f32<isa>>(this, post_op));
} else if (post_op.is_quantization()) {
Expand Down Expand Up @@ -275,7 +276,7 @@ struct jit_uni_interpolate_kernel_f32 : public jit_uni_interpolate_kernel, publi
std::vector<size_t> store_pool_vec_idxs;
std::vector<size_t> load_pool_gpr_idxs;

std::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
std::vector<std::shared_ptr<jit_uni_eltwise_injector<isa>>> eltwise_injectors;
std::vector<std::shared_ptr<jit_uni_depthwise_injector_f32<isa>>> depthwise_injectors;
std::vector<std::shared_ptr<jit_uni_quantization_injector_f32<isa>>> quantization_injectors;

Expand Down
2 changes: 1 addition & 1 deletion src/plugins/intel_cpu/src/nodes/kernels/x64/brgemm_kernel.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ void BrgemmKernel::init_brgemm_copy_b(
brgemm_matmul_conf_t brgCopyKernelConf;
brgCopyKernelConf.src_dt = is_avx_f16_only ? dnnl_data_type_t::dnnl_f32 : dt_in0;
brgCopyKernelConf.wei_dt = is_avx_f16_only ? dnnl_data_type_t::dnnl_f32 : dt_in1;
brgCopyKernelConf.orig_wei_dt = dt_in1;
brgCopyKernelConf.orig_wei_dt = static_cast<dnnl_data_type_t>(DnnlExtensionUtils::ElementTypeToDataType(inType));
brgCopyKernelConf.wei_n_blk = N_blk;
brgCopyKernelConf.wei_tag = transpose ? dnnl_ba : dnnl_ab;
brgCopyKernelConf.copy_B_wei_stride = copy_B_wei_stride;
Expand Down
3 changes: 2 additions & 1 deletion src/plugins/intel_cpu/src/nodes/kernels/x64/mlp_kernel.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,12 +550,13 @@ void GateUpCombine::generate() {
const auto zmm_up = zmm0;
const auto ymm_dst = ymm5;

auto injector = std::make_shared<jit_uni_eltwise_injector_f32<dnnl::impl::cpu::x64::avx512_core>>(
auto injector = std::make_shared<jit_uni_eltwise_injector<dnnl::impl::cpu::x64::avx512_core>>(
this,
m_act_alg,
1.f,
1.0f,
1.f,
data_type::f32,
true, // save_state, true due to additional r15 is used.
Xbyak::Reg64(Xbyak::Operand::R10), // p_table
Xbyak::Opmask(1), // k_mask
Expand Down
3 changes: 2 additions & 1 deletion src/plugins/intel_cpu/src/nodes/kernels/x64/non_max_suppression.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

#include "utils/general_utils.h"

using namespace dnnl::impl;
using namespace dnnl::impl::cpu;

#define GET_OFF(field) offsetof(NmsCallArgs, field)
Expand All @@ -20,7 +21,7 @@ void NonMaxSuppression<isa>::generate() {
load_scalar_emitter.reset(new jit_load_emitter(this, isa, ov::element::f32, ov::element::f32, scalar_step));

exp_injector.reset(
new x64::jit_uni_eltwise_injector_f32<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.f));
new x64::jit_uni_eltwise_injector<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.f, data_type::f32));

this->preamble();

Expand Down
2 changes: 1 addition & 1 deletion src/plugins/intel_cpu/src/nodes/kernels/x64/non_max_suppression.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ class NonMaxSuppression : public JitKernel<NmsCompileParams, NmsCallArgs> {
Xbyak::Opmask k_mask = Xbyak::Opmask(7);
Xbyak::Opmask k_mask_one = Xbyak::Opmask(6);

std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector_f32<isa>> exp_injector;
std::shared_ptr<dnnl::impl::cpu::x64::jit_uni_eltwise_injector<isa>> exp_injector;

inline void hard_nms();

Expand Down
13 changes: 7 additions & 6 deletions src/plugins/intel_cpu/src/nodes/mvn.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -949,11 +949,12 @@ struct jit_uni_mvn_kernel_f32 : public jit_uni_mvn_kernel, public jit_generator
for (int i = 0; i < p.len(); i++) {
auto& post_op = p.entry_[i];
if (post_op.is_eltwise()) {
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this,
post_op.eltwise.alg,
post_op.eltwise.alpha,
post_op.eltwise.beta,
post_op.eltwise.scale));
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector<isa>>(this,
post_op.eltwise.alg,
post_op.eltwise.alpha,
post_op.eltwise.beta,
post_op.eltwise.scale,
data_type::f32));
} else if (post_op.is_depthwise()) {
depthwise_injectors.push_back(std::make_shared<jit_uni_depthwise_injector_f32<isa>>(this, post_op));
} else if (post_op.is_quantization()) {
Expand Down Expand Up @@ -1093,7 +1094,7 @@ struct jit_uni_mvn_kernel_f32 : public jit_uni_mvn_kernel, public jit_generator

const int tile_size[kTileNum] = {8, 4, 2, 1};

std::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
std::vector<std::shared_ptr<jit_uni_eltwise_injector<isa>>> eltwise_injectors;
std::vector<std::shared_ptr<jit_uni_depthwise_injector_f32<isa>>> depthwise_injectors;
std::vector<std::shared_ptr<jit_uni_quantization_injector_f32<isa>>> quantization_injectors;

Expand Down
13 changes: 7 additions & 6 deletions src/plugins/intel_cpu/src/nodes/normalize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,11 +224,12 @@ struct jit_uni_normalize_kernel_f32 : public jit_uni_normalize_kernel, public ji
for (int i = 0; i < p.len(); i++) {
auto& post_op = p.entry_[i];
if (post_op.is_eltwise()) {
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this,
post_op.eltwise.alg,
post_op.eltwise.alpha,
post_op.eltwise.beta,
post_op.eltwise.scale));
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector<isa>>(this,
post_op.eltwise.alg,
post_op.eltwise.alpha,
post_op.eltwise.beta,
post_op.eltwise.scale,
data_type::f32));
} else if (post_op.is_depthwise()) {
depthwise_injectors.push_back(std::make_shared<jit_uni_depthwise_injector_f32<isa>>(this, post_op));
} else if (post_op.is_quantization()) {
Expand Down Expand Up @@ -310,7 +311,7 @@ struct jit_uni_normalize_kernel_f32 : public jit_uni_normalize_kernel, public ji

std::unique_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16 = nullptr;

std::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
std::vector<std::shared_ptr<jit_uni_eltwise_injector<isa>>> eltwise_injectors;
std::vector<std::shared_ptr<jit_uni_depthwise_injector_f32<isa>>> depthwise_injectors;
std::vector<std::shared_ptr<jit_uni_quantization_injector_f32<isa>>> quantization_injectors;

Expand Down
33 changes: 21 additions & 12 deletions src/plugins/intel_cpu/src/nodes/reduce.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,12 @@ struct jit_uni_reduce_kernel_f32 : public jit_uni_reduce_kernel, public jit_gene

void generate() override {
if (jcp_.reduce_mode == Algorithm::ReduceLogSumExp) {
exp_injector =
std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this, alg_kind::eltwise_exp, 0.f, 0.f, 1.f);
exp_injector = std::make_shared<jit_uni_eltwise_injector<isa>>(this,
alg_kind::eltwise_exp,
0.f,
0.f,
1.f,
data_type::f32);
}

if (mayiuse(avx512_core))
Expand Down Expand Up @@ -244,7 +248,7 @@ struct jit_uni_reduce_kernel_f32 : public jit_uni_reduce_kernel, public jit_gene
Xbyak::Label l_table;

std::shared_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16;
std::shared_ptr<jit_uni_eltwise_injector_f32<isa>> exp_injector;
std::shared_ptr<jit_uni_eltwise_injector<isa>> exp_injector;

inline void reduce_main() {
// ================================================================
Expand Down Expand Up @@ -1206,11 +1210,12 @@ struct jit_uni_reduce_post_kernel_f32 : public jit_uni_reduce_post_kernel, publi
for (int i = 0; i < p.len(); i++) {
auto& post_op = p.entry_[i];
if (post_op.is_eltwise()) {
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this,
post_op.eltwise.alg,
post_op.eltwise.alpha,
post_op.eltwise.beta,
post_op.eltwise.scale));
eltwise_injectors.push_back(std::make_shared<jit_uni_eltwise_injector<isa>>(this,
post_op.eltwise.alg,
post_op.eltwise.alpha,
post_op.eltwise.beta,
post_op.eltwise.scale,
data_type::f32));
} else if (post_op.is_depthwise()) {
depthwise_injectors.push_back(std::make_shared<jit_uni_depthwise_injector_f32<isa>>(this, post_op));
} else if (post_op.is_quantization()) {
Expand All @@ -1224,8 +1229,12 @@ struct jit_uni_reduce_post_kernel_f32 : public jit_uni_reduce_post_kernel, publi
}

if (jcp_.reduce_mode == Algorithm::ReduceLogSum || jcp_.reduce_mode == Algorithm::ReduceLogSumExp) {
log_injector =
std::make_shared<jit_uni_eltwise_injector_f32<isa>>(this, alg_kind::eltwise_log, 0.f, 0.f, 1.f);
log_injector = std::make_shared<jit_uni_eltwise_injector<isa>>(this,
alg_kind::eltwise_log,
0.f,
0.f,
1.f,
data_type::f32);
}

if (mayiuse(avx512_core))
Expand Down Expand Up @@ -1336,9 +1345,9 @@ struct jit_uni_reduce_post_kernel_f32 : public jit_uni_reduce_post_kernel, publi
Vmm vmm_d_bias = Vmm(8);

std::shared_ptr<jit_uni_vcvtneps2bf16> uni_vcvtneps2bf16;
std::shared_ptr<jit_uni_eltwise_injector_f32<isa>> log_injector;
std::shared_ptr<jit_uni_eltwise_injector<isa>> log_injector;

std::vector<std::shared_ptr<jit_uni_eltwise_injector_f32<isa>>> eltwise_injectors;
std::vector<std::shared_ptr<jit_uni_eltwise_injector<isa>>> eltwise_injectors;
std::vector<std::shared_ptr<jit_uni_depthwise_injector_f32<isa>>> depthwise_injectors;
std::vector<std::shared_ptr<jit_uni_quantization_injector_f32<isa>>> quantization_injectors;

Expand Down
5 changes: 3 additions & 2 deletions src/plugins/intel_cpu/src/nodes/region_yolo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "openvino/opsets/opset1.hpp"
#include "utils/bfloat16.hpp"

using namespace dnnl::impl;
using namespace dnnl::impl::cpu;
using namespace dnnl::impl::cpu::x64;
using namespace dnnl::impl::utils;
Expand Down Expand Up @@ -46,7 +47,7 @@ struct jit_uni_logistic_kernel_f32 : public jit_uni_logistic_kernel, public jit_

void generate() override {
exp_injector.reset(
new jit_uni_eltwise_injector_f32<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.f));
new jit_uni_eltwise_injector<isa>(this, dnnl::impl::alg_kind::eltwise_exp, 0.f, 0.f, 1.f, data_type::f32));

if (mayiuse(avx512_core))
uni_vcvtneps2bf16.reset(new jit_uni_vcvtneps2bf16(this, isa));
Expand Down Expand Up @@ -134,7 +135,7 @@ struct jit_uni_logistic_kernel_f32 : public jit_uni_logistic_kernel, public jit_

Xbyak::Label l_table;

std::shared_ptr<jit_uni_eltwise_injector_f32<isa>> exp_injector;
std::shared_ptr<jit_uni_eltwise_injector<isa>> exp_injector;

jit_logistic_config_params jcp_;

Expand Down
Loading
Loading