10#ifndef EIGEN_COMPLEX_AVX512_H
11#define EIGEN_COMPLEX_AVX512_H
21 EIGEN_STRONG_INLINE
explicit Packet8cf(
const __m512& a) : v(a) {}
50 typedef std::complex<float> type;
52 typedef Packet16f as_real;
57 masked_load_available=
false,
58 masked_store_available=
false
63template<> EIGEN_STRONG_INLINE Packet8cf padd<Packet8cf>(
const Packet8cf& a,
const Packet8cf& b) {
return Packet8cf(_mm512_add_ps(a.v,b.v)); }
64template<> EIGEN_STRONG_INLINE Packet8cf psub<Packet8cf>(
const Packet8cf& a,
const Packet8cf& b) {
return Packet8cf(_mm512_sub_ps(a.v,b.v)); }
65template<> EIGEN_STRONG_INLINE Packet8cf pnegate(
const Packet8cf& a)
67 return Packet8cf(pnegate(a.v));
69template<> EIGEN_STRONG_INLINE Packet8cf pconj(
const Packet8cf& a)
71 const __m512 mask = _mm512_castsi512_ps(_mm512_setr_epi32(
72 0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,
73 0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000,0x00000000,0x80000000));
74 return Packet8cf(pxor(a.v,mask));
77template<> EIGEN_STRONG_INLINE Packet8cf pmul<Packet8cf>(
const Packet8cf& a,
const Packet8cf& b)
79 __m512 tmp2 = _mm512_mul_ps(_mm512_movehdup_ps(a.v), _mm512_permute_ps(b.v, _MM_SHUFFLE(2,3,0,1)));
80 return Packet8cf(_mm512_fmaddsub_ps(_mm512_moveldup_ps(a.v), b.v, tmp2));
83template<> EIGEN_STRONG_INLINE Packet8cf pand <Packet8cf>(
const Packet8cf& a,
const Packet8cf& b) {
return Packet8cf(pand(a.v,b.v)); }
84template<> EIGEN_STRONG_INLINE Packet8cf por <Packet8cf>(
const Packet8cf& a,
const Packet8cf& b) {
return Packet8cf(por(a.v,b.v)); }
85template<> EIGEN_STRONG_INLINE Packet8cf pxor <Packet8cf>(
const Packet8cf& a,
const Packet8cf& b) {
return Packet8cf(pxor(a.v,b.v)); }
86template<> EIGEN_STRONG_INLINE Packet8cf pandnot<Packet8cf>(
const Packet8cf& a,
const Packet8cf& b) {
return Packet8cf(pandnot(a.v,b.v)); }
89EIGEN_STRONG_INLINE Packet8cf pcmp_eq(
const Packet8cf& a,
const Packet8cf& b) {
90 __m512 eq = pcmp_eq<Packet16f>(a.v, b.v);
91 return Packet8cf(pand(eq, _mm512_permute_ps(eq, 0xB1)));
94template<> EIGEN_STRONG_INLINE Packet8cf pload <Packet8cf>(
const std::complex<float>* from) { EIGEN_DEBUG_ALIGNED_LOAD
return Packet8cf(pload<Packet16f>(&numext::real_ref(*from))); }
95template<> EIGEN_STRONG_INLINE Packet8cf ploadu<Packet8cf>(
const std::complex<float>* from) { EIGEN_DEBUG_UNALIGNED_LOAD
return Packet8cf(ploadu<Packet16f>(&numext::real_ref(*from))); }
98template<> EIGEN_STRONG_INLINE Packet8cf pset1<Packet8cf>(
const std::complex<float>& from)
100 const float re = std::real(from);
101 const float im = std::imag(from);
102 return Packet8cf(_mm512_set_ps(im, re, im, re, im, re, im, re, im, re, im, re, im, re, im, re));
105template<> EIGEN_STRONG_INLINE Packet8cf ploaddup<Packet8cf>(
const std::complex<float>* from)
107 return Packet8cf( _mm512_castpd_ps( ploaddup<Packet8d>((
const double*)(
const void*)from )) );
109template<> EIGEN_STRONG_INLINE Packet8cf ploadquad<Packet8cf>(
const std::complex<float>* from)
111 return Packet8cf( _mm512_castpd_ps( ploadquad<Packet8d>((
const double*)(
const void*)from )) );
114template<> EIGEN_STRONG_INLINE
void pstore <std::complex<float> >(std::complex<float>* to,
const Packet8cf& from) { EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v); }
115template<> EIGEN_STRONG_INLINE
void pstoreu<std::complex<float> >(std::complex<float>* to,
const Packet8cf& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v); }
117template<> EIGEN_DEVICE_FUNC
inline Packet8cf pgather<std::complex<float>, Packet8cf>(
const std::complex<float>* from,
Index stride)
119 return Packet8cf(_mm512_castpd_ps(pgather<double,Packet8d>((
const double*)(
const void*)from, stride)));
122template<> EIGEN_DEVICE_FUNC
inline void pscatter<std::complex<float>, Packet8cf>(std::complex<float>* to,
const Packet8cf& from,
Index stride)
124 pscatter((
double*)(
void*)to, _mm512_castps_pd(from.v), stride);
127template<> EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet8cf>(
const Packet8cf& a)
129 return pfirst(Packet2cf(_mm512_castps512_ps128(a.v)));
132template<> EIGEN_STRONG_INLINE Packet8cf preverse(
const Packet8cf& a) {
133 return Packet8cf(_mm512_castsi512_ps(
134 _mm512_permutexvar_epi64( _mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7),
135 _mm512_castps_si512(a.v))));
138template<> EIGEN_STRONG_INLINE std::complex<float> predux<Packet8cf>(
const Packet8cf& a)
140 return predux(padd(Packet4cf(extract256<0>(a.v)),
141 Packet4cf(extract256<1>(a.v))));
144template<> EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet8cf>(
const Packet8cf& a)
146 return predux_mul(pmul(Packet4cf(extract256<0>(a.v)),
147 Packet4cf(extract256<1>(a.v))));
151EIGEN_STRONG_INLINE Packet4cf predux_half_dowto4<Packet8cf>(
const Packet8cf& a) {
152 __m256 lane0 = extract256<0>(a.v);
153 __m256 lane1 = extract256<1>(a.v);
154 __m256 res = _mm256_add_ps(lane0, lane1);
155 return Packet4cf(res);
158EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet8cf,Packet16f)
160template<> EIGEN_STRONG_INLINE Packet8cf pdiv<Packet8cf>(
const Packet8cf& a,
const Packet8cf& b)
162 Packet8cf num = pmul(a, pconj(b));
163 __m512 tmp = _mm512_mul_ps(b.v, b.v);
164 __m512 tmp2 = _mm512_shuffle_ps(tmp,tmp,0xB1);
165 __m512 denom = _mm512_add_ps(tmp, tmp2);
166 return Packet8cf(_mm512_div_ps(num.v, denom));
169template<> EIGEN_STRONG_INLINE Packet8cf pcplxflip<Packet8cf>(
const Packet8cf& x)
171 return Packet8cf(_mm512_shuffle_ps(x.v, x.v, _MM_SHUFFLE(2, 3, 0 ,1)));
178 EIGEN_STRONG_INLINE
explicit Packet4cd(
const __m512d& a) : v(a) {}
207 typedef std::complex<double> type;
209 typedef Packet8d as_real;
214 masked_load_available=
false,
215 masked_store_available=
false
220template<> EIGEN_STRONG_INLINE Packet4cd psub<Packet4cd>(
const Packet4cd& a,
const Packet4cd& b) {
return Packet4cd(_mm512_sub_pd(a.v,b.v)); }
221template<> EIGEN_STRONG_INLINE Packet4cd pnegate(
const Packet4cd& a) {
return Packet4cd(pnegate(a.v)); }
222template<> EIGEN_STRONG_INLINE Packet4cd pconj(
const Packet4cd& a)
224 const __m512d mask = _mm512_castsi512_pd(
225 _mm512_set_epi32(0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0,
226 0x80000000,0x0,0x0,0x0,0x80000000,0x0,0x0,0x0));
227 return Packet4cd(pxor(a.v,mask));
230template<> EIGEN_STRONG_INLINE Packet4cd pmul<Packet4cd>(
const Packet4cd& a,
const Packet4cd& b)
232 __m512d tmp1 = _mm512_shuffle_pd(a.v,a.v,0x0);
233 __m512d tmp2 = _mm512_shuffle_pd(a.v,a.v,0xFF);
234 __m512d tmp3 = _mm512_shuffle_pd(b.v,b.v,0x55);
235 __m512d odd = _mm512_mul_pd(tmp2, tmp3);
236 return Packet4cd(_mm512_fmaddsub_pd(tmp1, b.v, odd));
239template<> EIGEN_STRONG_INLINE Packet4cd ptrue<Packet4cd>(
const Packet4cd& a) {
return Packet4cd(ptrue(Packet8d(a.v))); }
240template<> EIGEN_STRONG_INLINE Packet4cd pand <Packet4cd>(
const Packet4cd& a,
const Packet4cd& b) {
return Packet4cd(pand(a.v,b.v)); }
241template<> EIGEN_STRONG_INLINE Packet4cd por <Packet4cd>(
const Packet4cd& a,
const Packet4cd& b) {
return Packet4cd(por(a.v,b.v)); }
242template<> EIGEN_STRONG_INLINE Packet4cd pxor <Packet4cd>(
const Packet4cd& a,
const Packet4cd& b) {
return Packet4cd(pxor(a.v,b.v)); }
243template<> EIGEN_STRONG_INLINE Packet4cd pandnot<Packet4cd>(
const Packet4cd& a,
const Packet4cd& b) {
return Packet4cd(pandnot(a.v,b.v)); }
246EIGEN_STRONG_INLINE Packet4cd pcmp_eq(
const Packet4cd& a,
const Packet4cd& b) {
247 __m512d eq = pcmp_eq<Packet8d>(a.v, b.v);
248 return Packet4cd(pand(eq, _mm512_permute_pd(eq, 0x55)));
251template<> EIGEN_STRONG_INLINE Packet4cd pload <Packet4cd>(
const std::complex<double>* from)
252{ EIGEN_DEBUG_ALIGNED_LOAD
return Packet4cd(pload<Packet8d>((
const double*)from)); }
253template<> EIGEN_STRONG_INLINE Packet4cd ploadu<Packet4cd>(
const std::complex<double>* from)
254{ EIGEN_DEBUG_UNALIGNED_LOAD
return Packet4cd(ploadu<Packet8d>((
const double*)from)); }
256template<> EIGEN_STRONG_INLINE Packet4cd pset1<Packet4cd>(
const std::complex<double>& from)
258 #ifdef EIGEN_VECTORIZE_AVX512DQ
259 return Packet4cd(_mm512_broadcast_f64x2(pset1<Packet1cd>(from).v));
261 return Packet4cd(_mm512_castps_pd(_mm512_broadcast_f32x4( _mm_castpd_ps(pset1<Packet1cd>(from).v))));
265template<> EIGEN_STRONG_INLINE Packet4cd ploaddup<Packet4cd>(
const std::complex<double>* from) {
266 return Packet4cd(_mm512_insertf64x4(
267 _mm512_castpd256_pd512(ploaddup<Packet2cd>(from).v), ploaddup<Packet2cd>(from+1).v, 1));
270template<> EIGEN_STRONG_INLINE
void pstore <std::complex<double> >(std::complex<double> * to,
const Packet4cd& from) { EIGEN_DEBUG_ALIGNED_STORE pstore((
double*)to, from.v); }
271template<> EIGEN_STRONG_INLINE
void pstoreu<std::complex<double> >(std::complex<double> * to,
const Packet4cd& from) { EIGEN_DEBUG_UNALIGNED_STORE pstoreu((
double*)to, from.v); }
273template<> EIGEN_DEVICE_FUNC
inline Packet4cd pgather<std::complex<double>, Packet4cd>(
const std::complex<double>* from,
Index stride)
275 return Packet4cd(_mm512_insertf64x4(_mm512_castpd256_pd512(
276 _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from+0*stride).v), ploadu<Packet1cd>(from+1*stride).v,1)),
277 _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from+2*stride).v), ploadu<Packet1cd>(from+3*stride).v,1), 1));
280template<> EIGEN_DEVICE_FUNC
inline void pscatter<std::complex<double>, Packet4cd>(std::complex<double>* to,
const Packet4cd& from,
Index stride)
282 __m512i fromi = _mm512_castpd_si512(from.v);
283 double* tod = (
double*)(
void*)to;
284 _mm_storeu_pd(tod+0*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,0)) );
285 _mm_storeu_pd(tod+2*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,1)) );
286 _mm_storeu_pd(tod+4*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,2)) );
287 _mm_storeu_pd(tod+6*stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi,3)) );
290template<> EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet4cd>(
const Packet4cd& a)
292 __m128d low = extract128<0>(a.v);
293 EIGEN_ALIGN16
double res[2];
294 _mm_store_pd(res, low);
295 return std::complex<double>(res[0],res[1]);
298template<> EIGEN_STRONG_INLINE Packet4cd preverse(
const Packet4cd& a) {
299 return Packet4cd(_mm512_shuffle_f64x2(a.v, a.v, (shuffle_mask<3,2,1,0>::mask)));
302template<> EIGEN_STRONG_INLINE std::complex<double> predux<Packet4cd>(
const Packet4cd& a)
304 return predux(padd(Packet2cd(_mm512_extractf64x4_pd(a.v,0)),
305 Packet2cd(_mm512_extractf64x4_pd(a.v,1))));
308template<> EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet4cd>(
const Packet4cd& a)
310 return predux_mul(pmul(Packet2cd(_mm512_extractf64x4_pd(a.v,0)),
311 Packet2cd(_mm512_extractf64x4_pd(a.v,1))));
317 {
return padd(pmul(x,y),c); }
321 return internal::pmul(a, pconj(b));
328 {
return padd(pmul(x,y),c); }
332 return internal::pmul(pconj(a), b);
339 {
return padd(pmul(x,y),c); }
343 return pconj(internal::pmul(a, b));
347EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(
Packet4cd,Packet8d)
352 __m512d tmp = _mm512_mul_pd(b.v, b.v);
353 __m512d denom = padd(_mm512_permute_pd(tmp,0x55), tmp);
354 return Packet4cd(_mm512_div_pd(num.v, denom));
357template<> EIGEN_STRONG_INLINE Packet4cd pcplxflip<Packet4cd>(
const Packet4cd& x)
359 return Packet4cd(_mm512_permute_pd(x.v,0x55));
362EIGEN_DEVICE_FUNC
inline void
363ptranspose(PacketBlock<Packet8cf,4>& kernel) {
364 PacketBlock<Packet8d,4> pb;
366 pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
367 pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
368 pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
369 pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
371 kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
372 kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
373 kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
374 kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
377EIGEN_DEVICE_FUNC
inline void
378ptranspose(PacketBlock<Packet8cf,8>& kernel) {
379 PacketBlock<Packet8d,8> pb;
381 pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
382 pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
383 pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
384 pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
385 pb.packet[4] = _mm512_castps_pd(kernel.packet[4].v);
386 pb.packet[5] = _mm512_castps_pd(kernel.packet[5].v);
387 pb.packet[6] = _mm512_castps_pd(kernel.packet[6].v);
388 pb.packet[7] = _mm512_castps_pd(kernel.packet[7].v);
390 kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
391 kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
392 kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
393 kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
394 kernel.packet[4].v = _mm512_castpd_ps(pb.packet[4]);
395 kernel.packet[5].v = _mm512_castpd_ps(pb.packet[5]);
396 kernel.packet[6].v = _mm512_castpd_ps(pb.packet[6]);
397 kernel.packet[7].v = _mm512_castpd_ps(pb.packet[7]);
400EIGEN_DEVICE_FUNC
inline void
401ptranspose(PacketBlock<Packet4cd,4>& kernel) {
402 __m512d T0 = _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<0,1,0,1>::mask));
403 __m512d T1 = _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<2,3,2,3>::mask));
404 __m512d T2 = _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<0,1,0,1>::mask));
405 __m512d T3 = _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<2,3,2,3>::mask));
407 kernel.packet[3] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<1,3,1,3>::mask)));
408 kernel.packet[2] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<0,2,0,2>::mask)));
409 kernel.packet[1] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<1,3,1,3>::mask)));
410 kernel.packet[0] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<0,2,0,2>::mask)));
413template<> EIGEN_STRONG_INLINE Packet4cd psqrt<Packet4cd>(
const Packet4cd& a) {
414 return psqrt_complex<Packet4cd>(a);
417template<> EIGEN_STRONG_INLINE Packet8cf psqrt<Packet8cf>(
const Packet8cf& a) {
418 return psqrt_complex<Packet8cf>(a);
Namespace containing all symbols from the Eigen library.
Definition common.h:81
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:74
Definition ConjHelper.h:63
Definition GenericPacketMath.h:43
Definition GenericPacketMath.h:107
Definition GenericPacketMath.h:133
Definition datatypes.h:12