Skip to content

Commit 550d516

Browse files
eddelbuetteleddelbuettel
committed
Update local diff to contain both Eigen upstream commits
1 parent 3959c8b commit 550d516

1 file changed

Lines changed: 208 additions & 4 deletions

File tree

‎patches/eigen-5.0.1-stan.diff‎

Lines changed: 208 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
diff --git o/inst/include/Eigen/src/Core/Fill.h w/inst/include/Eigen/src/Core/Fill.h
1+
diff --git c/inst/include/Eigen/src/Core/Fill.h w/inst/include/Eigen/src/Core/Fill.h
22
index 779ef26..6f45935 100644
3-
--- o/inst/include/Eigen/src/Core/Fill.h
3+
--- c/inst/include/Eigen/src/Core/Fill.h
44
+++ w/inst/include/Eigen/src/Core/Fill.h
55
@@ -93,8 +93,10 @@ struct eigen_fill_impl<Xpr, /*use_fill*/ true> {
6-
6+
77
template <typename Xpr>
88
struct eigen_memset_helper {
99
- static constexpr bool value =
@@ -13,5 +13,209 @@ index 779ef26..6f45935 100644
1313
+ !static_cast<bool>(NumTraits<Scalar>::RequireInitialization) &&
1414
+ eigen_fill_helper<Xpr>::value;
1515
};
16-
16+
1717
template <typename Xpr>
18+
diff --git c/inst/include/Eigen/src/Core/arch/AVX/TypeCasting.h w/inst/include/Eigen/src/Core/arch/AVX/TypeCasting.h
19+
index 767e2d5..9feb38f 100644
20+
--- c/inst/include/Eigen/src/Core/arch/AVX/TypeCasting.h
21+
+++ w/inst/include/Eigen/src/Core/arch/AVX/TypeCasting.h
22+
@@ -56,6 +56,40 @@ struct type_casting_traits<int64_t, double> : vectorized_type_casting_traits<int
23+
#endif
24+
#endif
25+
26+
+EIGEN_STRONG_INLINE __m256 _eigen_mm256_set_m128(__m128 hi, __m128 lo) {
27+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 810)
28+
+ __m256 result = _mm256_castps128_ps256(lo);
29+
+ return _mm256_insertf128_ps(result, hi, 1);
30+
+#else
31+
+ return _mm256_set_m128(hi, lo);
32+
+#endif
33+
+}
34+
+
35+
+EIGEN_STRONG_INLINE __m256d _eigen_mm256_set_m128d(__m128d hi, __m128d lo) {
36+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 810)
37+
+ __m256d result = _mm256_castpd128_pd256(lo);
38+
+ return _mm256_insertf128_pd(result, hi, 1);
39+
+#else
40+
+ return _mm256_set_m128d(hi, lo);
41+
+#endif
42+
+}
43+
+
44+
+EIGEN_STRONG_INLINE __m256i _eigen_mm256_set_m128i(__m128i hi, __m128i lo) {
45+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 810)
46+
+#if defined(EIGEN_VECTORIZE_AVX2)
47+
+ __m256i result = _mm256_castsi128_si256(lo);
48+
+ return _mm256_inserti128_si256(result, hi, 1);
49+
+#else
50+
+ EIGEN_ALIGN32 int32_t tmp[8];
51+
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(tmp), lo);
52+
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(tmp + 4), hi);
53+
+ return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(tmp));
54+
+#endif
55+
+#else
56+
+ return _mm256_set_m128i(hi, lo);
57+
+#endif
58+
+}
59+
+
60+
template <>
61+
EIGEN_STRONG_INLINE Packet16b pcast<Packet8f, Packet16b>(const Packet8f& a, const Packet8f& b) {
62+
__m256 nonzero_a = _mm256_cmp_ps(a, pzero(a), _CMP_NEQ_UQ);
63+
@@ -109,7 +143,7 @@ EIGEN_STRONG_INLINE Packet8i pcast<Packet8f, Packet8i>(const Packet8f& a) {
64+
65+
template <>
66+
EIGEN_STRONG_INLINE Packet8i pcast<Packet4d, Packet8i>(const Packet4d& a, const Packet4d& b) {
67+
- return _mm256_set_m128i(_mm256_cvttpd_epi32(b), _mm256_cvttpd_epi32(a));
68+
+ return _eigen_mm256_set_m128i(_mm256_cvttpd_epi32(b), _mm256_cvttpd_epi32(a));
69+
}
70+
71+
template <>
72+
@@ -124,7 +158,7 @@ EIGEN_STRONG_INLINE Packet8f pcast<Packet8i, Packet8f>(const Packet8i& a) {
73+
74+
template <>
75+
EIGEN_STRONG_INLINE Packet8f pcast<Packet4d, Packet8f>(const Packet4d& a, const Packet4d& b) {
76+
- return _mm256_set_m128(_mm256_cvtpd_ps(b), _mm256_cvtpd_ps(a));
77+
+ return _eigen_mm256_set_m128(_mm256_cvtpd_ps(b), _mm256_cvtpd_ps(a));
78+
}
79+
80+
template <>
81+
@@ -249,7 +283,7 @@ EIGEN_STRONG_INLINE Packet4d pcast<Packet4l, Packet4d>(const Packet4l& a) {
82+
83+
template <>
84+
EIGEN_STRONG_INLINE Packet4d pcast<Packet2l, Packet4d>(const Packet2l& a, const Packet2l& b) {
85+
- return _mm256_set_m128d((pcast<Packet2l, Packet2d>(b)), (pcast<Packet2l, Packet2d>(a)));
86+
+ return _eigen_mm256_set_m128d((pcast<Packet2l, Packet2d>(b)), (pcast<Packet2l, Packet2d>(a)));
87+
}
88+
89+
template <>
90+
diff --git c/inst/include/Eigen/src/Core/arch/AVX512/PacketMath.h w/inst/include/Eigen/src/Core/arch/AVX512/PacketMath.h
91+
index b76c8a7..baeefa3 100644
92+
--- c/inst/include/Eigen/src/Core/arch/AVX512/PacketMath.h
93+
+++ w/inst/include/Eigen/src/Core/arch/AVX512/PacketMath.h
94+
@@ -44,6 +44,54 @@ typedef eigen_packet_wrapper<__m512i, 6> Packet32s;
95+
typedef eigen_packet_wrapper<__m256i, 6> Packet16s;
96+
typedef eigen_packet_wrapper<__m128i, 6> Packet8s;
97+
98+
+EIGEN_STRONG_INLINE Packet16i _eigen_mm512_loadu_epi32(const int* from) {
99+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
100+
+ return _mm512_loadu_si512(reinterpret_cast<const void*>(from));
101+
+#else
102+
+ return _mm512_loadu_epi32(from);
103+
+#endif
104+
+}
105+
+
106+
+EIGEN_STRONG_INLINE Packet16i _eigen_mm512_loadu_epi64(const int64_t* from) {
107+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
108+
+ return _mm512_loadu_si512(reinterpret_cast<const void*>(from));
109+
+#else
110+
+ return _mm512_loadu_epi64(from);
111+
+#endif
112+
+}
113+
+
114+
+EIGEN_STRONG_INLINE void _eigen_mm512_storeu_epi32(void* to, const Packet16i& from) {
115+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
116+
+ _mm512_storeu_si512(to, from);
117+
+#else
118+
+ _mm512_storeu_epi32(to, from);
119+
+#endif
120+
+}
121+
+
122+
+EIGEN_STRONG_INLINE void _eigen_mm512_storeu_epi64(void* to, const Packet16i& from) {
123+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
124+
+ _mm512_storeu_si512(to, from);
125+
+#else
126+
+ _mm512_storeu_epi64(to, from);
127+
+#endif
128+
+}
129+
+
130+
+EIGEN_STRONG_INLINE void _eigen_mm256_storeu_epi32(void* to, const __m256i& from) {
131+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
132+
+ _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
133+
+#else
134+
+ _mm256_storeu_epi32(to, from);
135+
+#endif
136+
+}
137+
+
138+
+EIGEN_STRONG_INLINE void _eigen_mm_storeu_epi32(void* to, const __m128i& from) {
139+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
140+
+ _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
141+
+#else
142+
+ _mm_storeu_epi32(to, from);
143+
+#endif
144+
+}
145+
+
146+
template <>
147+
struct is_arithmetic<__m512> {
148+
enum { value = true };
149+
@@ -1033,11 +1081,11 @@ EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) {
150+
}
151+
template <>
152+
EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) {
153+
- EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_epi32(from);
154+
+ EIGEN_DEBUG_UNALIGNED_LOAD return _eigen_mm512_loadu_epi32(from);
155+
}
156+
template <>
157+
EIGEN_STRONG_INLINE Packet8l ploadu<Packet8l>(const int64_t* from) {
158+
- EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_epi64(from);
159+
+ EIGEN_DEBUG_UNALIGNED_LOAD return _eigen_mm512_loadu_epi64(from);
160+
}
161+
162+
template <>
163+
@@ -1158,11 +1206,11 @@ EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) {
164+
}
165+
template <>
166+
EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) {
167+
- EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_epi32(to, from);
168+
+ EIGEN_DEBUG_UNALIGNED_STORE _eigen_mm512_storeu_epi32(to, from);
169+
}
170+
template <>
171+
EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet8l& from) {
172+
- EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_epi64(to, from);
173+
+ EIGEN_DEBUG_UNALIGNED_STORE _eigen_mm512_storeu_epi64(to, from);
174+
}
175+
template <>
176+
EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from, uint16_t umask) {
177+
@@ -3019,19 +3067,19 @@ EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet8s>(numext::int16_t* out,
178+
template <>
179+
EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet32s>(numext::int16_t* out, const Packet32s& x) {
180+
EIGEN_DEBUG_UNALIGNED_STORE
181+
- _mm512_storeu_epi32(out, x);
182+
+ _eigen_mm512_storeu_epi32(out, x);
183+
}
184+
185+
template <>
186+
EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet16s>(numext::int16_t* out, const Packet16s& x) {
187+
EIGEN_DEBUG_UNALIGNED_STORE
188+
- _mm256_storeu_epi32(out, x);
189+
+ _eigen_mm256_storeu_epi32(out, x);
190+
}
191+
192+
template <>
193+
EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet8s>(numext::int16_t* out, const Packet8s& x) {
194+
EIGEN_DEBUG_UNALIGNED_STORE
195+
- _mm_storeu_epi32(out, x);
196+
+ _eigen_mm_storeu_epi32(out, x);
197+
}
198+
199+
template <>
200+
diff --git c/inst/include/Eigen/src/Core/arch/AVX512/TypeCasting.h w/inst/include/Eigen/src/Core/arch/AVX512/TypeCasting.h
201+
index fc55fd8..e970d43 100644
202+
--- c/inst/include/Eigen/src/Core/arch/AVX512/TypeCasting.h
203+
+++ w/inst/include/Eigen/src/Core/arch/AVX512/TypeCasting.h
204+
@@ -52,9 +52,17 @@ struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfl
205+
template <>
206+
struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {};
207+
208+
+EIGEN_STRONG_INLINE __mmask16 _eigen_mm512_cmpneq_ps_mask(__m512 a, __m512 b) {
209+
+#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 810)
210+
+ return _mm512_cmp_ps_mask(a, b, _CMP_NEQ_UQ);
211+
+#else
212+
+ return _mm512_cmpneq_ps_mask(a, b);
213+
+#endif
214+
+}
215+
+
216+
template <>
217+
EIGEN_STRONG_INLINE Packet16b pcast<Packet16f, Packet16b>(const Packet16f& a) {
218+
- __mmask16 mask = _mm512_cmpneq_ps_mask(a, pzero(a));
219+
+ __mmask16 mask = _eigen_mm512_cmpneq_ps_mask(a, pzero(a));
220+
return _mm512_maskz_cvtepi32_epi8(mask, _mm512_set1_epi32(1));
221+
}

0 commit comments

Comments
 (0)