[Pcre-svn] [1184] code/trunk/src: ARM64 first character fixe…

Revision: 1184

          http://www.exim.org/viewvc/pcre2?view=rev&revision=1184
Author:   zherczeg
Date:     2019-11-12 13:10:44 +0000 (Tue, 12 Nov 2019)
Log Message:
-----------
ARM64 first character fixes by Sebastian Pop.

Modified Paths:
--------------
    code/trunk/src/pcre2_jit_neon_inc.h
    code/trunk/src/pcre2_jit_simd_inc.h

Modified: code/trunk/src/pcre2_jit_neon_inc.h
===================================================================
--- code/trunk/src/pcre2_jit_neon_inc.h    2019-11-08 16:04:43 UTC (rev 1183)
+++ code/trunk/src/pcre2_jit_neon_inc.h    2019-11-12 13:10:44 UTC (rev 1184)
@@ -112,14 +112,14 @@
 vect_t cmp1a, cmp1b, cmp2a, cmp2b;
 const sljit_u32 diff = IN_UCHARS(offs1 - offs2);
 PCRE2_UCHAR char1a = ic.c.c1;
-PCRE2_UCHAR char1b = ic.c.c2;
 PCRE2_UCHAR char2a = ic.c.c3;
-PCRE2_UCHAR char2b = ic.c.c4;

# ifdef FFCPS_CHAR1A2A
cmp1a = VDUPQ(char1a);
cmp2a = VDUPQ(char2a);
# else
+PCRE2_UCHAR char1b = ic.c.c2;
+PCRE2_UCHAR char2b = ic.c.c4;
if (char1a == char1b)
cmp1a = VDUPQ(char1a);
else
@@ -159,10 +159,17 @@
}
# endif

-str_ptr += offs1;
+str_ptr += IN_UCHARS(offs1);
#endif

+#if PCRE2_CODE_UNIT_WIDTH != 8
+vect_t char_mask = VDUPQ(0xff);
+#endif
+
+#if defined(FF_UTF)
restart:;
+#endif
+
#if defined(FFCPS)
sljit_u8 *p1 = str_ptr - diff;
#endif
@@ -169,7 +176,10 @@
sljit_s32 align_offset = ((uint64_t)str_ptr & 0xf);
str_ptr = (sljit_u8 *) ((uint64_t)str_ptr & ~0xf);
vect_t data = VLD1Q(str_ptr);
-
+#if PCRE2_CODE_UNIT_WIDTH != 8
+data = VANDQ(data, char_mask);
+#endif
+
#if defined(FFCS)
vect_t eq = VCEQQ(data, vc1);

@@ -186,7 +196,17 @@
# if defined(FFCPS_DIFF1)
vect_t prev_data = data;
# endif
-vect_t data2 = VLD1Q(str_ptr - diff);
+
+vect_t data2;
+if (p1 < str_ptr)
+ {
+ data2 = VLD1Q(str_ptr - diff);
+#if PCRE2_CODE_UNIT_WIDTH != 8
+ data2 = VANDQ(data2, char_mask);
+#endif
+ }
+else
+ data2 = shift_left_n_lanes(data, offs1 - offs2);

data = fast_forward_char_pair_compare(compare1_type, data, cmp1a, cmp1b);
data2 = fast_forward_char_pair_compare(compare2_type, data2, cmp2a, cmp2b);
@@ -223,6 +243,9 @@
while (str_ptr < str_end)
{
vect_t orig_data = VLD1Q(str_ptr);
+#if PCRE2_CODE_UNIT_WIDTH != 8
+ orig_data = VANDQ(orig_data, char_mask);
+#endif
data = orig_data;

#if defined(FFCS)
@@ -240,9 +263,12 @@

#if defined(FFCPS)
# if defined (FFCPS_DIFF1)
- data2 = VEXTQ(prev_data, data, 15);
+ data2 = VEXTQ(prev_data, data, VECTOR_FACTOR - 1);
# else
data2 = VLD1Q(str_ptr - diff);
+# if PCRE2_CODE_UNIT_WIDTH != 8
+ data2 = VANDQ(data2, char_mask);
+# endif
# endif

# ifdef FFCPS_CHAR1A2A

Modified: code/trunk/src/pcre2_jit_simd_inc.h
===================================================================
--- code/trunk/src/pcre2_jit_simd_inc.h    2019-11-08 16:04:43 UTC (rev 1183)
+++ code/trunk/src/pcre2_jit_simd_inc.h    2019-11-12 13:10:44 UTC (rev 1184)
@@ -655,8 +655,9 @@
 #endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */

 #if PCRE2_CODE_UNIT_WIDTH == 8
+# define VECTOR_FACTOR 16
 # define vect_t uint8x16_t
-# define VLD1Q vld1q_u8
+# define VLD1Q(X) vld1q_u8((sljit_u8 *)(X))
 # define VCEQQ vceqq_u8
 # define VORRQ vorrq_u8
 # define VST1Q vst1q_u8
@@ -668,8 +669,9 @@
        uint64_t dw[2];
 } quad_word;
 #elif PCRE2_CODE_UNIT_WIDTH == 16
+# define VECTOR_FACTOR 8
 # define vect_t uint16x8_t
-# define VLD1Q vld1q_u16
+# define VLD1Q(X) vld1q_u16((sljit_u16 *)(X))
 # define VCEQQ vceqq_u16
 # define VORRQ vorrq_u16
 # define VST1Q vst1q_u16
@@ -681,8 +683,9 @@
        uint64_t dw[2];
 } quad_word;
 #else
+# define VECTOR_FACTOR 4
 # define vect_t uint32x4_t
-# define VLD1Q vld1q_u32
+# define VLD1Q(X) vld1q_u32((sljit_u32 *)(X))
 # define VCEQQ vceqq_u32
 # define VORRQ vorrq_u32
 # define VST1Q vst1q_u32
@@ -697,23 +700,29 @@

#define FFCS
#include "pcre2_jit_neon_inc.h"
-#define FF_UTF
-#include "pcre2_jit_neon_inc.h"
+#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
+# define FF_UTF
+# include "pcre2_jit_neon_inc.h"
+# undef FF_UTF
+#endif
#undef FFCS
-#undef FF_UTF

#define FFCS_2
#include "pcre2_jit_neon_inc.h"
-#define FF_UTF
-#include "pcre2_jit_neon_inc.h"
-#undef FF_UTF
+#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
+# define FF_UTF
+# include "pcre2_jit_neon_inc.h"
+# undef FF_UTF
+#endif
#undef FFCS_2

#define FFCS_MASK
#include "pcre2_jit_neon_inc.h"
-#define FF_UTF
-#include "pcre2_jit_neon_inc.h"
-#undef FF_UTF
+#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
+# define FF_UTF
+# include "pcre2_jit_neon_inc.h"
+# undef FF_UTF
+#endif
#undef FFCS_MASK

#define JIT_HAS_FAST_FORWARD_CHAR_SIMD 1
@@ -842,6 +851,29 @@
#endif
}

+/* ARM doesn't have a shift left across lanes. */
+static SLJIT_INLINE vect_t shift_left_n_lanes(vect_t a, sljit_u8 n)
+{
+vect_t zero = VDUPQ(0);
+SLJIT_ASSERT(0 < n && n < VECTOR_FACTOR);
+/* VEXTQ takes an immediate as last argument. */
+#define C(X) case X: return VEXTQ(zero, a, VECTOR_FACTOR - X);
+switch (n)
+  {
+  C(1); C(2); C(3);
+#if PCRE2_CODE_UNIT_WIDTH != 32
+  C(4); C(5); C(6); C(7);
+# if PCRE2_CODE_UNIT_WIDTH != 16
+  C(8); C(9); C(10); C(11); C(12); C(13); C(14); C(15);
+# endif
+#endif
+  default:
+    /* Based on the ASSERT(0 < n && n < VECTOR_FACTOR) above, this won't
+       happen. The return is still here for compilers to not warn. */
+    return a;
+  }
+}
+
 #define FFCPS
 #define FFCPS_DIFF1
 #define FFCPS_CHAR1A2A
@@ -848,9 +880,11 @@

#define FFCPS_0
#include "pcre2_jit_neon_inc.h"
-#define FF_UTF
-#include "pcre2_jit_neon_inc.h"
-#undef FF_UTF
+#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
+# define FF_UTF
+# include "pcre2_jit_neon_inc.h"
+# undef FF_UTF
+#endif
#undef FFCPS_0

#undef FFCPS_CHAR1A2A
@@ -857,9 +891,11 @@

#define FFCPS_1
#include "pcre2_jit_neon_inc.h"
-#define FF_UTF
-#include "pcre2_jit_neon_inc.h"
-#undef FF_UTF
+#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
+# define FF_UTF
+# include "pcre2_jit_neon_inc.h"
+# undef FF_UTF
+#endif
#undef FFCPS_1

#undef FFCPS_DIFF1
@@ -866,9 +902,11 @@

#define FFCPS_DEFAULT
#include "pcre2_jit_neon_inc.h"
-#define FF_UTF
-#include "pcre2_jit_neon_inc.h"
-#undef FF_UTF
+#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
+# define FF_UTF
+# include "pcre2_jit_neon_inc.h"
+# undef FF_UTF
+#endif
#undef FFCPS

#define JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD 1