PowerPC LE memset

http://sourceware.org/ml/libc-alpha/2013-08/msg00104.html

One of the things I noticed when looking at power7 timing is that rlwimi
is cracked and the two resulting insns have a register dependency.
That makes it a little slower than the equivalent rldimi.

	* sysdeps/powerpc/powerpc64/memset.S: Replace rlwimi with
        insrdi.  Formatting.
	* sysdeps/powerpc/powerpc64/power4/memset.S: Likewise.
	* sysdeps/powerpc/powerpc64/power6/memset.S: Likewise.
	* sysdeps/powerpc/powerpc64/power7/memset.S: Likewise.
	* sysdeps/powerpc/powerpc32/power4/memset.S: Likewise.
	* sysdeps/powerpc/powerpc32/power6/memset.S: Likewise.
	* sysdeps/powerpc/powerpc32/power7/memset.S: Likewise.

diff --git a/ChangeLog b/ChangeLog
index 3632248e96e85a2e59571b1d3aa31ada401c1e70..99d2b70499a086ad321542d465a401ef2dba36ee 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+2013-10-04  Alan Modra  <amodra@gmail.com>
+
+       * sysdeps/powerpc/powerpc64/memset.S: Replace rlwimi with
+        insrdi.  Formatting.
+       * sysdeps/powerpc/powerpc64/power4/memset.S: Likewise.
+       * sysdeps/powerpc/powerpc64/power6/memset.S: Likewise.
+       * sysdeps/powerpc/powerpc64/power7/memset.S: Likewise.
+       * sysdeps/powerpc/powerpc32/power4/memset.S: Likewise.
+       * sysdeps/powerpc/powerpc32/power6/memset.S: Likewise.
+       * sysdeps/powerpc/powerpc32/power7/memset.S: Likewise.
+
 2013-10-04  Alan Modra  <amodra@gmail.com>
 
        * sysdeps/powerpc/powerpc32/power4/memcpy.S: Add little endian support.

diff --git a/sysdeps/powerpc/powerpc32/power4/memset.S b/sysdeps/powerpc/powerpc32/power4/memset.S
index c2d288b38bbca19e4a35bd3fe07b306e58004fad..4fd9d8cb4a602e923256e2e88764f8b1572b1e96 100644 (file)
--- a/sysdeps/powerpc/powerpc32/power4/memset.S
+++ b/sysdeps/powerpc/powerpc32/power4/memset.S
@@ -50,7 +50,7 @@ L(_memset):
 
 /* Align to word boundary.  */
        cmplwi  cr5, rLEN, 31
-       rlwimi  rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
+       insrdi  rCHR, rCHR, 8, 48     /* Replicate byte to halfword.  */
        beq+    L(aligned)
        mtcrf   0x01, rMEMP0
        subfic  rALIGN, rALIGN, 4
@@ -65,7 +65,7 @@ L(g0):
 /* Handle the case of size < 31.  */
 L(aligned):
        mtcrf   0x01, rLEN
-       rlwimi  rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
+       insrdi  rCHR, rCHR, 16, 32    /* Replicate halfword to word.  */
        ble     cr5, L(medium)
 /* Align to 32-byte boundary.  */
        andi.   rALIGN, rMEMP, 0x1C

diff --git a/sysdeps/powerpc/powerpc32/power6/memset.S b/sysdeps/powerpc/powerpc32/power6/memset.S
index ce066300148c0d7f93e0a2e7e035083aaa8de3ec..508d5607964519ae32b6895d5001ed05dd06a737 100644 (file)
--- a/sysdeps/powerpc/powerpc32/power6/memset.S
+++ b/sysdeps/powerpc/powerpc32/power6/memset.S
@@ -48,7 +48,7 @@ L(_memset):
        ble-    cr1, L(small)
 /* Align to word boundary.  */
        cmplwi  cr5, rLEN, 31
-       rlwimi  rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
+       insrdi  rCHR, rCHR, 8, 48       /* Replicate byte to halfword.  */
        beq+    L(aligned)
        mtcrf   0x01, rMEMP0
        subfic  rALIGN, rALIGN, 4
@@ -64,7 +64,7 @@ L(g0):
 /* Handle the case of size < 31.  */
 L(aligned):
        mtcrf   0x01, rLEN
-       rlwimi  rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
+       insrdi  rCHR, rCHR, 16, 32      /* Replicate halfword to word.  */
        ble     cr5, L(medium)
 /* Align to 32-byte boundary.  */
        andi.   rALIGN, rMEMP, 0x1C

diff --git a/sysdeps/powerpc/powerpc32/power7/memset.S b/sysdeps/powerpc/powerpc32/power7/memset.S
index 360ea717f4e3cad03456f1601907e1526b1e6d72..aadda2558f1694c3162440eaee2f74ab6c3327b0 100644 (file)
--- a/sysdeps/powerpc/powerpc32/power7/memset.S
+++ b/sysdeps/powerpc/powerpc32/power7/memset.S
@@ -35,8 +35,8 @@ L(_memset):
        cfi_offset(31,-8)
 
        /* Replicate byte to word.  */
-       rlwimi  4,4,8,16,23
-       rlwimi  4,4,16,0,15
+       insrdi  4,4,8,48
+       insrdi  4,4,16,32
 
        ble     cr6,L(small)    /* If length <= 8, use short copy code.  */
 

diff --git a/sysdeps/powerpc/powerpc64/memset.S b/sysdeps/powerpc/powerpc64/memset.S
index 6acf149c8ae2446ba536b2d6893acd8136dcc09a..1027a592c970b7f029bf8d4b7a663840b1b5ce3e 100644 (file)
--- a/sysdeps/powerpc/powerpc64/memset.S
+++ b/sysdeps/powerpc/powerpc64/memset.S
@@ -55,14 +55,14 @@ L(_memset):
 
 /* Align to doubleword boundary.  */
        cmpldi  cr5, rLEN, 31
-       rlwimi  rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
+       insrdi  rCHR, rCHR, 8, 48       /* Replicate byte to halfword.  */
        beq+    L(aligned2)
        mtcrf   0x01, rMEMP0
        subfic  rALIGN, rALIGN, 8
        cror    28,30,31                /* Detect odd word aligned.  */
        add     rMEMP, rMEMP, rALIGN
        sub     rLEN, rLEN, rALIGN
-       rlwimi  rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
+       insrdi  rCHR, rCHR, 16, 32      /* Replicate halfword to word.  */
        bt      29, L(g4)
 /* Process the even word of doubleword.  */
        bf+     31, L(g2)
@@ -84,14 +84,14 @@ L(g0):
 
 /* Handle the case of size < 31.  */
 L(aligned2):
-       rlwimi  rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
+       insrdi  rCHR, rCHR, 16, 32      /* Replicate halfword to word.  */
 L(aligned):
        mtcrf   0x01, rLEN
        ble     cr5, L(medium)
 /* Align to 32-byte boundary.  */
        andi.   rALIGN, rMEMP, 0x18
        subfic  rALIGN, rALIGN, 0x20
-       insrdi  rCHR,rCHR,32,0 /* Replicate word to double word. */
+       insrdi  rCHR, rCHR, 32, 0       /* Replicate word to double word. */
        beq     L(caligned)
        mtcrf   0x01, rALIGN
        add     rMEMP, rMEMP, rALIGN
@@ -212,7 +212,7 @@ L(le4):
 /* Memset of 0-31 bytes.  */
        .align 5
 L(medium):
-       insrdi  rCHR,rCHR,32,0 /* Replicate word to double word.  */
+       insrdi  rCHR, rCHR, 32, 0       /* Replicate word to double word.  */
        cmpldi  cr1, rLEN, 16
 L(medium_tail2):
        add     rMEMP, rMEMP, rLEN

diff --git a/sysdeps/powerpc/powerpc64/power4/memset.S b/sysdeps/powerpc/powerpc64/power4/memset.S
index dbecee8b9728896c1a45809c6d5b3dac16df7bf7..ad0d3812816b9653b41a5e80e717cdc16d97dae2 100644 (file)
--- a/sysdeps/powerpc/powerpc64/power4/memset.S
+++ b/sysdeps/powerpc/powerpc64/power4/memset.S
@@ -50,14 +50,14 @@ L(_memset):
 
 /* Align to doubleword boundary.  */
        cmpldi  cr5, rLEN, 31
-       rlwimi  rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
+       insrdi  rCHR, rCHR, 8, 48       /* Replicate byte to halfword.  */
        beq+    L(aligned2)
        mtcrf   0x01, rMEMP0
        subfic  rALIGN, rALIGN, 8
        cror    28,30,31                /* Detect odd word aligned.  */
        add     rMEMP, rMEMP, rALIGN
        sub     rLEN, rLEN, rALIGN
-       rlwimi  rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
+       insrdi  rCHR, rCHR, 16, 32      /* Replicate halfword to word.  */
        bt      29, L(g4)
 /* Process the even word of doubleword.  */
        bf+     31, L(g2)
@@ -79,14 +79,14 @@ L(g0):
 
 /* Handle the case of size < 31.  */
 L(aligned2):
-       rlwimi  rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
+       insrdi  rCHR, rCHR, 16, 32      /* Replicate halfword to word.  */
 L(aligned):
        mtcrf   0x01, rLEN
        ble     cr5, L(medium)
 /* Align to 32-byte boundary.  */
        andi.   rALIGN, rMEMP, 0x18
        subfic  rALIGN, rALIGN, 0x20
-       insrdi  rCHR,rCHR,32,0 /* Replicate word to double word. */
+       insrdi  rCHR, rCHR, 32, 0       /* Replicate word to double word. */
        beq     L(caligned)
        mtcrf   0x01, rALIGN
        add     rMEMP, rMEMP, rALIGN
@@ -146,24 +146,24 @@ L(zloopstart):
 L(getCacheAligned):
        cmpldi  cr1,rLEN,32
        andi.   rTMP,rMEMP,127
-       blt             cr1,L(handletail32)
-       beq             L(cacheAligned)
+       blt     cr1,L(handletail32)
+       beq     L(cacheAligned)
        addi    rMEMP,rMEMP,32
        addi    rLEN,rLEN,-32
-       std             rCHR,-32(rMEMP)
-       std             rCHR,-24(rMEMP)
-       std             rCHR,-16(rMEMP)
-       std             rCHR,-8(rMEMP)
-       b               L(getCacheAligned)
+       std     rCHR,-32(rMEMP)
+       std     rCHR,-24(rMEMP)
+       std     rCHR,-16(rMEMP)
+       std     rCHR,-8(rMEMP)
+       b       L(getCacheAligned)
 
 /* Now we are aligned to the cache line and can use dcbz.  */
 L(cacheAligned):
        cmpld   cr1,rLEN,rCLS
-       blt             cr1,L(handletail32)
+       blt     cr1,L(handletail32)
        dcbz    0,rMEMP
        subf    rLEN,rCLS,rLEN
-       add             rMEMP,rMEMP,rCLS
-       b               L(cacheAligned)
+       add     rMEMP,rMEMP,rCLS
+       b       L(cacheAligned)
 
 /* We are here because the cache line size was set and was not 32-bytes
    and the remainder (rLEN) is less than the actual cache line size.
@@ -200,7 +200,7 @@ L(le4):
 /* Memset of 0-31 bytes.  */
        .align 5
 L(medium):
-       insrdi  rCHR,rCHR,32,0 /* Replicate word to double word.  */
+       insrdi  rCHR, rCHR, 32, 0       /* Replicate word to double word.  */
        cmpldi  cr1, rLEN, 16
 L(medium_tail2):
        add     rMEMP, rMEMP, rLEN

diff --git a/sysdeps/powerpc/powerpc64/power6/memset.S b/sysdeps/powerpc/powerpc64/power6/memset.S
index 541a45fd361e21b6de022942f9035872b7e09e6f..a3864cc894c39af955e0ed0b74c13ae73c1c8cd0 100644 (file)
--- a/sysdeps/powerpc/powerpc64/power6/memset.S
+++ b/sysdeps/powerpc/powerpc64/power6/memset.S
@@ -47,14 +47,14 @@ L(_memset):
 
 /* Align to doubleword boundary.  */
        cmpldi  cr5, rLEN, 31
-       rlwimi  rCHR, rCHR, 8, 16, 23 /* Replicate byte to halfword.  */
+       insrdi  rCHR, rCHR, 8, 48       /* Replicate byte to halfword.  */
        beq+    L(aligned2)
        mtcrf   0x01, rMEMP0
        subfic  rALIGN, rALIGN, 8
        cror    28,30,31                /* Detect odd word aligned.  */
        add     rMEMP, rMEMP, rALIGN
        sub     rLEN, rLEN, rALIGN
-       rlwimi  rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
+       insrdi  rCHR, rCHR, 16, 32      /* Replicate halfword to word.  */
        bt      29, L(g4)
 /* Process the even word of doubleword.  */
        bf+     31, L(g2)
@@ -76,14 +76,14 @@ L(g0):
 
 /* Handle the case of size < 31.  */
 L(aligned2):
-       rlwimi  rCHR, rCHR, 16, 0, 15 /* Replicate halfword to word.  */
+       insrdi  rCHR, rCHR, 16, 32      /* Replicate halfword to word.  */
 L(aligned):
        mtcrf   0x01, rLEN
        ble     cr5, L(medium)
 /* Align to 32-byte boundary.  */
        andi.   rALIGN, rMEMP, 0x18
        subfic  rALIGN, rALIGN, 0x20
-       insrdi  rCHR,rCHR,32,0 /* Replicate word to double word. */
+       insrdi  rCHR, rCHR, 32, 0       /* Replicate word to double word. */
        beq     L(caligned)
        mtcrf   0x01, rALIGN
        add     rMEMP, rMEMP, rALIGN
@@ -344,7 +344,7 @@ L(le4):
 /* Memset of 0-31 bytes.  */
        .align 5
 L(medium):
-       insrdi  rCHR,rCHR,32,0 /* Replicate word to double word.  */
+       insrdi  rCHR, rCHR, 32, 0       /* Replicate word to double word.  */
        cmpldi  cr1, rLEN, 16
 L(medium_tail2):
        add     rMEMP, rMEMP, rLEN

diff --git a/sysdeps/powerpc/powerpc64/power7/memset.S b/sysdeps/powerpc/powerpc64/power7/memset.S
index b24cfa163a279aa4694c8ffa2186ee07de52da49..8b081e87cb2d1c51d5296824138580f9ce8e7e77 100644 (file)
--- a/sysdeps/powerpc/powerpc64/power7/memset.S
+++ b/sysdeps/powerpc/powerpc64/power7/memset.S
@@ -32,8 +32,8 @@ L(_memset):
        mr      10,3
 
        /* Replicate byte to word.  */
-       rlwimi  4,4,8,16,23
-       rlwimi  4,4,16,0,15
+       insrdi  4,4,8,48
+       insrdi  4,4,16,32
        ble     cr6,L(small)    /* If length <= 8, use short copy code.  */
 
        neg     0,3
@@ -321,7 +321,7 @@ L(medium):
        clrldi  0,0,62
        beq     L(medium_aligned)
 
-       /* Force 4-bytes alignment for SRC.  */
+       /* Force 4-bytes alignment for DST.  */
        mtocrf  0x01,0
        subf    5,0,5
 1:     /* Copy 1 byte.  */