2011-08-24 11:12:33 +00:00
|
|
|
|
diff -up openssl-1.0.0d/crypto/aes/asm/aesni-x86.pl.intelopts openssl-1.0.0d/crypto/aes/asm/aesni-x86.pl
|
2011-11-03 09:18:52 +00:00
|
|
|
|
--- openssl-1.0.0d/crypto/aes/asm/aesni-x86.pl.intelopts 2011-08-24 12:50:55.000000000 +0200
|
|
|
|
|
|
+++ openssl-1.0.0d/crypto/aes/asm/aesni-x86.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
|
|
|
|
|
|
# ====================================================================
|
|
|
|
|
|
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
|
|
|
|
|
|
@@ -11,10 +11,37 @@
|
|
|
|
|
|
# OpenSSL context it's used with Intel engine, but can also be used as
|
|
|
|
|
|
# drop-in replacement for crypto/aes/asm/aes-586.pl [see below for
|
|
|
|
|
|
# details].
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Performance.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# To start with see corresponding paragraph in aesni-x86_64.pl...
|
|
|
|
|
|
+# Instead of filling table similar to one found there I've chosen to
|
|
|
|
|
|
+# summarize *comparison* results for raw ECB, CTR and CBC benchmarks.
|
|
|
|
|
|
+# The simplified table below represents 32-bit performance relative
|
|
|
|
|
|
+# to 64-bit one in every given point. Ratios vary for different
|
|
|
|
|
|
+# encryption modes, therefore interval values.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# 16-byte 64-byte 256-byte 1-KB 8-KB
|
|
|
|
|
|
+# 53-67% 67-84% 91-94% 95-98% 97-99.5%
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Lower ratios for smaller block sizes are perfectly understandable,
|
|
|
|
|
|
+# because function call overhead is higher in 32-bit mode. Largest
|
|
|
|
|
|
+# 8-KB block performance is virtually same: 32-bit code is less than
|
|
|
|
|
|
+# 1% slower for ECB, CBC and CCM, and ~3% slower otherwise.
|
|
|
|
|
|
+
|
|
|
|
|
|
+# January 2011
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# See aesni-x86_64.pl for details. Unlike x86_64 version this module
|
|
|
|
|
|
+# interleaves at most 6 aes[enc|dec] instructions, because there are
|
|
|
|
|
|
+# not enough registers for 8x interleave [which should be optimal for
|
|
|
|
|
|
+# Sandy Bridge]. Actually, performance results for 6x interleave
|
|
|
|
|
|
+# factor presented in aesni-x86_64.pl (except for CTR) are for this
|
|
|
|
|
|
+# module.
|
|
|
|
|
|
|
|
|
|
|
|
$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
|
|
|
|
|
|
# generates drop-in replacement for
|
|
|
|
|
|
# crypto/aes/asm/aes-586.pl:-)
|
|
|
|
|
|
+$inline=1; # inline _aesni_[en|de]crypt
|
|
|
|
|
|
|
|
|
|
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|
|
|
|
|
push(@INC,"${dir}","${dir}../../perlasm");
|
|
|
|
|
|
@@ -22,7 +49,8 @@ require "x86asm.pl";
|
|
|
|
|
|
|
|
|
|
|
|
&asm_init($ARGV[0],$0);
|
|
|
|
|
|
|
|
|
|
|
|
-$movekey = eval($RREFIX eq "aseni" ? "*movaps" : "*movups");
|
|
|
|
|
|
+if ($PREFIX eq "aesni") { $movekey=*movups; }
|
|
|
|
|
|
+else { $movekey=*movups; }
|
|
|
|
|
|
|
|
|
|
|
|
$len="eax";
|
|
|
|
|
|
$rounds="ecx";
|
|
|
|
|
|
@@ -32,114 +60,144 @@ $out="edi";
|
|
|
|
|
|
$rounds_="ebx"; # backup copy for $rounds
|
|
|
|
|
|
$key_="ebp"; # backup copy for $key
|
|
|
|
|
|
|
|
|
|
|
|
-$inout0="xmm0";
|
|
|
|
|
|
-$inout1="xmm1";
|
|
|
|
|
|
-$inout2="xmm2";
|
|
|
|
|
|
-$rndkey0="xmm3";
|
|
|
|
|
|
-$rndkey1="xmm4";
|
|
|
|
|
|
-$ivec="xmm5";
|
|
|
|
|
|
-$in0="xmm6";
|
|
|
|
|
|
-$in1="xmm7"; $inout3="xmm7";
|
|
|
|
|
|
-
|
|
|
|
|
|
+$rndkey0="xmm0";
|
|
|
|
|
|
+$rndkey1="xmm1";
|
|
|
|
|
|
+$inout0="xmm2";
|
|
|
|
|
|
+$inout1="xmm3";
|
|
|
|
|
|
+$inout2="xmm4";
|
|
|
|
|
|
+$inout3="xmm5"; $in1="xmm5";
|
|
|
|
|
|
+$inout4="xmm6"; $in0="xmm6";
|
|
|
|
|
|
+$inout5="xmm7"; $ivec="xmm7";
|
|
|
|
|
|
+
|
|
|
|
|
|
+# AESNI extenstion
|
|
|
|
|
|
+sub aeskeygenassist
|
|
|
|
|
|
+{ my($dst,$src,$imm)=@_;
|
|
|
|
|
|
+ if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
|
|
|
|
|
|
+ { &data_byte(0x66,0x0f,0x3a,0xdf,0xc0|($1<<3)|$2,$imm); }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+sub aescommon
|
|
|
|
|
|
+{ my($opcodelet,$dst,$src)=@_;
|
|
|
|
|
|
+ if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
|
|
|
|
|
|
+ { &data_byte(0x66,0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2);}
|
|
|
|
|
|
+}
|
|
|
|
|
|
+sub aesimc { aescommon(0xdb,@_); }
|
|
|
|
|
|
+sub aesenc { aescommon(0xdc,@_); }
|
|
|
|
|
|
+sub aesenclast { aescommon(0xdd,@_); }
|
|
|
|
|
|
+sub aesdec { aescommon(0xde,@_); }
|
|
|
|
|
|
+sub aesdeclast { aescommon(0xdf,@_); }
|
|
|
|
|
|
+
|
|
|
|
|
|
# Inline version of internal aesni_[en|de]crypt1
|
|
|
|
|
|
+{ my $sn;
|
|
|
|
|
|
sub aesni_inline_generate1
|
|
|
|
|
|
-{ my $p=shift;
|
|
|
|
|
|
+{ my ($p,$inout,$ivec)=@_; $inout=$inout0 if (!defined($inout));
|
|
|
|
|
|
+ $sn++;
|
|
|
|
|
|
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(16,$key));
|
|
|
|
|
|
+ &xorps ($ivec,$rndkey0) if (defined($ivec));
|
|
|
|
|
|
&lea ($key,&DWP(32,$key));
|
|
|
|
|
|
- &pxor ($inout0,$rndkey0);
|
|
|
|
|
|
- &set_label("${p}1_loop");
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ &xorps ($inout,$ivec) if (defined($ivec));
|
|
|
|
|
|
+ &xorps ($inout,$rndkey0) if (!defined($ivec));
|
|
|
|
|
|
+ &set_label("${p}1_loop_$sn");
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey1)";
|
|
|
|
|
|
&dec ($rounds);
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(0,$key));
|
|
|
|
|
|
&lea ($key,&DWP(16,$key));
|
|
|
|
|
|
- &jnz (&label("${p}1_loop"));
|
|
|
|
|
|
- eval"&aes${p}last ($inout0,$rndkey1)";
|
|
|
|
|
|
-}
|
|
|
|
|
|
+ &jnz (&label("${p}1_loop_$sn"));
|
|
|
|
|
|
+ eval"&aes${p}last ($inout,$rndkey1)";
|
|
|
|
|
|
+}}
|
|
|
|
|
|
|
|
|
|
|
|
sub aesni_generate1 # fully unrolled loop
|
|
|
|
|
|
-{ my $p=shift;
|
|
|
|
|
|
+{ my ($p,$inout)=@_; $inout=$inout0 if (!defined($inout));
|
|
|
|
|
|
|
|
|
|
|
|
&function_begin_B("_aesni_${p}rypt1");
|
|
|
|
|
|
- &$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
+ &movups ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(0x10,$key));
|
|
|
|
|
|
- &cmp ($rounds,11);
|
|
|
|
|
|
- &pxor ($inout0,$rndkey0);
|
|
|
|
|
|
+ &xorps ($inout,$rndkey0);
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0x20,$key));
|
|
|
|
|
|
&lea ($key,&DWP(0x30,$key));
|
|
|
|
|
|
+ &cmp ($rounds,11);
|
|
|
|
|
|
&jb (&label("${p}128"));
|
|
|
|
|
|
&lea ($key,&DWP(0x20,$key));
|
|
|
|
|
|
&je (&label("${p}192"));
|
|
|
|
|
|
&lea ($key,&DWP(0x20,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(-0x40,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey0)";
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(-0x30,$key));
|
|
|
|
|
|
&set_label("${p}192");
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(-0x20,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey0)";
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(-0x10,$key));
|
|
|
|
|
|
&set_label("${p}128");
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(0,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey0)";
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0x10,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(0x20,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey0)";
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0x30,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(0x40,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey0)";
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0x50,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(0x60,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey0)";
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0x70,$key));
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
- eval"&aes${p}last ($inout0,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p}last ($inout,$rndkey0)";
|
|
|
|
|
|
&ret();
|
|
|
|
|
|
&function_end_B("_aesni_${p}rypt1");
|
|
|
|
|
|
}
|
|
|
|
|
|
-
|
|
|
|
|
|
+
|
|
|
|
|
|
# void $PREFIX_encrypt (const void *inp,void *out,const AES_KEY *key);
|
|
|
|
|
|
-# &aesni_generate1("dec");
|
|
|
|
|
|
+&aesni_generate1("enc") if (!$inline);
|
|
|
|
|
|
&function_begin_B("${PREFIX}_encrypt");
|
|
|
|
|
|
&mov ("eax",&wparam(0));
|
|
|
|
|
|
&mov ($key,&wparam(2));
|
|
|
|
|
|
&movups ($inout0,&QWP(0,"eax"));
|
|
|
|
|
|
&mov ($rounds,&DWP(240,$key));
|
|
|
|
|
|
&mov ("eax",&wparam(1));
|
|
|
|
|
|
- &aesni_inline_generate1("enc"); # &call ("_aesni_encrypt1");
|
|
|
|
|
|
+ if ($inline)
|
|
|
|
|
|
+ { &aesni_inline_generate1("enc"); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &call ("_aesni_encrypt1"); }
|
|
|
|
|
|
&movups (&QWP(0,"eax"),$inout0);
|
|
|
|
|
|
&ret ();
|
|
|
|
|
|
&function_end_B("${PREFIX}_encrypt");
|
|
|
|
|
|
|
|
|
|
|
|
# void $PREFIX_decrypt (const void *inp,void *out,const AES_KEY *key);
|
|
|
|
|
|
-# &aesni_generate1("dec");
|
|
|
|
|
|
+&aesni_generate1("dec") if(!$inline);
|
|
|
|
|
|
&function_begin_B("${PREFIX}_decrypt");
|
|
|
|
|
|
&mov ("eax",&wparam(0));
|
|
|
|
|
|
&mov ($key,&wparam(2));
|
|
|
|
|
|
&movups ($inout0,&QWP(0,"eax"));
|
|
|
|
|
|
&mov ($rounds,&DWP(240,$key));
|
|
|
|
|
|
&mov ("eax",&wparam(1));
|
|
|
|
|
|
- &aesni_inline_generate1("dec"); # &call ("_aesni_decrypt1");
|
|
|
|
|
|
+ if ($inline)
|
|
|
|
|
|
+ { &aesni_inline_generate1("dec"); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &call ("_aesni_decrypt1"); }
|
|
|
|
|
|
&movups (&QWP(0,"eax"),$inout0);
|
|
|
|
|
|
&ret ();
|
|
|
|
|
|
&function_end_B("${PREFIX}_decrypt");
|
|
|
|
|
|
-
|
|
|
|
|
|
-# _aesni_[en|de]crypt[34] are private interfaces, N denotes interleave
|
|
|
|
|
|
-# factor. Why 3x subroutine is used in loops? Even though aes[enc|dec]
|
|
|
|
|
|
-# latency is 6, it turned out that it can be scheduled only every
|
|
|
|
|
|
-# *second* cycle. Thus 3x interleave is the one providing optimal
|
|
|
|
|
|
+
|
|
|
|
|
|
+# _aesni_[en|de]cryptN are private interfaces, N denotes interleave
|
|
|
|
|
|
+# factor. Why 3x subroutine were originally used in loops? Even though
|
|
|
|
|
|
+# aes[enc|dec] latency was originally 6, it could be scheduled only
|
|
|
|
|
|
+# every *2nd* cycle. Thus 3x interleave was the one providing optimal
|
|
|
|
|
|
# utilization, i.e. when subroutine's throughput is virtually same as
|
|
|
|
|
|
# of non-interleaved subroutine [for number of input blocks up to 3].
|
|
|
|
|
|
-# This is why it makes no sense to implement 2x subroutine. As soon
|
|
|
|
|
|
-# as/if Intel improves throughput by making it possible to schedule
|
|
|
|
|
|
-# the instructions in question *every* cycles I would have to
|
|
|
|
|
|
-# implement 6x interleave and use it in loop...
|
|
|
|
|
|
+# This is why it makes no sense to implement 2x subroutine.
|
|
|
|
|
|
+# aes[enc|dec] latency in next processor generation is 8, but the
|
|
|
|
|
|
+# instructions can be scheduled every cycle. Optimal interleave for
|
|
|
|
|
|
+# new processor is therefore 8x, but it's unfeasible to accommodate it
|
|
|
|
|
|
+# in XMM registers addreassable in 32-bit mode and therefore 6x is
|
|
|
|
|
|
+# used instead...
|
|
|
|
|
|
+
|
|
|
|
|
|
sub aesni_generate3
|
|
|
|
|
|
{ my $p=shift;
|
|
|
|
|
|
|
|
|
|
|
|
@@ -148,24 +206,24 @@ sub aesni_generate3
|
|
|
|
|
|
&shr ($rounds,1);
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(16,$key));
|
|
|
|
|
|
&lea ($key,&DWP(32,$key));
|
|
|
|
|
|
- &pxor ($inout0,$rndkey0);
|
|
|
|
|
|
+ &xorps ($inout0,$rndkey0);
|
|
|
|
|
|
&pxor ($inout1,$rndkey0);
|
|
|
|
|
|
&pxor ($inout2,$rndkey0);
|
|
|
|
|
|
- &jmp (&label("${p}3_loop"));
|
|
|
|
|
|
- &set_label("${p}3_loop",16);
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &set_label("${p}3_loop");
|
|
|
|
|
|
+ eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
eval"&aes${p} ($inout1,$rndkey1)";
|
|
|
|
|
|
&dec ($rounds);
|
|
|
|
|
|
eval"&aes${p} ($inout2,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(16,$key));
|
|
|
|
|
|
eval"&aes${p} ($inout0,$rndkey0)";
|
|
|
|
|
|
- &lea ($key,&DWP(32,$key));
|
|
|
|
|
|
eval"&aes${p} ($inout1,$rndkey0)";
|
|
|
|
|
|
+ &lea ($key,&DWP(32,$key));
|
|
|
|
|
|
eval"&aes${p} ($inout2,$rndkey0)";
|
|
|
|
|
|
+ &$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
&jnz (&label("${p}3_loop"));
|
|
|
|
|
|
eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
- &$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
eval"&aes${p} ($inout1,$rndkey1)";
|
|
|
|
|
|
eval"&aes${p} ($inout2,$rndkey1)";
|
|
|
|
|
|
eval"&aes${p}last ($inout0,$rndkey0)";
|
|
|
|
|
|
@@ -187,27 +245,28 @@ sub aesni_generate4
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(16,$key));
|
|
|
|
|
|
&shr ($rounds,1);
|
|
|
|
|
|
&lea ($key,&DWP(32,$key));
|
|
|
|
|
|
- &pxor ($inout0,$rndkey0);
|
|
|
|
|
|
+ &xorps ($inout0,$rndkey0);
|
|
|
|
|
|
&pxor ($inout1,$rndkey0);
|
|
|
|
|
|
&pxor ($inout2,$rndkey0);
|
|
|
|
|
|
&pxor ($inout3,$rndkey0);
|
|
|
|
|
|
- &jmp (&label("${p}3_loop"));
|
|
|
|
|
|
- &set_label("${p}3_loop",16);
|
|
|
|
|
|
- eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &set_label("${p}4_loop");
|
|
|
|
|
|
+ eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
eval"&aes${p} ($inout1,$rndkey1)";
|
|
|
|
|
|
&dec ($rounds);
|
|
|
|
|
|
eval"&aes${p} ($inout2,$rndkey1)";
|
|
|
|
|
|
eval"&aes${p} ($inout3,$rndkey1)";
|
|
|
|
|
|
&$movekey ($rndkey1,&QWP(16,$key));
|
|
|
|
|
|
eval"&aes${p} ($inout0,$rndkey0)";
|
|
|
|
|
|
- &lea ($key,&DWP(32,$key));
|
|
|
|
|
|
eval"&aes${p} ($inout1,$rndkey0)";
|
|
|
|
|
|
+ &lea ($key,&DWP(32,$key));
|
|
|
|
|
|
eval"&aes${p} ($inout2,$rndkey0)";
|
|
|
|
|
|
eval"&aes${p} ($inout3,$rndkey0)";
|
|
|
|
|
|
- &jnz (&label("${p}3_loop"));
|
|
|
|
|
|
+ &$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
+ &jnz (&label("${p}4_loop"));
|
|
|
|
|
|
+
|
|
|
|
|
|
eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
- &$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
eval"&aes${p} ($inout1,$rndkey1)";
|
|
|
|
|
|
eval"&aes${p} ($inout2,$rndkey1)";
|
|
|
|
|
|
eval"&aes${p} ($inout3,$rndkey1)";
|
|
|
|
|
|
@@ -218,12 +277,76 @@ sub aesni_generate4
|
|
|
|
|
|
&ret();
|
|
|
|
|
|
&function_end_B("_aesni_${p}rypt4");
|
|
|
|
|
|
}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub aesni_generate6
|
|
|
|
|
|
+{ my $p=shift;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &function_begin_B("_aesni_${p}rypt6");
|
|
|
|
|
|
+ &static_label("_aesni_${p}rypt6_enter");
|
|
|
|
|
|
+ &$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
+ &shr ($rounds,1);
|
|
|
|
|
|
+ &$movekey ($rndkey1,&QWP(16,$key));
|
|
|
|
|
|
+ &lea ($key,&DWP(32,$key));
|
|
|
|
|
|
+ &xorps ($inout0,$rndkey0);
|
|
|
|
|
|
+ &pxor ($inout1,$rndkey0); # pxor does better here
|
|
|
|
|
|
+ eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ &pxor ($inout2,$rndkey0);
|
|
|
|
|
|
+ eval"&aes${p} ($inout1,$rndkey1)";
|
|
|
|
|
|
+ &pxor ($inout3,$rndkey0);
|
|
|
|
|
|
+ &dec ($rounds);
|
|
|
|
|
|
+ eval"&aes${p} ($inout2,$rndkey1)";
|
|
|
|
|
|
+ &pxor ($inout4,$rndkey0);
|
|
|
|
|
|
+ eval"&aes${p} ($inout3,$rndkey1)";
|
|
|
|
|
|
+ &pxor ($inout5,$rndkey0);
|
|
|
|
|
|
+ eval"&aes${p} ($inout4,$rndkey1)";
|
|
|
|
|
|
+ &$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
+ eval"&aes${p} ($inout5,$rndkey1)";
|
|
|
|
|
|
+ &jmp (&label("_aesni_${p}rypt6_enter"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &set_label("${p}6_loop",16);
|
|
|
|
|
|
+ eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout1,$rndkey1)";
|
|
|
|
|
|
+ &dec ($rounds);
|
|
|
|
|
|
+ eval"&aes${p} ($inout2,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout3,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout4,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout5,$rndkey1)";
|
|
|
|
|
|
+ &set_label("_aesni_${p}rypt6_enter",16);
|
|
|
|
|
|
+ &$movekey ($rndkey1,&QWP(16,$key));
|
|
|
|
|
|
+ eval"&aes${p} ($inout0,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout1,$rndkey0)";
|
|
|
|
|
|
+ &lea ($key,&DWP(32,$key));
|
|
|
|
|
|
+ eval"&aes${p} ($inout2,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout3,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout4,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout5,$rndkey0)";
|
|
|
|
|
|
+ &$movekey ($rndkey0,&QWP(0,$key));
|
|
|
|
|
|
+ &jnz (&label("${p}6_loop"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval"&aes${p} ($inout0,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout1,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout2,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout3,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout4,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p} ($inout5,$rndkey1)";
|
|
|
|
|
|
+ eval"&aes${p}last ($inout0,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p}last ($inout1,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p}last ($inout2,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p}last ($inout3,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p}last ($inout4,$rndkey0)";
|
|
|
|
|
|
+ eval"&aes${p}last ($inout5,$rndkey0)";
|
|
|
|
|
|
+ &ret();
|
|
|
|
|
|
+ &function_end_B("_aesni_${p}rypt6");
|
|
|
|
|
|
+}
|
|
|
|
|
|
&aesni_generate3("enc") if ($PREFIX eq "aesni");
|
|
|
|
|
|
&aesni_generate3("dec");
|
|
|
|
|
|
&aesni_generate4("enc") if ($PREFIX eq "aesni");
|
|
|
|
|
|
&aesni_generate4("dec");
|
|
|
|
|
|
-
|
|
|
|
|
|
+&aesni_generate6("enc") if ($PREFIX eq "aesni");
|
|
|
|
|
|
+&aesni_generate6("dec");
|
|
|
|
|
|
+
|
|
|
|
|
|
if ($PREFIX eq "aesni") {
|
|
|
|
|
|
+######################################################################
|
|
|
|
|
|
# void aesni_ecb_encrypt (const void *in, void *out,
|
|
|
|
|
|
# size_t length, const AES_KEY *key,
|
|
|
|
|
|
# int enc);
|
|
|
|
|
|
@@ -232,62 +355,93 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&mov ($out,&wparam(1));
|
|
|
|
|
|
&mov ($len,&wparam(2));
|
|
|
|
|
|
&mov ($key,&wparam(3));
|
|
|
|
|
|
- &mov ($rounds,&wparam(4));
|
|
|
|
|
|
- &cmp ($len,16);
|
|
|
|
|
|
- &jb (&label("ecb_ret"));
|
|
|
|
|
|
+ &mov ($rounds_,&wparam(4));
|
|
|
|
|
|
&and ($len,-16);
|
|
|
|
|
|
- &test ($rounds,$rounds)
|
|
|
|
|
|
+ &jz (&label("ecb_ret"));
|
|
|
|
|
|
&mov ($rounds,&DWP(240,$key));
|
|
|
|
|
|
+ &test ($rounds_,$rounds_);
|
|
|
|
|
|
+ &jz (&label("ecb_decrypt"));
|
|
|
|
|
|
+
|
|
|
|
|
|
&mov ($key_,$key); # backup $key
|
|
|
|
|
|
&mov ($rounds_,$rounds); # backup $rounds
|
|
|
|
|
|
- &jz (&label("ecb_decrypt"));
|
|
|
|
|
|
+ &cmp ($len,0x60);
|
|
|
|
|
|
+ &jb (&label("ecb_enc_tail"));
|
|
|
|
|
|
|
|
|
|
|
|
- &sub ($len,0x40);
|
|
|
|
|
|
- &jbe (&label("ecb_enc_tail"));
|
|
|
|
|
|
- &jmp (&label("ecb_enc_loop3"));
|
|
|
|
|
|
+ &movdqu ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
+ &movdqu ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
+ &movdqu ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
+ &movdqu ($inout3,&QWP(0x30,$inp));
|
|
|
|
|
|
+ &movdqu ($inout4,&QWP(0x40,$inp));
|
|
|
|
|
|
+ &movdqu ($inout5,&QWP(0x50,$inp));
|
|
|
|
|
|
+ &lea ($inp,&DWP(0x60,$inp));
|
|
|
|
|
|
+ &sub ($len,0x60);
|
|
|
|
|
|
+ &jmp (&label("ecb_enc_loop6_enter"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("ecb_enc_loop6",16);
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
+ &movdqu ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
+ &movdqu ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
+ &movdqu ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
+ &movdqu ($inout3,&QWP(0x30,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x40,$out),$inout4);
|
|
|
|
|
|
+ &movdqu ($inout4,&QWP(0x40,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x50,$out),$inout5);
|
|
|
|
|
|
+ &lea ($out,&DWP(0x60,$out));
|
|
|
|
|
|
+ &movdqu ($inout5,&QWP(0x50,$inp));
|
|
|
|
|
|
+ &lea ($inp,&DWP(0x60,$inp));
|
|
|
|
|
|
+&set_label("ecb_enc_loop6_enter");
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &call ("_aesni_encrypt6");
|
|
|
|
|
|
|
|
|
|
|
|
-&set_label("ecb_enc_loop3",16);
|
|
|
|
|
|
- &movups ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
- &movups ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
- &movups ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
- &call ("_aesni_encrypt3");
|
|
|
|
|
|
- &sub ($len,0x30);
|
|
|
|
|
|
- &lea ($inp,&DWP(0x30,$inp));
|
|
|
|
|
|
- &lea ($out,&DWP(0x30,$out));
|
|
|
|
|
|
- &movups (&QWP(-0x30,$out),$inout0);
|
|
|
|
|
|
&mov ($key,$key_); # restore $key
|
|
|
|
|
|
- &movups (&QWP(-0x20,$out),$inout1);
|
|
|
|
|
|
&mov ($rounds,$rounds_); # restore $rounds
|
|
|
|
|
|
- &movups (&QWP(-0x10,$out),$inout2);
|
|
|
|
|
|
- &ja (&label("ecb_enc_loop3"));
|
|
|
|
|
|
+ &sub ($len,0x60);
|
|
|
|
|
|
+ &jnc (&label("ecb_enc_loop6"));
|
|
|
|
|
|
|
|
|
|
|
|
-&set_label("ecb_enc_tail");
|
|
|
|
|
|
- &add ($len,0x40);
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
+ &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
+ &movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
+ &movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
+ &movups (&QWP(0x40,$out),$inout4);
|
|
|
|
|
|
+ &movups (&QWP(0x50,$out),$inout5);
|
|
|
|
|
|
+ &lea ($out,&DWP(0x60,$out));
|
|
|
|
|
|
+ &add ($len,0x60);
|
|
|
|
|
|
&jz (&label("ecb_ret"));
|
|
|
|
|
|
|
|
|
|
|
|
- &cmp ($len,0x10);
|
|
|
|
|
|
+&set_label("ecb_enc_tail");
|
|
|
|
|
|
&movups ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
- &je (&label("ecb_enc_one"));
|
|
|
|
|
|
&cmp ($len,0x20);
|
|
|
|
|
|
+ &jb (&label("ecb_enc_one"));
|
|
|
|
|
|
&movups ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
&je (&label("ecb_enc_two"));
|
|
|
|
|
|
- &cmp ($len,0x30);
|
|
|
|
|
|
&movups ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
- &je (&label("ecb_enc_three"));
|
|
|
|
|
|
+ &cmp ($len,0x40);
|
|
|
|
|
|
+ &jb (&label("ecb_enc_three"));
|
|
|
|
|
|
&movups ($inout3,&QWP(0x30,$inp));
|
|
|
|
|
|
- &call ("_aesni_encrypt4");
|
|
|
|
|
|
+ &je (&label("ecb_enc_four"));
|
|
|
|
|
|
+ &movups ($inout4,&QWP(0x40,$inp));
|
|
|
|
|
|
+ &xorps ($inout5,$inout5);
|
|
|
|
|
|
+ &call ("_aesni_encrypt6");
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
&movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
&movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
&movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
+ &movups (&QWP(0x40,$out),$inout4);
|
|
|
|
|
|
jmp (&label("ecb_ret"));
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("ecb_enc_one",16);
|
|
|
|
|
|
- &aesni_inline_generate1("enc"); # &call ("_aesni_encrypt1");
|
|
|
|
|
|
+ if ($inline)
|
|
|
|
|
|
+ { &aesni_inline_generate1("enc"); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &call ("_aesni_encrypt1"); }
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
&jmp (&label("ecb_ret"));
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("ecb_enc_two",16);
|
|
|
|
|
|
+ &xorps ($inout2,$inout2);
|
|
|
|
|
|
&call ("_aesni_encrypt3");
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
&movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
@@ -300,53 +454,95 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
&jmp (&label("ecb_ret"));
|
|
|
|
|
|
|
|
|
|
|
|
+&set_label("ecb_enc_four",16);
|
|
|
|
|
|
+ &call ("_aesni_encrypt4");
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
+ &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
+ &movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
+ &movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
+ &jmp (&label("ecb_ret"));
|
|
|
|
|
|
+######################################################################
|
|
|
|
|
|
&set_label("ecb_decrypt",16);
|
|
|
|
|
|
- &sub ($len,0x40);
|
|
|
|
|
|
- &jbe (&label("ecb_dec_tail"));
|
|
|
|
|
|
- &jmp (&label("ecb_dec_loop3"));
|
|
|
|
|
|
+ &mov ($key_,$key); # backup $key
|
|
|
|
|
|
+ &mov ($rounds_,$rounds); # backup $rounds
|
|
|
|
|
|
+ &cmp ($len,0x60);
|
|
|
|
|
|
+ &jb (&label("ecb_dec_tail"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqu ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
+ &movdqu ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
+ &movdqu ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
+ &movdqu ($inout3,&QWP(0x30,$inp));
|
|
|
|
|
|
+ &movdqu ($inout4,&QWP(0x40,$inp));
|
|
|
|
|
|
+ &movdqu ($inout5,&QWP(0x50,$inp));
|
|
|
|
|
|
+ &lea ($inp,&DWP(0x60,$inp));
|
|
|
|
|
|
+ &sub ($len,0x60);
|
|
|
|
|
|
+ &jmp (&label("ecb_dec_loop6_enter"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("ecb_dec_loop6",16);
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
+ &movdqu ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
+ &movdqu ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
+ &movdqu ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
+ &movdqu ($inout3,&QWP(0x30,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x40,$out),$inout4);
|
|
|
|
|
|
+ &movdqu ($inout4,&QWP(0x40,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x50,$out),$inout5);
|
|
|
|
|
|
+ &lea ($out,&DWP(0x60,$out));
|
|
|
|
|
|
+ &movdqu ($inout5,&QWP(0x50,$inp));
|
|
|
|
|
|
+ &lea ($inp,&DWP(0x60,$inp));
|
|
|
|
|
|
+&set_label("ecb_dec_loop6_enter");
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &call ("_aesni_decrypt6");
|
|
|
|
|
|
|
|
|
|
|
|
-&set_label("ecb_dec_loop3",16);
|
|
|
|
|
|
- &movups ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
- &movups ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
- &movups ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
- &call ("_aesni_decrypt3");
|
|
|
|
|
|
- &sub ($len,0x30);
|
|
|
|
|
|
- &lea ($inp,&DWP(0x30,$inp));
|
|
|
|
|
|
- &lea ($out,&DWP(0x30,$out));
|
|
|
|
|
|
- &movups (&QWP(-0x30,$out),$inout0);
|
|
|
|
|
|
&mov ($key,$key_); # restore $key
|
|
|
|
|
|
- &movups (&QWP(-0x20,$out),$inout1);
|
|
|
|
|
|
&mov ($rounds,$rounds_); # restore $rounds
|
|
|
|
|
|
- &movups (&QWP(-0x10,$out),$inout2);
|
|
|
|
|
|
- &ja (&label("ecb_dec_loop3"));
|
|
|
|
|
|
+ &sub ($len,0x60);
|
|
|
|
|
|
+ &jnc (&label("ecb_dec_loop6"));
|
|
|
|
|
|
|
|
|
|
|
|
-&set_label("ecb_dec_tail");
|
|
|
|
|
|
- &add ($len,0x40);
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
+ &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
+ &movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
+ &movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
+ &movups (&QWP(0x40,$out),$inout4);
|
|
|
|
|
|
+ &movups (&QWP(0x50,$out),$inout5);
|
|
|
|
|
|
+ &lea ($out,&DWP(0x60,$out));
|
|
|
|
|
|
+ &add ($len,0x60);
|
|
|
|
|
|
&jz (&label("ecb_ret"));
|
|
|
|
|
|
|
|
|
|
|
|
- &cmp ($len,0x10);
|
|
|
|
|
|
+&set_label("ecb_dec_tail");
|
|
|
|
|
|
&movups ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
- &je (&label("ecb_dec_one"));
|
|
|
|
|
|
&cmp ($len,0x20);
|
|
|
|
|
|
+ &jb (&label("ecb_dec_one"));
|
|
|
|
|
|
&movups ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
&je (&label("ecb_dec_two"));
|
|
|
|
|
|
- &cmp ($len,0x30);
|
|
|
|
|
|
&movups ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
- &je (&label("ecb_dec_three"));
|
|
|
|
|
|
+ &cmp ($len,0x40);
|
|
|
|
|
|
+ &jb (&label("ecb_dec_three"));
|
|
|
|
|
|
&movups ($inout3,&QWP(0x30,$inp));
|
|
|
|
|
|
- &call ("_aesni_decrypt4");
|
|
|
|
|
|
+ &je (&label("ecb_dec_four"));
|
|
|
|
|
|
+ &movups ($inout4,&QWP(0x40,$inp));
|
|
|
|
|
|
+ &xorps ($inout5,$inout5);
|
|
|
|
|
|
+ &call ("_aesni_decrypt6");
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
&movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
&movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
&movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
+ &movups (&QWP(0x40,$out),$inout4);
|
|
|
|
|
|
&jmp (&label("ecb_ret"));
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("ecb_dec_one",16);
|
|
|
|
|
|
- &aesni_inline_generate1("dec"); # &call ("_aesni_decrypt3");
|
|
|
|
|
|
+ if ($inline)
|
|
|
|
|
|
+ { &aesni_inline_generate1("dec"); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &call ("_aesni_decrypt1"); }
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
&jmp (&label("ecb_ret"));
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("ecb_dec_two",16);
|
|
|
|
|
|
+ &xorps ($inout2,$inout2);
|
|
|
|
|
|
&call ("_aesni_decrypt3");
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
&movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
@@ -357,28 +553,42 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
&movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
&movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
+ &jmp (&label("ecb_ret"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("ecb_dec_four",16);
|
|
|
|
|
|
+ &call ("_aesni_decrypt4");
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
+ &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
+ &movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
+ &movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("ecb_ret");
|
|
|
|
|
|
&function_end("aesni_ecb_encrypt");
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
+######################################################################
|
|
|
|
|
|
# void $PREFIX_cbc_encrypt (const void *inp, void *out,
|
|
|
|
|
|
# size_t length, const AES_KEY *key,
|
|
|
|
|
|
# unsigned char *ivp,const int enc);
|
|
|
|
|
|
&function_begin("${PREFIX}_cbc_encrypt");
|
|
|
|
|
|
&mov ($inp,&wparam(0));
|
|
|
|
|
|
+ &mov ($rounds_,"esp");
|
|
|
|
|
|
&mov ($out,&wparam(1));
|
|
|
|
|
|
+ &sub ($rounds_,24);
|
|
|
|
|
|
&mov ($len,&wparam(2));
|
|
|
|
|
|
+ &and ($rounds_,-16);
|
|
|
|
|
|
&mov ($key,&wparam(3));
|
|
|
|
|
|
- &test ($len,$len);
|
|
|
|
|
|
&mov ($key_,&wparam(4));
|
|
|
|
|
|
- &jz (&label("cbc_ret"));
|
|
|
|
|
|
+ &test ($len,$len);
|
|
|
|
|
|
+ &jz (&label("cbc_abort"));
|
|
|
|
|
|
|
|
|
|
|
|
&cmp (&wparam(5),0);
|
|
|
|
|
|
- &movups ($ivec,&QWP(0,$key_)); # load IV
|
|
|
|
|
|
+ &xchg ($rounds_,"esp"); # alloca
|
|
|
|
|
|
+ &movups ($ivec,&QWP(0,$key_)); # load IV
|
|
|
|
|
|
&mov ($rounds,&DWP(240,$key));
|
|
|
|
|
|
- &mov ($key_,$key); # backup $key
|
|
|
|
|
|
- &mov ($rounds_,$rounds); # backup $rounds
|
|
|
|
|
|
+ &mov ($key_,$key); # backup $key
|
|
|
|
|
|
+ &mov (&DWP(16,"esp"),$rounds_); # save original %esp
|
|
|
|
|
|
+ &mov ($rounds_,$rounds); # backup $rounds
|
|
|
|
|
|
&je (&label("cbc_decrypt"));
|
|
|
|
|
|
|
|
|
|
|
|
&movaps ($inout0,$ivec);
|
|
|
|
|
|
@@ -388,15 +598,17 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&jmp (&label("cbc_enc_loop"));
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("cbc_enc_loop",16);
|
|
|
|
|
|
- &movups ($ivec,&QWP(0,$inp));
|
|
|
|
|
|
+ &movups ($ivec,&QWP(0,$inp)); # input actually
|
|
|
|
|
|
&lea ($inp,&DWP(16,$inp));
|
|
|
|
|
|
- &pxor ($inout0,$ivec);
|
|
|
|
|
|
- &aesni_inline_generate1("enc"); # &call ("_aesni_encrypt3");
|
|
|
|
|
|
- &sub ($len,16);
|
|
|
|
|
|
- &lea ($out,&DWP(16,$out));
|
|
|
|
|
|
+ if ($inline)
|
|
|
|
|
|
+ { &aesni_inline_generate1("enc",$inout0,$ivec); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &xorps($inout0,$ivec); &call("_aesni_encrypt1"); }
|
|
|
|
|
|
&mov ($rounds,$rounds_); # restore $rounds
|
|
|
|
|
|
&mov ($key,$key_); # restore $key
|
|
|
|
|
|
- &movups (&QWP(-16,$out),$inout0);
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0); # store output
|
|
|
|
|
|
+ &lea ($out,&DWP(16,$out));
|
|
|
|
|
|
+ &sub ($len,16);
|
|
|
|
|
|
&jnc (&label("cbc_enc_loop"));
|
|
|
|
|
|
&add ($len,16);
|
|
|
|
|
|
&jnz (&label("cbc_enc_tail"));
|
|
|
|
|
|
@@ -415,90 +627,151 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&mov ($inp,$out); # $inp and $out are the same
|
|
|
|
|
|
&mov ($key,$key_); # restore $key
|
|
|
|
|
|
&jmp (&label("cbc_enc_loop"));
|
|
|
|
|
|
-
|
|
|
|
|
|
+######################################################################
|
|
|
|
|
|
&set_label("cbc_decrypt",16);
|
|
|
|
|
|
- &sub ($len,0x40);
|
|
|
|
|
|
+ &cmp ($len,0x50);
|
|
|
|
|
|
&jbe (&label("cbc_dec_tail"));
|
|
|
|
|
|
- &jmp (&label("cbc_dec_loop3"));
|
|
|
|
|
|
+ &movaps (&QWP(0,"esp"),$ivec); # save IV
|
|
|
|
|
|
+ &sub ($len,0x50);
|
|
|
|
|
|
+ &jmp (&label("cbc_dec_loop6_enter"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("cbc_dec_loop6",16);
|
|
|
|
|
|
+ &movaps (&QWP(0,"esp"),$rndkey0); # save IV
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout5);
|
|
|
|
|
|
+ &lea ($out,&DWP(0x10,$out));
|
|
|
|
|
|
+&set_label("cbc_dec_loop6_enter");
|
|
|
|
|
|
+ &movdqu ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
+ &movdqu ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
+ &movdqu ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
+ &movdqu ($inout3,&QWP(0x30,$inp));
|
|
|
|
|
|
+ &movdqu ($inout4,&QWP(0x40,$inp));
|
|
|
|
|
|
+ &movdqu ($inout5,&QWP(0x50,$inp));
|
|
|
|
|
|
|
|
|
|
|
|
-&set_label("cbc_dec_loop3",16);
|
|
|
|
|
|
- &movups ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
- &movups ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
- &movups ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
- &movaps ($in0,$inout0);
|
|
|
|
|
|
- &movaps ($in1,$inout1);
|
|
|
|
|
|
- &call ("_aesni_decrypt3");
|
|
|
|
|
|
- &sub ($len,0x30);
|
|
|
|
|
|
- &lea ($inp,&DWP(0x30,$inp));
|
|
|
|
|
|
- &lea ($out,&DWP(0x30,$out));
|
|
|
|
|
|
- &pxor ($inout0,$ivec);
|
|
|
|
|
|
- &pxor ($inout1,$in0);
|
|
|
|
|
|
- &movups ($ivec,&QWP(-0x10,$inp));
|
|
|
|
|
|
- &pxor ($inout2,$in1);
|
|
|
|
|
|
- &movups (&QWP(-0x30,$out),$inout0);
|
|
|
|
|
|
- &mov ($rounds,$rounds_) # restore $rounds
|
|
|
|
|
|
- &movups (&QWP(-0x20,$out),$inout1);
|
|
|
|
|
|
- &mov ($key,$key_); # restore $key
|
|
|
|
|
|
- &movups (&QWP(-0x10,$out),$inout2);
|
|
|
|
|
|
- &ja (&label("cbc_dec_loop3"));
|
|
|
|
|
|
+ &call ("_aesni_decrypt6");
|
|
|
|
|
|
|
|
|
|
|
|
+ &movups ($rndkey1,&QWP(0,$inp));
|
|
|
|
|
|
+ &movups ($rndkey0,&QWP(0x10,$inp));
|
|
|
|
|
|
+ &xorps ($inout0,&QWP(0,"esp")); # ^=IV
|
|
|
|
|
|
+ &xorps ($inout1,$rndkey1);
|
|
|
|
|
|
+ &movups ($rndkey1,&QWP(0x20,$inp));
|
|
|
|
|
|
+ &xorps ($inout2,$rndkey0);
|
|
|
|
|
|
+ &movups ($rndkey0,&QWP(0x30,$inp));
|
|
|
|
|
|
+ &xorps ($inout3,$rndkey1);
|
|
|
|
|
|
+ &movups ($rndkey1,&QWP(0x40,$inp));
|
|
|
|
|
|
+ &xorps ($inout4,$rndkey0);
|
|
|
|
|
|
+ &movups ($rndkey0,&QWP(0x50,$inp)); # IV
|
|
|
|
|
|
+ &xorps ($inout5,$rndkey1);
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
+ &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
+ &lea ($inp,&DWP(0x60,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
+ &mov ($rounds,$rounds_) # restore $rounds
|
|
|
|
|
|
+ &movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
+ &mov ($key,$key_); # restore $key
|
|
|
|
|
|
+ &movups (&QWP(0x40,$out),$inout4);
|
|
|
|
|
|
+ &lea ($out,&DWP(0x50,$out));
|
|
|
|
|
|
+ &sub ($len,0x60);
|
|
|
|
|
|
+ &ja (&label("cbc_dec_loop6"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movaps ($inout0,$inout5);
|
|
|
|
|
|
+ &movaps ($ivec,$rndkey0);
|
|
|
|
|
|
+ &add ($len,0x50);
|
|
|
|
|
|
+ &jle (&label("cbc_dec_tail_collected"));
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
+ &lea ($out,&DWP(0x10,$out));
|
|
|
|
|
|
&set_label("cbc_dec_tail");
|
|
|
|
|
|
- &add ($len,0x40);
|
|
|
|
|
|
- &jz (&label("cbc_ret"));
|
|
|
|
|
|
-
|
|
|
|
|
|
&movups ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
- &cmp ($len,0x10);
|
|
|
|
|
|
&movaps ($in0,$inout0);
|
|
|
|
|
|
+ &cmp ($len,0x10);
|
|
|
|
|
|
&jbe (&label("cbc_dec_one"));
|
|
|
|
|
|
+
|
|
|
|
|
|
&movups ($inout1,&QWP(0x10,$inp));
|
|
|
|
|
|
- &cmp ($len,0x20);
|
|
|
|
|
|
&movaps ($in1,$inout1);
|
|
|
|
|
|
+ &cmp ($len,0x20);
|
|
|
|
|
|
&jbe (&label("cbc_dec_two"));
|
|
|
|
|
|
+
|
|
|
|
|
|
&movups ($inout2,&QWP(0x20,$inp));
|
|
|
|
|
|
&cmp ($len,0x30);
|
|
|
|
|
|
&jbe (&label("cbc_dec_three"));
|
|
|
|
|
|
+
|
|
|
|
|
|
&movups ($inout3,&QWP(0x30,$inp));
|
|
|
|
|
|
- &call ("_aesni_decrypt4");
|
|
|
|
|
|
+ &cmp ($len,0x40);
|
|
|
|
|
|
+ &jbe (&label("cbc_dec_four"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movups ($inout4,&QWP(0x40,$inp));
|
|
|
|
|
|
+ &movaps (&QWP(0,"esp"),$ivec); # save IV
|
|
|
|
|
|
+ &movups ($inout0,&QWP(0,$inp));
|
|
|
|
|
|
+ &xorps ($inout5,$inout5);
|
|
|
|
|
|
+ &call ("_aesni_decrypt6");
|
|
|
|
|
|
+ &movups ($rndkey1,&QWP(0,$inp));
|
|
|
|
|
|
&movups ($rndkey0,&QWP(0x10,$inp));
|
|
|
|
|
|
+ &xorps ($inout0,&QWP(0,"esp")); # ^= IV
|
|
|
|
|
|
+ &xorps ($inout1,$rndkey1);
|
|
|
|
|
|
&movups ($rndkey1,&QWP(0x20,$inp));
|
|
|
|
|
|
- &pxor ($inout0,$ivec);
|
|
|
|
|
|
- &pxor ($inout1,$in0);
|
|
|
|
|
|
- &movups ($ivec,&QWP(0x30,$inp));
|
|
|
|
|
|
+ &xorps ($inout2,$rndkey0);
|
|
|
|
|
|
+ &movups ($rndkey0,&QWP(0x30,$inp));
|
|
|
|
|
|
+ &xorps ($inout3,$rndkey1);
|
|
|
|
|
|
+ &movups ($ivec,&QWP(0x40,$inp)); # IV
|
|
|
|
|
|
+ &xorps ($inout4,$rndkey0);
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
- &pxor ($inout2,$rndkey0);
|
|
|
|
|
|
- &pxor ($inout3,$rndkey1);
|
|
|
|
|
|
&movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
&movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
- &movaps ($inout0,$inout3);
|
|
|
|
|
|
- &lea ($out,&DWP(0x30,$out));
|
|
|
|
|
|
+ &movups (&QWP(0x30,$out),$inout3);
|
|
|
|
|
|
+ &lea ($out,&DWP(0x40,$out));
|
|
|
|
|
|
+ &movaps ($inout0,$inout4);
|
|
|
|
|
|
+ &sub ($len,0x50);
|
|
|
|
|
|
&jmp (&label("cbc_dec_tail_collected"));
|
|
|
|
|
|
|
|
|
|
|
|
-&set_label("cbc_dec_one");
|
|
|
|
|
|
- &aesni_inline_generate1("dec"); # &call ("_aesni_decrypt3");
|
|
|
|
|
|
- &pxor ($inout0,$ivec);
|
|
|
|
|
|
+&set_label("cbc_dec_one",16);
|
|
|
|
|
|
+ if ($inline)
|
|
|
|
|
|
+ { &aesni_inline_generate1("dec"); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &call ("_aesni_decrypt1"); }
|
|
|
|
|
|
+ &xorps ($inout0,$ivec);
|
|
|
|
|
|
&movaps ($ivec,$in0);
|
|
|
|
|
|
+ &sub ($len,0x10);
|
|
|
|
|
|
&jmp (&label("cbc_dec_tail_collected"));
|
|
|
|
|
|
|
|
|
|
|
|
-&set_label("cbc_dec_two");
|
|
|
|
|
|
+&set_label("cbc_dec_two",16);
|
|
|
|
|
|
+ &xorps ($inout2,$inout2);
|
|
|
|
|
|
&call ("_aesni_decrypt3");
|
|
|
|
|
|
- &pxor ($inout0,$ivec);
|
|
|
|
|
|
- &pxor ($inout1,$in0);
|
|
|
|
|
|
+ &xorps ($inout0,$ivec);
|
|
|
|
|
|
+ &xorps ($inout1,$in0);
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
&movaps ($inout0,$inout1);
|
|
|
|
|
|
- &movaps ($ivec,$in1);
|
|
|
|
|
|
&lea ($out,&DWP(0x10,$out));
|
|
|
|
|
|
+ &movaps ($ivec,$in1);
|
|
|
|
|
|
+ &sub ($len,0x20);
|
|
|
|
|
|
&jmp (&label("cbc_dec_tail_collected"));
|
|
|
|
|
|
|
|
|
|
|
|
-&set_label("cbc_dec_three");
|
|
|
|
|
|
+&set_label("cbc_dec_three",16);
|
|
|
|
|
|
&call ("_aesni_decrypt3");
|
|
|
|
|
|
- &pxor ($inout0,$ivec);
|
|
|
|
|
|
- &pxor ($inout1,$in0);
|
|
|
|
|
|
- &pxor ($inout2,$in1);
|
|
|
|
|
|
+ &xorps ($inout0,$ivec);
|
|
|
|
|
|
+ &xorps ($inout1,$in0);
|
|
|
|
|
|
+ &xorps ($inout2,$in1);
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
- &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
&movaps ($inout0,$inout2);
|
|
|
|
|
|
- &movups ($ivec,&QWP(0x20,$inp));
|
|
|
|
|
|
+ &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
&lea ($out,&DWP(0x20,$out));
|
|
|
|
|
|
+ &movups ($ivec,&QWP(0x20,$inp));
|
|
|
|
|
|
+ &sub ($len,0x30);
|
|
|
|
|
|
+ &jmp (&label("cbc_dec_tail_collected"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("cbc_dec_four",16);
|
|
|
|
|
|
+ &call ("_aesni_decrypt4");
|
|
|
|
|
|
+ &movups ($rndkey1,&QWP(0x10,$inp));
|
|
|
|
|
|
+ &movups ($rndkey0,&QWP(0x20,$inp));
|
|
|
|
|
|
+ &xorps ($inout0,$ivec);
|
|
|
|
|
|
+ &movups ($ivec,&QWP(0x30,$inp));
|
|
|
|
|
|
+ &xorps ($inout1,$in0);
|
|
|
|
|
|
+ &movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
+ &xorps ($inout2,$rndkey1);
|
|
|
|
|
|
+ &movups (&QWP(0x10,$out),$inout1);
|
|
|
|
|
|
+ &xorps ($inout3,$rndkey0);
|
|
|
|
|
|
+ &movups (&QWP(0x20,$out),$inout2);
|
|
|
|
|
|
+ &lea ($out,&DWP(0x30,$out));
|
|
|
|
|
|
+ &movaps ($inout0,$inout3);
|
|
|
|
|
|
+ &sub ($len,0x40);
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("cbc_dec_tail_collected");
|
|
|
|
|
|
&and ($len,15);
|
|
|
|
|
|
@@ -506,21 +779,21 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&movups (&QWP(0,$out),$inout0);
|
|
|
|
|
|
&jmp (&label("cbc_ret"));
|
|
|
|
|
|
|
|
|
|
|
|
-&set_label("cbc_dec_tail_partial");
|
|
|
|
|
|
- &mov ($key_,"esp");
|
|
|
|
|
|
- &sub ("esp",16);
|
|
|
|
|
|
- &and ("esp",-16);
|
|
|
|
|
|
+&set_label("cbc_dec_tail_partial",16);
|
|
|
|
|
|
&movaps (&QWP(0,"esp"),$inout0);
|
|
|
|
|
|
+ &mov ("ecx",16);
|
|
|
|
|
|
&mov ($inp,"esp");
|
|
|
|
|
|
- &mov ("ecx",$len);
|
|
|
|
|
|
+ &sub ("ecx",$len);
|
|
|
|
|
|
&data_word(0xA4F3F689); # rep movsb
|
|
|
|
|
|
- &mov ("esp",$key_);
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("cbc_ret");
|
|
|
|
|
|
+ &mov ("esp",&DWP(16,"esp")); # pull original %esp
|
|
|
|
|
|
&mov ($key_,&wparam(4));
|
|
|
|
|
|
&movups (&QWP(0,$key_),$ivec); # output IV
|
|
|
|
|
|
+&set_label("cbc_abort");
|
|
|
|
|
|
&function_end("${PREFIX}_cbc_encrypt");
|
|
|
|
|
|
-
|
|
|
|
|
|
+
|
|
|
|
|
|
+######################################################################
|
|
|
|
|
|
# Mechanical port from aesni-x86_64.pl.
|
|
|
|
|
|
#
|
|
|
|
|
|
# _aesni_set_encrypt_key is private interface,
|
|
|
|
|
|
@@ -539,7 +812,7 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&jz (&label("bad_pointer"));
|
|
|
|
|
|
|
|
|
|
|
|
&movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey
|
|
|
|
|
|
- &pxor ("xmm4","xmm4"); # low dword of xmm4 is assumed 0
|
|
|
|
|
|
+ &xorps ("xmm4","xmm4"); # low dword of xmm4 is assumed 0
|
|
|
|
|
|
&lea ($key,&DWP(16,$key));
|
|
|
|
|
|
&cmp ($rounds,256);
|
|
|
|
|
|
&je (&label("14rounds"));
|
|
|
|
|
|
@@ -581,11 +854,11 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&lea ($key,&DWP(16,$key));
|
|
|
|
|
|
&set_label("key_128_cold");
|
|
|
|
|
|
&shufps ("xmm4","xmm0",0b00010000);
|
|
|
|
|
|
- &pxor ("xmm0","xmm4");
|
|
|
|
|
|
- &shufps ("xmm4","xmm0",0b10001100,);
|
|
|
|
|
|
- &pxor ("xmm0","xmm4");
|
|
|
|
|
|
- &pshufd ("xmm1","xmm1",0b11111111); # critical path
|
|
|
|
|
|
- &pxor ("xmm0","xmm1");
|
|
|
|
|
|
+ &xorps ("xmm0","xmm4");
|
|
|
|
|
|
+ &shufps ("xmm4","xmm0",0b10001100);
|
|
|
|
|
|
+ &xorps ("xmm0","xmm4");
|
|
|
|
|
|
+ &shufps ("xmm1","xmm1",0b11111111); # critical path
|
|
|
|
|
|
+ &xorps ("xmm0","xmm1");
|
|
|
|
|
|
&ret();
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("12rounds",16);
|
|
|
|
|
|
@@ -620,11 +893,11 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&movaps ("xmm5","xmm2");
|
|
|
|
|
|
&set_label("key_192b_warm");
|
|
|
|
|
|
&shufps ("xmm4","xmm0",0b00010000);
|
|
|
|
|
|
- &movaps ("xmm3","xmm2");
|
|
|
|
|
|
- &pxor ("xmm0","xmm4");
|
|
|
|
|
|
+ &movdqa ("xmm3","xmm2");
|
|
|
|
|
|
+ &xorps ("xmm0","xmm4");
|
|
|
|
|
|
&shufps ("xmm4","xmm0",0b10001100);
|
|
|
|
|
|
&pslldq ("xmm3",4);
|
|
|
|
|
|
- &pxor ("xmm0","xmm4");
|
|
|
|
|
|
+ &xorps ("xmm0","xmm4");
|
|
|
|
|
|
&pshufd ("xmm1","xmm1",0b01010101); # critical path
|
|
|
|
|
|
&pxor ("xmm2","xmm3");
|
|
|
|
|
|
&pxor ("xmm0","xmm1");
|
|
|
|
|
|
@@ -683,11 +956,11 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&lea ($key,&DWP(16,$key));
|
|
|
|
|
|
&set_label("key_256a_cold");
|
|
|
|
|
|
&shufps ("xmm4","xmm0",0b00010000);
|
|
|
|
|
|
- &pxor ("xmm0","xmm4");
|
|
|
|
|
|
+ &xorps ("xmm0","xmm4");
|
|
|
|
|
|
&shufps ("xmm4","xmm0",0b10001100);
|
|
|
|
|
|
- &pxor ("xmm0","xmm4");
|
|
|
|
|
|
- &pshufd ("xmm1","xmm1",0b11111111); # critical path
|
|
|
|
|
|
- &pxor ("xmm0","xmm1");
|
|
|
|
|
|
+ &xorps ("xmm0","xmm4");
|
|
|
|
|
|
+ &shufps ("xmm1","xmm1",0b11111111); # critical path
|
|
|
|
|
|
+ &xorps ("xmm0","xmm1");
|
|
|
|
|
|
&ret();
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("key_256b",16);
|
|
|
|
|
|
@@ -695,11 +968,11 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&lea ($key,&DWP(16,$key));
|
|
|
|
|
|
|
|
|
|
|
|
&shufps ("xmm4","xmm2",0b00010000);
|
|
|
|
|
|
- &pxor ("xmm2","xmm4");
|
|
|
|
|
|
+ &xorps ("xmm2","xmm4");
|
|
|
|
|
|
&shufps ("xmm4","xmm2",0b10001100);
|
|
|
|
|
|
- &pxor ("xmm2","xmm4");
|
|
|
|
|
|
- &pshufd ("xmm1","xmm1",0b10101010); # critical path
|
|
|
|
|
|
- &pxor ("xmm2","xmm1");
|
|
|
|
|
|
+ &xorps ("xmm2","xmm4");
|
|
|
|
|
|
+ &shufps ("xmm1","xmm1",0b10101010); # critical path
|
|
|
|
|
|
+ &xorps ("xmm2","xmm1");
|
|
|
|
|
|
&ret();
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("bad_pointer",4);
|
|
|
|
|
|
@@ -747,9 +1020,9 @@ if ($PREFIX eq "aesni") {
|
|
|
|
|
|
&aesimc ("xmm1","xmm1");
|
|
|
|
|
|
&lea ($key,&DWP(16,$key));
|
|
|
|
|
|
&lea ("eax",&DWP(-16,"eax"));
|
|
|
|
|
|
- &cmp ("eax",$key);
|
|
|
|
|
|
&$movekey (&QWP(16,"eax"),"xmm0");
|
|
|
|
|
|
&$movekey (&QWP(-16,$key),"xmm1");
|
|
|
|
|
|
+ &cmp ("eax",$key);
|
|
|
|
|
|
&ja (&label("dec_key_inverse"));
|
|
|
|
|
|
|
|
|
|
|
|
&$movekey ("xmm0",&QWP(0,$key)); # inverse middle
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/aes/asm/aesni-x86_64.pl.intelopts openssl-1.0.0d/crypto/aes/asm/aesni-x86_64.pl
|
2011-11-03 09:18:52 +00:00
|
|
|
|
--- openssl-1.0.0d/crypto/aes/asm/aesni-x86_64.pl.intelopts 2011-08-24 12:50:55.000000000 +0200
|
|
|
|
|
|
+++ openssl-1.0.0d/crypto/aes/asm/aesni-x86_64.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
#
|
|
|
|
|
|
# ====================================================================
|
|
|
|
|
|
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
|
|
|
|
|
|
@@ -11,6 +11,145 @@
|
|
|
|
|
|
# OpenSSL context it's used with Intel engine, but can also be used as
|
|
|
|
|
|
# drop-in replacement for crypto/aes/asm/aes-x86_64.pl [see below for
|
|
|
|
|
|
# details].
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Performance.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Given aes(enc|dec) instructions' latency asymptotic performance for
|
|
|
|
|
|
+# non-parallelizable modes such as CBC encrypt is 3.75 cycles per byte
|
|
|
|
|
|
+# processed with 128-bit key. And given their throughput asymptotic
|
|
|
|
|
|
+# performance for parallelizable modes is 1.25 cycles per byte. Being
|
|
|
|
|
|
+# asymptotic limit it's not something you commonly achieve in reality,
|
|
|
|
|
|
+# but how close does one get? Below are results collected for
|
|
|
|
|
|
+# different modes and block sized. Pairs of numbers are for en-/
|
|
|
|
|
|
+# decryption.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# 16-byte 64-byte 256-byte 1-KB 8-KB
|
|
|
|
|
|
+# ECB 4.25/4.25 1.38/1.38 1.28/1.28 1.26/1.26 1.26/1.26
|
|
|
|
|
|
+# CTR 5.42/5.42 1.92/1.92 1.44/1.44 1.28/1.28 1.26/1.26
|
|
|
|
|
|
+# CBC 4.38/4.43 4.15/1.43 4.07/1.32 4.07/1.29 4.06/1.28
|
|
|
|
|
|
+# CCM 5.66/9.42 4.42/5.41 4.16/4.40 4.09/4.15 4.06/4.07
|
|
|
|
|
|
+# OFB 5.42/5.42 4.64/4.64 4.44/4.44 4.39/4.39 4.38/4.38
|
|
|
|
|
|
+# CFB 5.73/5.85 5.56/5.62 5.48/5.56 5.47/5.55 5.47/5.55
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# ECB, CTR, CBC and CCM results are free from EVP overhead. This means
|
|
|
|
|
|
+# that otherwise used 'openssl speed -evp aes-128-??? -engine aesni
|
|
|
|
|
|
+# [-decrypt]' will exhibit 10-15% worse results for smaller blocks.
|
|
|
|
|
|
+# The results were collected with specially crafted speed.c benchmark
|
|
|
|
|
|
+# in order to compare them with results reported in "Intel Advanced
|
|
|
|
|
|
+# Encryption Standard (AES) New Instruction Set" White Paper Revision
|
|
|
|
|
|
+# 3.0 dated May 2010. All above results are consistently better. This
|
|
|
|
|
|
+# module also provides better performance for block sizes smaller than
|
|
|
|
|
|
+# 128 bytes in points *not* represented in the above table.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Looking at the results for 8-KB buffer.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# CFB and OFB results are far from the limit, because implementation
|
|
|
|
|
|
+# uses "generic" CRYPTO_[c|o]fb128_encrypt interfaces relying on
|
|
|
|
|
|
+# single-block aesni_encrypt, which is not the most optimal way to go.
|
|
|
|
|
|
+# CBC encrypt result is unexpectedly high and there is no documented
|
|
|
|
|
|
+# explanation for it. Seemingly there is a small penalty for feeding
|
|
|
|
|
|
+# the result back to AES unit the way it's done in CBC mode. There is
|
|
|
|
|
|
+# nothing one can do and the result appears optimal. CCM result is
|
|
|
|
|
|
+# identical to CBC, because CBC-MAC is essentially CBC encrypt without
|
|
|
|
|
|
+# saving output. CCM CTR "stays invisible," because it's neatly
|
|
|
|
|
|
+# interleaved wih CBC-MAC. This provides ~30% improvement over
|
|
|
|
|
|
+# "straghtforward" CCM implementation with CTR and CBC-MAC performed
|
|
|
|
|
|
+# disjointly. Parallelizable modes practically achieve the theoretical
|
|
|
|
|
|
+# limit.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Looking at how results vary with buffer size.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Curves are practically saturated at 1-KB buffer size. In most cases
|
|
|
|
|
|
+# "256-byte" performance is >95%, and "64-byte" is ~90% of "8-KB" one.
|
|
|
|
|
|
+# CTR curve doesn't follow this pattern and is "slowest" changing one
|
|
|
|
|
|
+# with "256-byte" result being 87% of "8-KB." This is because overhead
|
|
|
|
|
|
+# in CTR mode is most computationally intensive. Small-block CCM
|
|
|
|
|
|
+# decrypt is slower than encrypt, because first CTR and last CBC-MAC
|
|
|
|
|
|
+# iterations can't be interleaved.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Results for 192- and 256-bit keys.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# EVP-free results were observed to scale perfectly with number of
|
|
|
|
|
|
+# rounds for larger block sizes, i.e. 192-bit result being 10/12 times
|
|
|
|
|
|
+# lower and 256-bit one - 10/14. Well, in CBC encrypt case differences
|
|
|
|
|
|
+# are a tad smaller, because the above mentioned penalty biases all
|
|
|
|
|
|
+# results by same constant value. In similar way function call
|
|
|
|
|
|
+# overhead affects small-block performance, as well as OFB and CFB
|
|
|
|
|
|
+# results. Differences are not large, most common coefficients are
|
|
|
|
|
|
+# 10/11.7 and 10/13.4 (as opposite to 10/12.0 and 10/14.0), but one
|
|
|
|
|
|
+# observe even 10/11.2 and 10/12.4 (CTR, OFB, CFB)...
|
|
|
|
|
|
+
|
|
|
|
|
|
+# January 2011
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# While Westmere processor features 6 cycles latency for aes[enc|dec]
|
|
|
|
|
|
+# instructions, which can be scheduled every second cycle, Sandy
|
|
|
|
|
|
+# Bridge spends 8 cycles per instruction, but it can schedule them
|
|
|
|
|
|
+# every cycle. This means that code targeting Westmere would perform
|
|
|
|
|
|
+# suboptimally on Sandy Bridge. Therefore this update.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# In addition, non-parallelizable CBC encrypt (as well as CCM) is
|
|
|
|
|
|
+# optimized. Relative improvement might appear modest, 8% on Westmere,
|
|
|
|
|
|
+# but in absolute terms it's 3.77 cycles per byte encrypted with
|
|
|
|
|
|
+# 128-bit key on Westmere, and 5.07 - on Sandy Bridge. These numbers
|
|
|
|
|
|
+# should be compared to asymptotic limits of 3.75 for Westmere and
|
|
|
|
|
|
+# 5.00 for Sandy Bridge. Actually, the fact that they get this close
|
|
|
|
|
|
+# to asymptotic limits is quite amazing. Indeed, the limit is
|
|
|
|
|
|
+# calculated as latency times number of rounds, 10 for 128-bit key,
|
|
|
|
|
|
+# and divided by 16, the number of bytes in block, or in other words
|
|
|
|
|
|
+# it accounts *solely* for aesenc instructions. But there are extra
|
|
|
|
|
|
+# instructions, and numbers so close to the asymptotic limits mean
|
|
|
|
|
|
+# that it's as if it takes as little as *one* additional cycle to
|
|
|
|
|
|
+# execute all of them. How is it possible? It is possible thanks to
|
|
|
|
|
|
+# out-of-order execution logic, which manages to overlap post-
|
|
|
|
|
|
+# processing of previous block, things like saving the output, with
|
|
|
|
|
|
+# actual encryption of current block, as well as pre-processing of
|
|
|
|
|
|
+# current block, things like fetching input and xor-ing it with
|
|
|
|
|
|
+# 0-round element of the key schedule, with actual encryption of
|
|
|
|
|
|
+# previous block. Keep this in mind...
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# For parallelizable modes, such as ECB, CBC decrypt, CTR, higher
|
|
|
|
|
|
+# performance is achieved by interleaving instructions working on
|
|
|
|
|
|
+# independent blocks. In which case asymptotic limit for such modes
|
|
|
|
|
|
+# can be obtained by dividing above mentioned numbers by AES
|
|
|
|
|
|
+# instructions' interleave factor. Westmere can execute at most 3
|
|
|
|
|
|
+# instructions at a time, meaning that optimal interleave factor is 3,
|
|
|
|
|
|
+# and that's where the "magic" number of 1.25 come from. "Optimal
|
|
|
|
|
|
+# interleave factor" means that increase of interleave factor does
|
|
|
|
|
|
+# not improve performance. The formula has proven to reflect reality
|
|
|
|
|
|
+# pretty well on Westmere... Sandy Bridge on the other hand can
|
|
|
|
|
|
+# execute up to 8 AES instructions at a time, so how does varying
|
|
|
|
|
|
+# interleave factor affect the performance? Here is table for ECB
|
|
|
|
|
|
+# (numbers are cycles per byte processed with 128-bit key):
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# instruction interleave factor 3x 6x 8x
|
|
|
|
|
|
+# theoretical asymptotic limit 1.67 0.83 0.625
|
|
|
|
|
|
+# measured performance for 8KB block 1.05 0.86 0.84
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# "as if" interleave factor 4.7x 5.8x 6.0x
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Further data for other parallelizable modes:
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# CBC decrypt 1.16 0.93 0.93
|
|
|
|
|
|
+# CTR 1.14 0.91 n/a
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Well, given 3x column it's probably inappropriate to call the limit
|
|
|
|
|
|
+# asymptotic, if it can be surpassed, isn't it? What happens there?
|
|
|
|
|
|
+# Rewind to CBC paragraph for the answer. Yes, out-of-order execution
|
|
|
|
|
|
+# magic is responsible for this. Processor overlaps not only the
|
|
|
|
|
|
+# additional instructions with AES ones, but even AES instuctions
|
|
|
|
|
|
+# processing adjacent triplets of independent blocks. In the 6x case
|
|
|
|
|
|
+# additional instructions still claim disproportionally small amount
|
|
|
|
|
|
+# of additional cycles, but in 8x case number of instructions must be
|
|
|
|
|
|
+# a tad too high for out-of-order logic to cope with, and AES unit
|
|
|
|
|
|
+# remains underutilized... As you can see 8x interleave is hardly
|
|
|
|
|
|
+# justifiable, so there no need to feel bad that 32-bit aesni-x86.pl
|
|
|
|
|
|
+# utilizies 6x interleave because of limited register bank capacity.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Higher interleave factors do have negative impact on Westmere
|
|
|
|
|
|
+# performance. While for ECB mode it's negligible ~1.5%, other
|
|
|
|
|
|
+# parallelizables perform ~5% worse, which is outweighed by ~25%
|
|
|
|
|
|
+# improvement on Sandy Bridge. To balance regression on Westmere
|
|
|
|
|
|
+# CTR mode was implemented with 6x aesenc interleave factor.
|
|
|
|
|
|
|
|
|
|
|
|
$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
|
|
|
|
|
|
# generates drop-in replacement for
|
|
|
|
|
|
@@ -29,7 +168,7 @@ die "can't locate x86_64-xlate.pl";
|
|
|
|
|
|
|
|
|
|
|
|
open STDOUT,"| $^X $xlate $flavour $output";
|
|
|
|
|
|
|
|
|
|
|
|
-$movkey = $PREFIX eq "aesni" ? "movaps" : "movups";
|
|
|
|
|
|
+$movkey = $PREFIX eq "aesni" ? "movups" : "movups";
|
|
|
|
|
|
@_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order
|
|
|
|
|
|
("%rdi","%rsi","%rdx","%rcx"); # Unix order
|
|
|
|
|
|
|
|
|
|
|
|
@@ -41,18 +180,20 @@ $inp="%rdi";
|
|
|
|
|
|
$out="%rsi";
|
|
|
|
|
|
$len="%rdx";
|
|
|
|
|
|
$key="%rcx"; # input to and changed by aesni_[en|de]cryptN !!!
|
|
|
|
|
|
-$ivp="%r8"; # cbc
|
|
|
|
|
|
+$ivp="%r8"; # cbc, ctr, ...
|
|
|
|
|
|
|
|
|
|
|
|
$rnds_="%r10d"; # backup copy for $rounds
|
|
|
|
|
|
$key_="%r11"; # backup copy for $key
|
|
|
|
|
|
|
|
|
|
|
|
# %xmm register layout
|
|
|
|
|
|
-$inout0="%xmm0"; $inout1="%xmm1";
|
|
|
|
|
|
-$inout2="%xmm2"; $inout3="%xmm3";
|
|
|
|
|
|
-$rndkey0="%xmm4"; $rndkey1="%xmm5";
|
|
|
|
|
|
+$rndkey0="%xmm0"; $rndkey1="%xmm1";
|
|
|
|
|
|
+$inout0="%xmm2"; $inout1="%xmm3";
|
|
|
|
|
|
+$inout2="%xmm4"; $inout3="%xmm5";
|
|
|
|
|
|
+$inout4="%xmm6"; $inout5="%xmm7";
|
|
|
|
|
|
+$inout6="%xmm8"; $inout7="%xmm9";
|
|
|
|
|
|
|
|
|
|
|
|
-$iv="%xmm6"; $in0="%xmm7"; # used in CBC decrypt
|
|
|
|
|
|
-$in1="%xmm8"; $in2="%xmm9";
|
|
|
|
|
|
+$in2="%xmm6"; $in1="%xmm7"; # used in CBC decrypt, CTR, ...
|
|
|
|
|
|
+$in0="%xmm8"; $iv="%xmm9";
|
|
|
|
|
|
|
|
|
|
|
|
# Inline version of internal aesni_[en|de]crypt1.
|
|
|
|
|
|
#
|
|
|
|
|
|
@@ -60,20 +201,29 @@ $in1="%xmm8"; $in2="%xmm9";
|
|
|
|
|
|
# cycles which take care of loop variables...
|
|
|
|
|
|
{ my $sn;
|
|
|
|
|
|
sub aesni_generate1 {
|
|
|
|
|
|
-my ($p,$key,$rounds)=@_;
|
|
|
|
|
|
+my ($p,$key,$rounds,$inout,$ivec)=@_; $inout=$inout0 if (!defined($inout));
|
|
|
|
|
|
++$sn;
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
$movkey ($key),$rndkey0
|
|
|
|
|
|
$movkey 16($key),$rndkey1
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___ if (defined($ivec));
|
|
|
|
|
|
+ xorps $rndkey0,$ivec
|
|
|
|
|
|
lea 32($key),$key
|
|
|
|
|
|
- pxor $rndkey0,$inout0
|
|
|
|
|
|
+ xorps $ivec,$inout
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___ if (!defined($ivec));
|
|
|
|
|
|
+ lea 32($key),$key
|
|
|
|
|
|
+ xorps $rndkey0,$inout
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
.Loop_${p}1_$sn:
|
|
|
|
|
|
- aes${p} $rndkey1,$inout0
|
|
|
|
|
|
+ aes${p} $rndkey1,$inout
|
|
|
|
|
|
dec $rounds
|
|
|
|
|
|
$movkey ($key),$rndkey1
|
|
|
|
|
|
lea 16($key),$key
|
|
|
|
|
|
jnz .Loop_${p}1_$sn # loop body is 16 bytes
|
|
|
|
|
|
- aes${p}last $rndkey1,$inout0
|
|
|
|
|
|
+ aes${p}last $rndkey1,$inout
|
|
|
|
|
|
___
|
|
|
|
|
|
}}
|
|
|
|
|
|
# void $PREFIX_[en|de]crypt (const void *inp,void *out,const AES_KEY *key);
|
|
|
|
|
|
@@ -86,7 +236,7 @@ $code.=<<___;
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
${PREFIX}_encrypt:
|
|
|
|
|
|
movups ($inp),$inout0 # load input
|
|
|
|
|
|
- mov 240($key),$rounds # pull $rounds
|
|
|
|
|
|
+ mov 240($key),$rounds # key->rounds
|
|
|
|
|
|
___
|
|
|
|
|
|
&aesni_generate1("enc",$key,$rounds);
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
@@ -99,7 +249,7 @@ $code.=<<___;
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
${PREFIX}_decrypt:
|
|
|
|
|
|
movups ($inp),$inout0 # load input
|
|
|
|
|
|
- mov 240($key),$rounds # pull $rounds
|
|
|
|
|
|
+ mov 240($key),$rounds # key->rounds
|
|
|
|
|
|
___
|
|
|
|
|
|
&aesni_generate1("dec",$key,$rounds);
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
@@ -109,16 +259,16 @@ $code.=<<___;
|
|
|
|
|
|
___
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
-# _aesni_[en|de]crypt[34] are private interfaces, N denotes interleave
|
|
|
|
|
|
-# factor. Why 3x subroutine is used in loops? Even though aes[enc|dec]
|
|
|
|
|
|
-# latency is 6, it turned out that it can be scheduled only every
|
|
|
|
|
|
-# *second* cycle. Thus 3x interleave is the one providing optimal
|
|
|
|
|
|
+# _aesni_[en|de]cryptN are private interfaces, N denotes interleave
|
|
|
|
|
|
+# factor. Why 3x subroutine were originally used in loops? Even though
|
|
|
|
|
|
+# aes[enc|dec] latency was originally 6, it could be scheduled only
|
|
|
|
|
|
+# every *2nd* cycle. Thus 3x interleave was the one providing optimal
|
|
|
|
|
|
# utilization, i.e. when subroutine's throughput is virtually same as
|
|
|
|
|
|
# of non-interleaved subroutine [for number of input blocks up to 3].
|
|
|
|
|
|
-# This is why it makes no sense to implement 2x subroutine. As soon
|
|
|
|
|
|
-# as/if Intel improves throughput by making it possible to schedule
|
|
|
|
|
|
-# the instructions in question *every* cycles I would have to
|
|
|
|
|
|
-# implement 6x interleave and use it in loop...
|
|
|
|
|
|
+# This is why it makes no sense to implement 2x subroutine.
|
|
|
|
|
|
+# aes[enc|dec] latency in next processor generation is 8, but the
|
|
|
|
|
|
+# instructions can be scheduled every cycle. Optimal interleave for
|
|
|
|
|
|
+# new processor is therefore 8x...
|
|
|
|
|
|
sub aesni_generate3 {
|
|
|
|
|
|
my $dir=shift;
|
|
|
|
|
|
# As already mentioned it takes in $key and $rounds, which are *not*
|
|
|
|
|
|
@@ -131,25 +281,25 @@ _aesni_${dir}rypt3:
|
|
|
|
|
|
shr \$1,$rounds
|
|
|
|
|
|
$movkey 16($key),$rndkey1
|
|
|
|
|
|
lea 32($key),$key
|
|
|
|
|
|
- pxor $rndkey0,$inout0
|
|
|
|
|
|
- pxor $rndkey0,$inout1
|
|
|
|
|
|
- pxor $rndkey0,$inout2
|
|
|
|
|
|
+ xorps $rndkey0,$inout0
|
|
|
|
|
|
+ xorps $rndkey0,$inout1
|
|
|
|
|
|
+ xorps $rndkey0,$inout2
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
|
|
|
|
|
|
.L${dir}_loop3:
|
|
|
|
|
|
aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
- $movkey ($key),$rndkey0
|
|
|
|
|
|
aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
dec $rounds
|
|
|
|
|
|
aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
- aes${dir} $rndkey0,$inout0
|
|
|
|
|
|
$movkey 16($key),$rndkey1
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout0
|
|
|
|
|
|
aes${dir} $rndkey0,$inout1
|
|
|
|
|
|
lea 32($key),$key
|
|
|
|
|
|
aes${dir} $rndkey0,$inout2
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
jnz .L${dir}_loop3
|
|
|
|
|
|
|
|
|
|
|
|
aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
- $movkey ($key),$rndkey0
|
|
|
|
|
|
aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
aes${dir}last $rndkey0,$inout0
|
|
|
|
|
|
@@ -175,28 +325,28 @@ _aesni_${dir}rypt4:
|
|
|
|
|
|
shr \$1,$rounds
|
|
|
|
|
|
$movkey 16($key),$rndkey1
|
|
|
|
|
|
lea 32($key),$key
|
|
|
|
|
|
- pxor $rndkey0,$inout0
|
|
|
|
|
|
- pxor $rndkey0,$inout1
|
|
|
|
|
|
- pxor $rndkey0,$inout2
|
|
|
|
|
|
- pxor $rndkey0,$inout3
|
|
|
|
|
|
+ xorps $rndkey0,$inout0
|
|
|
|
|
|
+ xorps $rndkey0,$inout1
|
|
|
|
|
|
+ xorps $rndkey0,$inout2
|
|
|
|
|
|
+ xorps $rndkey0,$inout3
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
|
|
|
|
|
|
.L${dir}_loop4:
|
|
|
|
|
|
aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
- $movkey ($key),$rndkey0
|
|
|
|
|
|
aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
dec $rounds
|
|
|
|
|
|
aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
aes${dir} $rndkey1,$inout3
|
|
|
|
|
|
- aes${dir} $rndkey0,$inout0
|
|
|
|
|
|
$movkey 16($key),$rndkey1
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout0
|
|
|
|
|
|
aes${dir} $rndkey0,$inout1
|
|
|
|
|
|
lea 32($key),$key
|
|
|
|
|
|
aes${dir} $rndkey0,$inout2
|
|
|
|
|
|
aes${dir} $rndkey0,$inout3
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
jnz .L${dir}_loop4
|
|
|
|
|
|
|
|
|
|
|
|
aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
- $movkey ($key),$rndkey0
|
|
|
|
|
|
aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
aes${dir} $rndkey1,$inout3
|
|
|
|
|
|
@@ -208,12 +358,158 @@ _aesni_${dir}rypt4:
|
|
|
|
|
|
.size _aesni_${dir}rypt4,.-_aesni_${dir}rypt4
|
|
|
|
|
|
___
|
|
|
|
|
|
}
|
|
|
|
|
|
+sub aesni_generate6 {
|
|
|
|
|
|
+my $dir=shift;
|
|
|
|
|
|
+# As already mentioned it takes in $key and $rounds, which are *not*
|
|
|
|
|
|
+# preserved. $inout[0-5] is cipher/clear text...
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+.type _aesni_${dir}rypt6,\@abi-omnipotent
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+_aesni_${dir}rypt6:
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
+ shr \$1,$rounds
|
|
|
|
|
|
+ $movkey 16($key),$rndkey1
|
|
|
|
|
|
+ lea 32($key),$key
|
|
|
|
|
|
+ xorps $rndkey0,$inout0
|
|
|
|
|
|
+ pxor $rndkey0,$inout1
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
+ pxor $rndkey0,$inout2
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
+ pxor $rndkey0,$inout3
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
+ pxor $rndkey0,$inout4
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout3
|
|
|
|
|
|
+ pxor $rndkey0,$inout5
|
|
|
|
|
|
+ dec $rounds
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout4
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout5
|
|
|
|
|
|
+ jmp .L${dir}_loop6_enter
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.L${dir}_loop6:
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
+ dec $rounds
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout3
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout4
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout5
|
|
|
|
|
|
+.L${dir}_loop6_enter: # happens to be 16-byte aligned
|
|
|
|
|
|
+ $movkey 16($key),$rndkey1
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout0
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout1
|
|
|
|
|
|
+ lea 32($key),$key
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout2
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout3
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout4
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout5
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
+ jnz .L${dir}_loop6
|
|
|
|
|
|
+
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout3
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout4
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout5
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout0
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout1
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout2
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout3
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout4
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout5
|
|
|
|
|
|
+ ret
|
|
|
|
|
|
+.size _aesni_${dir}rypt6,.-_aesni_${dir}rypt6
|
|
|
|
|
|
+___
|
|
|
|
|
|
+}
|
|
|
|
|
|
+sub aesni_generate8 {
|
|
|
|
|
|
+my $dir=shift;
|
|
|
|
|
|
+# As already mentioned it takes in $key and $rounds, which are *not*
|
|
|
|
|
|
+# preserved. $inout[0-7] is cipher/clear text...
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+.type _aesni_${dir}rypt8,\@abi-omnipotent
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+_aesni_${dir}rypt8:
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
+ shr \$1,$rounds
|
|
|
|
|
|
+ $movkey 16($key),$rndkey1
|
|
|
|
|
|
+ lea 32($key),$key
|
|
|
|
|
|
+ xorps $rndkey0,$inout0
|
|
|
|
|
|
+ xorps $rndkey0,$inout1
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
+ pxor $rndkey0,$inout2
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
+ pxor $rndkey0,$inout3
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
+ pxor $rndkey0,$inout4
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout3
|
|
|
|
|
|
+ pxor $rndkey0,$inout5
|
|
|
|
|
|
+ dec $rounds
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout4
|
|
|
|
|
|
+ pxor $rndkey0,$inout6
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout5
|
|
|
|
|
|
+ pxor $rndkey0,$inout7
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout6
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout7
|
|
|
|
|
|
+ $movkey 16($key),$rndkey1
|
|
|
|
|
|
+ jmp .L${dir}_loop8_enter
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.L${dir}_loop8:
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
+ dec $rounds
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout3
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout4
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout5
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout6
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout7
|
|
|
|
|
|
+ $movkey 16($key),$rndkey1
|
|
|
|
|
|
+.L${dir}_loop8_enter: # happens to be 16-byte aligned
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout0
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout1
|
|
|
|
|
|
+ lea 32($key),$key
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout2
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout3
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout4
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout5
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout6
|
|
|
|
|
|
+ aes${dir} $rndkey0,$inout7
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
+ jnz .L${dir}_loop8
|
|
|
|
|
|
+
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout0
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout1
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout2
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout3
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout4
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout5
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout6
|
|
|
|
|
|
+ aes${dir} $rndkey1,$inout7
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout0
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout1
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout2
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout3
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout4
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout5
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout6
|
|
|
|
|
|
+ aes${dir}last $rndkey0,$inout7
|
|
|
|
|
|
+ ret
|
|
|
|
|
|
+.size _aesni_${dir}rypt8,.-_aesni_${dir}rypt8
|
|
|
|
|
|
+___
|
|
|
|
|
|
+}
|
|
|
|
|
|
&aesni_generate3("enc") if ($PREFIX eq "aesni");
|
|
|
|
|
|
&aesni_generate3("dec");
|
|
|
|
|
|
&aesni_generate4("enc") if ($PREFIX eq "aesni");
|
|
|
|
|
|
&aesni_generate4("dec");
|
|
|
|
|
|
+&aesni_generate6("enc") if ($PREFIX eq "aesni");
|
|
|
|
|
|
+&aesni_generate6("dec");
|
|
|
|
|
|
+&aesni_generate8("enc") if ($PREFIX eq "aesni");
|
|
|
|
|
|
+&aesni_generate8("dec");
|
|
|
|
|
|
|
|
|
|
|
|
if ($PREFIX eq "aesni") {
|
|
|
|
|
|
+########################################################################
|
|
|
|
|
|
# void aesni_ecb_encrypt (const void *in, void *out,
|
|
|
|
|
|
# size_t length, const AES_KEY *key,
|
|
|
|
|
|
# int enc);
|
|
|
|
|
|
@@ -222,54 +518,98 @@ $code.=<<___;
|
|
|
|
|
|
.type aesni_ecb_encrypt,\@function,5
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
aesni_ecb_encrypt:
|
|
|
|
|
|
- cmp \$16,$len # check length
|
|
|
|
|
|
- jb .Lecb_ret
|
|
|
|
|
|
-
|
|
|
|
|
|
- mov 240($key),$rounds # pull $rounds
|
|
|
|
|
|
and \$-16,$len
|
|
|
|
|
|
+ jz .Lecb_ret
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov 240($key),$rounds # key->rounds
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
mov $key,$key_ # backup $key
|
|
|
|
|
|
- test %r8d,%r8d # 5th argument
|
|
|
|
|
|
mov $rounds,$rnds_ # backup $rounds
|
|
|
|
|
|
+ test %r8d,%r8d # 5th argument
|
|
|
|
|
|
jz .Lecb_decrypt
|
|
|
|
|
|
#--------------------------- ECB ENCRYPT ------------------------------#
|
|
|
|
|
|
- sub \$0x40,$len
|
|
|
|
|
|
- jbe .Lecb_enc_tail
|
|
|
|
|
|
- jmp .Lecb_enc_loop3
|
|
|
|
|
|
+ cmp \$0x80,$len
|
|
|
|
|
|
+ jb .Lecb_enc_tail
|
|
|
|
|
|
+
|
|
|
|
|
|
+ movdqu ($inp),$inout0
|
|
|
|
|
|
+ movdqu 0x10($inp),$inout1
|
|
|
|
|
|
+ movdqu 0x20($inp),$inout2
|
|
|
|
|
|
+ movdqu 0x30($inp),$inout3
|
|
|
|
|
|
+ movdqu 0x40($inp),$inout4
|
|
|
|
|
|
+ movdqu 0x50($inp),$inout5
|
|
|
|
|
|
+ movdqu 0x60($inp),$inout6
|
|
|
|
|
|
+ movdqu 0x70($inp),$inout7
|
|
|
|
|
|
+ lea 0x80($inp),$inp
|
|
|
|
|
|
+ sub \$0x80,$len
|
|
|
|
|
|
+ jmp .Lecb_enc_loop8_enter
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
-.Lecb_enc_loop3:
|
|
|
|
|
|
- movups ($inp),$inout0
|
|
|
|
|
|
- movups 0x10($inp),$inout1
|
|
|
|
|
|
- movups 0x20($inp),$inout2
|
|
|
|
|
|
- call _aesni_encrypt3
|
|
|
|
|
|
- sub \$0x30,$len
|
|
|
|
|
|
- lea 0x30($inp),$inp
|
|
|
|
|
|
- lea 0x30($out),$out
|
|
|
|
|
|
- movups $inout0,-0x30($out)
|
|
|
|
|
|
- mov $rnds_,$rounds # restore $rounds
|
|
|
|
|
|
- movups $inout1,-0x20($out)
|
|
|
|
|
|
+.Lecb_enc_loop8:
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
mov $key_,$key # restore $key
|
|
|
|
|
|
- movups $inout2,-0x10($out)
|
|
|
|
|
|
- ja .Lecb_enc_loop3
|
|
|
|
|
|
+ movdqu ($inp),$inout0
|
|
|
|
|
|
+ mov $rnds_,$rounds # restore $rounds
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movdqu 0x10($inp),$inout1
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movdqu 0x20($inp),$inout2
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movdqu 0x30($inp),$inout3
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ movdqu 0x40($inp),$inout4
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
+ movdqu 0x50($inp),$inout5
|
|
|
|
|
|
+ movups $inout6,0x60($out)
|
|
|
|
|
|
+ movdqu 0x60($inp),$inout6
|
|
|
|
|
|
+ movups $inout7,0x70($out)
|
|
|
|
|
|
+ lea 0x80($out),$out
|
|
|
|
|
|
+ movdqu 0x70($inp),$inout7
|
|
|
|
|
|
+ lea 0x80($inp),$inp
|
|
|
|
|
|
+.Lecb_enc_loop8_enter:
|
|
|
|
|
|
|
|
|
|
|
|
-.Lecb_enc_tail:
|
|
|
|
|
|
- add \$0x40,$len
|
|
|
|
|
|
+ call _aesni_encrypt8
|
|
|
|
|
|
+
|
|
|
|
|
|
+ sub \$0x80,$len
|
|
|
|
|
|
+ jnc .Lecb_enc_loop8
|
|
|
|
|
|
+
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ mov $key_,$key # restore $key
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ mov $rnds_,$rounds # restore $rounds
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
+ movups $inout6,0x60($out)
|
|
|
|
|
|
+ movups $inout7,0x70($out)
|
|
|
|
|
|
+ lea 0x80($out),$out
|
|
|
|
|
|
+ add \$0x80,$len
|
|
|
|
|
|
jz .Lecb_ret
|
|
|
|
|
|
|
|
|
|
|
|
- cmp \$0x10,$len
|
|
|
|
|
|
+.Lecb_enc_tail:
|
|
|
|
|
|
movups ($inp),$inout0
|
|
|
|
|
|
- je .Lecb_enc_one
|
|
|
|
|
|
cmp \$0x20,$len
|
|
|
|
|
|
+ jb .Lecb_enc_one
|
|
|
|
|
|
movups 0x10($inp),$inout1
|
|
|
|
|
|
je .Lecb_enc_two
|
|
|
|
|
|
- cmp \$0x30,$len
|
|
|
|
|
|
movups 0x20($inp),$inout2
|
|
|
|
|
|
- je .Lecb_enc_three
|
|
|
|
|
|
+ cmp \$0x40,$len
|
|
|
|
|
|
+ jb .Lecb_enc_three
|
|
|
|
|
|
movups 0x30($inp),$inout3
|
|
|
|
|
|
- call _aesni_encrypt4
|
|
|
|
|
|
+ je .Lecb_enc_four
|
|
|
|
|
|
+ movups 0x40($inp),$inout4
|
|
|
|
|
|
+ cmp \$0x60,$len
|
|
|
|
|
|
+ jb .Lecb_enc_five
|
|
|
|
|
|
+ movups 0x50($inp),$inout5
|
|
|
|
|
|
+ je .Lecb_enc_six
|
|
|
|
|
|
+ movdqu 0x60($inp),$inout6
|
|
|
|
|
|
+ call _aesni_encrypt8
|
|
|
|
|
|
movups $inout0,($out)
|
|
|
|
|
|
movups $inout1,0x10($out)
|
|
|
|
|
|
movups $inout2,0x20($out)
|
|
|
|
|
|
movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
+ movups $inout6,0x60($out)
|
|
|
|
|
|
jmp .Lecb_ret
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lecb_enc_one:
|
|
|
|
|
|
@@ -280,6 +620,7 @@ $code.=<<___;
|
|
|
|
|
|
jmp .Lecb_ret
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lecb_enc_two:
|
|
|
|
|
|
+ xorps $inout2,$inout2
|
|
|
|
|
|
call _aesni_encrypt3
|
|
|
|
|
|
movups $inout0,($out)
|
|
|
|
|
|
movups $inout1,0x10($out)
|
|
|
|
|
|
@@ -291,47 +632,121 @@ $code.=<<___;
|
|
|
|
|
|
movups $inout1,0x10($out)
|
|
|
|
|
|
movups $inout2,0x20($out)
|
|
|
|
|
|
jmp .Lecb_ret
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lecb_enc_four:
|
|
|
|
|
|
+ call _aesni_encrypt4
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ jmp .Lecb_ret
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lecb_enc_five:
|
|
|
|
|
|
+ xorps $inout5,$inout5
|
|
|
|
|
|
+ call _aesni_encrypt6
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ jmp .Lecb_ret
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lecb_enc_six:
|
|
|
|
|
|
+ call _aesni_encrypt6
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
+ jmp .Lecb_ret
|
|
|
|
|
|
#--------------------------- ECB DECRYPT ------------------------------#
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lecb_decrypt:
|
|
|
|
|
|
- sub \$0x40,$len
|
|
|
|
|
|
- jbe .Lecb_dec_tail
|
|
|
|
|
|
- jmp .Lecb_dec_loop3
|
|
|
|
|
|
+ cmp \$0x80,$len
|
|
|
|
|
|
+ jb .Lecb_dec_tail
|
|
|
|
|
|
+
|
|
|
|
|
|
+ movdqu ($inp),$inout0
|
|
|
|
|
|
+ movdqu 0x10($inp),$inout1
|
|
|
|
|
|
+ movdqu 0x20($inp),$inout2
|
|
|
|
|
|
+ movdqu 0x30($inp),$inout3
|
|
|
|
|
|
+ movdqu 0x40($inp),$inout4
|
|
|
|
|
|
+ movdqu 0x50($inp),$inout5
|
|
|
|
|
|
+ movdqu 0x60($inp),$inout6
|
|
|
|
|
|
+ movdqu 0x70($inp),$inout7
|
|
|
|
|
|
+ lea 0x80($inp),$inp
|
|
|
|
|
|
+ sub \$0x80,$len
|
|
|
|
|
|
+ jmp .Lecb_dec_loop8_enter
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
-.Lecb_dec_loop3:
|
|
|
|
|
|
- movups ($inp),$inout0
|
|
|
|
|
|
- movups 0x10($inp),$inout1
|
|
|
|
|
|
- movups 0x20($inp),$inout2
|
|
|
|
|
|
- call _aesni_decrypt3
|
|
|
|
|
|
- sub \$0x30,$len
|
|
|
|
|
|
- lea 0x30($inp),$inp
|
|
|
|
|
|
- lea 0x30($out),$out
|
|
|
|
|
|
- movups $inout0,-0x30($out)
|
|
|
|
|
|
- mov $rnds_,$rounds # restore $rounds
|
|
|
|
|
|
- movups $inout1,-0x20($out)
|
|
|
|
|
|
+.Lecb_dec_loop8:
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
mov $key_,$key # restore $key
|
|
|
|
|
|
- movups $inout2,-0x10($out)
|
|
|
|
|
|
- ja .Lecb_dec_loop3
|
|
|
|
|
|
+ movdqu ($inp),$inout0
|
|
|
|
|
|
+ mov $rnds_,$rounds # restore $rounds
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movdqu 0x10($inp),$inout1
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movdqu 0x20($inp),$inout2
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movdqu 0x30($inp),$inout3
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ movdqu 0x40($inp),$inout4
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
+ movdqu 0x50($inp),$inout5
|
|
|
|
|
|
+ movups $inout6,0x60($out)
|
|
|
|
|
|
+ movdqu 0x60($inp),$inout6
|
|
|
|
|
|
+ movups $inout7,0x70($out)
|
|
|
|
|
|
+ lea 0x80($out),$out
|
|
|
|
|
|
+ movdqu 0x70($inp),$inout7
|
|
|
|
|
|
+ lea 0x80($inp),$inp
|
|
|
|
|
|
+.Lecb_dec_loop8_enter:
|
|
|
|
|
|
+
|
|
|
|
|
|
+ call _aesni_decrypt8
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $movkey ($key_),$rndkey0
|
|
|
|
|
|
+ sub \$0x80,$len
|
|
|
|
|
|
+ jnc .Lecb_dec_loop8
|
|
|
|
|
|
|
|
|
|
|
|
-.Lecb_dec_tail:
|
|
|
|
|
|
- add \$0x40,$len
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ mov $key_,$key # restore $key
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ mov $rnds_,$rounds # restore $rounds
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
+ movups $inout6,0x60($out)
|
|
|
|
|
|
+ movups $inout7,0x70($out)
|
|
|
|
|
|
+ lea 0x80($out),$out
|
|
|
|
|
|
+ add \$0x80,$len
|
|
|
|
|
|
jz .Lecb_ret
|
|
|
|
|
|
|
|
|
|
|
|
- cmp \$0x10,$len
|
|
|
|
|
|
+.Lecb_dec_tail:
|
|
|
|
|
|
movups ($inp),$inout0
|
|
|
|
|
|
- je .Lecb_dec_one
|
|
|
|
|
|
cmp \$0x20,$len
|
|
|
|
|
|
+ jb .Lecb_dec_one
|
|
|
|
|
|
movups 0x10($inp),$inout1
|
|
|
|
|
|
je .Lecb_dec_two
|
|
|
|
|
|
- cmp \$0x30,$len
|
|
|
|
|
|
movups 0x20($inp),$inout2
|
|
|
|
|
|
- je .Lecb_dec_three
|
|
|
|
|
|
+ cmp \$0x40,$len
|
|
|
|
|
|
+ jb .Lecb_dec_three
|
|
|
|
|
|
movups 0x30($inp),$inout3
|
|
|
|
|
|
- call _aesni_decrypt4
|
|
|
|
|
|
+ je .Lecb_dec_four
|
|
|
|
|
|
+ movups 0x40($inp),$inout4
|
|
|
|
|
|
+ cmp \$0x60,$len
|
|
|
|
|
|
+ jb .Lecb_dec_five
|
|
|
|
|
|
+ movups 0x50($inp),$inout5
|
|
|
|
|
|
+ je .Lecb_dec_six
|
|
|
|
|
|
+ movups 0x60($inp),$inout6
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
+ call _aesni_decrypt8
|
|
|
|
|
|
movups $inout0,($out)
|
|
|
|
|
|
movups $inout1,0x10($out)
|
|
|
|
|
|
movups $inout2,0x20($out)
|
|
|
|
|
|
movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
+ movups $inout6,0x60($out)
|
|
|
|
|
|
jmp .Lecb_ret
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lecb_dec_one:
|
|
|
|
|
|
@@ -342,6 +757,7 @@ $code.=<<___;
|
|
|
|
|
|
jmp .Lecb_ret
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lecb_dec_two:
|
|
|
|
|
|
+ xorps $inout2,$inout2
|
|
|
|
|
|
call _aesni_decrypt3
|
|
|
|
|
|
movups $inout0,($out)
|
|
|
|
|
|
movups $inout1,0x10($out)
|
|
|
|
|
|
@@ -352,6 +768,34 @@ $code.=<<___;
|
|
|
|
|
|
movups $inout0,($out)
|
|
|
|
|
|
movups $inout1,0x10($out)
|
|
|
|
|
|
movups $inout2,0x20($out)
|
|
|
|
|
|
+ jmp .Lecb_ret
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lecb_dec_four:
|
|
|
|
|
|
+ call _aesni_decrypt4
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ jmp .Lecb_ret
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lecb_dec_five:
|
|
|
|
|
|
+ xorps $inout5,$inout5
|
|
|
|
|
|
+ call _aesni_decrypt6
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ jmp .Lecb_ret
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lecb_dec_six:
|
|
|
|
|
|
+ call _aesni_decrypt6
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
|
|
|
|
|
|
.Lecb_ret:
|
|
|
|
|
|
ret
|
|
|
|
|
|
@@ -362,7 +806,8 @@ ___
|
|
|
|
|
|
# void $PREFIX_cbc_encrypt (const void *inp, void *out,
|
|
|
|
|
|
# size_t length, const AES_KEY *key,
|
|
|
|
|
|
# unsigned char *ivp,const int enc);
|
|
|
|
|
|
-$reserved = $win64?0x40:-0x18; # used in decrypt
|
|
|
|
|
|
+{
|
|
|
|
|
|
+my $reserved = $win64?0x40:-0x18; # used in decrypt
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
.globl ${PREFIX}_cbc_encrypt
|
|
|
|
|
|
.type ${PREFIX}_cbc_encrypt,\@function,6
|
|
|
|
|
|
@@ -371,30 +816,30 @@ ${PREFIX}_cbc_encrypt:
|
|
|
|
|
|
test $len,$len # check length
|
|
|
|
|
|
jz .Lcbc_ret
|
|
|
|
|
|
|
|
|
|
|
|
- mov 240($key),$rnds_ # pull $rounds
|
|
|
|
|
|
+ mov 240($key),$rnds_ # key->rounds
|
|
|
|
|
|
mov $key,$key_ # backup $key
|
|
|
|
|
|
test %r9d,%r9d # 6th argument
|
|
|
|
|
|
jz .Lcbc_decrypt
|
|
|
|
|
|
#--------------------------- CBC ENCRYPT ------------------------------#
|
|
|
|
|
|
movups ($ivp),$inout0 # load iv as initial state
|
|
|
|
|
|
- cmp \$16,$len
|
|
|
|
|
|
mov $rnds_,$rounds
|
|
|
|
|
|
+ cmp \$16,$len
|
|
|
|
|
|
jb .Lcbc_enc_tail
|
|
|
|
|
|
sub \$16,$len
|
|
|
|
|
|
jmp .Lcbc_enc_loop
|
|
|
|
|
|
-.align 16
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
.Lcbc_enc_loop:
|
|
|
|
|
|
movups ($inp),$inout1 # load input
|
|
|
|
|
|
lea 16($inp),$inp
|
|
|
|
|
|
- pxor $inout1,$inout0
|
|
|
|
|
|
+ #xorps $inout1,$inout0
|
|
|
|
|
|
___
|
|
|
|
|
|
- &aesni_generate1("enc",$key,$rounds);
|
|
|
|
|
|
+ &aesni_generate1("enc",$key,$rounds,$inout0,$inout1);
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
- sub \$16,$len
|
|
|
|
|
|
- lea 16($out),$out
|
|
|
|
|
|
mov $rnds_,$rounds # restore $rounds
|
|
|
|
|
|
mov $key_,$key # restore $key
|
|
|
|
|
|
- movups $inout0,-16($out) # store output
|
|
|
|
|
|
+ movups $inout0,0($out) # store output
|
|
|
|
|
|
+ lea 16($out),$out
|
|
|
|
|
|
+ sub \$16,$len
|
|
|
|
|
|
jnc .Lcbc_enc_loop
|
|
|
|
|
|
add \$16,$len
|
|
|
|
|
|
jnz .Lcbc_enc_tail
|
|
|
|
|
|
@@ -429,92 +874,238 @@ $code.=<<___ if ($win64);
|
|
|
|
|
|
___
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
movups ($ivp),$iv
|
|
|
|
|
|
- sub \$0x40,$len
|
|
|
|
|
|
mov $rnds_,$rounds
|
|
|
|
|
|
+ cmp \$0x70,$len
|
|
|
|
|
|
jbe .Lcbc_dec_tail
|
|
|
|
|
|
- jmp .Lcbc_dec_loop3
|
|
|
|
|
|
-.align 16
|
|
|
|
|
|
-.Lcbc_dec_loop3:
|
|
|
|
|
|
- movups ($inp),$inout0
|
|
|
|
|
|
+ shr \$1,$rnds_
|
|
|
|
|
|
+ sub \$0x70,$len
|
|
|
|
|
|
+ mov $rnds_,$rounds
|
|
|
|
|
|
+ movaps $iv,$reserved(%rsp)
|
|
|
|
|
|
+ jmp .Lcbc_dec_loop8_enter
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lcbc_dec_loop8:
|
|
|
|
|
|
+ movaps $rndkey0,$reserved(%rsp) # save IV
|
|
|
|
|
|
+ movups $inout7,($out)
|
|
|
|
|
|
+ lea 0x10($out),$out
|
|
|
|
|
|
+.Lcbc_dec_loop8_enter:
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
+ movups ($inp),$inout0 # load input
|
|
|
|
|
|
movups 0x10($inp),$inout1
|
|
|
|
|
|
- movups 0x20($inp),$inout2
|
|
|
|
|
|
- movaps $inout0,$in0
|
|
|
|
|
|
- movaps $inout1,$in1
|
|
|
|
|
|
- movaps $inout2,$in2
|
|
|
|
|
|
- call _aesni_decrypt3
|
|
|
|
|
|
- sub \$0x30,$len
|
|
|
|
|
|
- lea 0x30($inp),$inp
|
|
|
|
|
|
- lea 0x30($out),$out
|
|
|
|
|
|
- pxor $iv,$inout0
|
|
|
|
|
|
- pxor $in0,$inout1
|
|
|
|
|
|
- movaps $in2,$iv
|
|
|
|
|
|
- pxor $in1,$inout2
|
|
|
|
|
|
- movups $inout0,-0x30($out)
|
|
|
|
|
|
- mov $rnds_,$rounds # restore $rounds
|
|
|
|
|
|
- movups $inout1,-0x20($out)
|
|
|
|
|
|
- mov $key_,$key # restore $key
|
|
|
|
|
|
- movups $inout2,-0x10($out)
|
|
|
|
|
|
- ja .Lcbc_dec_loop3
|
|
|
|
|
|
+ $movkey 16($key),$rndkey1
|
|
|
|
|
|
|
|
|
|
|
|
-.Lcbc_dec_tail:
|
|
|
|
|
|
- add \$0x40,$len
|
|
|
|
|
|
- movups $iv,($ivp)
|
|
|
|
|
|
- jz .Lcbc_dec_ret
|
|
|
|
|
|
+ lea 32($key),$key
|
|
|
|
|
|
+ movdqu 0x20($inp),$inout2
|
|
|
|
|
|
+ xorps $rndkey0,$inout0
|
|
|
|
|
|
+ movdqu 0x30($inp),$inout3
|
|
|
|
|
|
+ xorps $rndkey0,$inout1
|
|
|
|
|
|
+ movdqu 0x40($inp),$inout4
|
|
|
|
|
|
+ aesdec $rndkey1,$inout0
|
|
|
|
|
|
+ pxor $rndkey0,$inout2
|
|
|
|
|
|
+ movdqu 0x50($inp),$inout5
|
|
|
|
|
|
+ aesdec $rndkey1,$inout1
|
|
|
|
|
|
+ pxor $rndkey0,$inout3
|
|
|
|
|
|
+ movdqu 0x60($inp),$inout6
|
|
|
|
|
|
+ aesdec $rndkey1,$inout2
|
|
|
|
|
|
+ pxor $rndkey0,$inout4
|
|
|
|
|
|
+ movdqu 0x70($inp),$inout7
|
|
|
|
|
|
+ aesdec $rndkey1,$inout3
|
|
|
|
|
|
+ pxor $rndkey0,$inout5
|
|
|
|
|
|
+ dec $rounds
|
|
|
|
|
|
+ aesdec $rndkey1,$inout4
|
|
|
|
|
|
+ pxor $rndkey0,$inout6
|
|
|
|
|
|
+ aesdec $rndkey1,$inout5
|
|
|
|
|
|
+ pxor $rndkey0,$inout7
|
|
|
|
|
|
+ $movkey ($key),$rndkey0
|
|
|
|
|
|
+ aesdec $rndkey1,$inout6
|
|
|
|
|
|
+ aesdec $rndkey1,$inout7
|
|
|
|
|
|
+ $movkey 16($key),$rndkey1
|
|
|
|
|
|
+
|
|
|
|
|
|
+ call .Ldec_loop8_enter
|
|
|
|
|
|
|
|
|
|
|
|
+ movups ($inp),$rndkey1 # re-load input
|
|
|
|
|
|
+ movups 0x10($inp),$rndkey0
|
|
|
|
|
|
+ xorps $reserved(%rsp),$inout0 # ^= IV
|
|
|
|
|
|
+ xorps $rndkey1,$inout1
|
|
|
|
|
|
+ movups 0x20($inp),$rndkey1
|
|
|
|
|
|
+ xorps $rndkey0,$inout2
|
|
|
|
|
|
+ movups 0x30($inp),$rndkey0
|
|
|
|
|
|
+ xorps $rndkey1,$inout3
|
|
|
|
|
|
+ movups 0x40($inp),$rndkey1
|
|
|
|
|
|
+ xorps $rndkey0,$inout4
|
|
|
|
|
|
+ movups 0x50($inp),$rndkey0
|
|
|
|
|
|
+ xorps $rndkey1,$inout5
|
|
|
|
|
|
+ movups 0x60($inp),$rndkey1
|
|
|
|
|
|
+ xorps $rndkey0,$inout6
|
|
|
|
|
|
+ movups 0x70($inp),$rndkey0 # IV
|
|
|
|
|
|
+ xorps $rndkey1,$inout7
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ mov $rnds_,$rounds # restore $rounds
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ mov $key_,$key # restore $key
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
+ lea 0x80($inp),$inp
|
|
|
|
|
|
+ movups $inout6,0x60($out)
|
|
|
|
|
|
+ lea 0x70($out),$out
|
|
|
|
|
|
+ sub \$0x80,$len
|
|
|
|
|
|
+ ja .Lcbc_dec_loop8
|
|
|
|
|
|
+
|
|
|
|
|
|
+ movaps $inout7,$inout0
|
|
|
|
|
|
+ movaps $rndkey0,$iv
|
|
|
|
|
|
+ add \$0x70,$len
|
|
|
|
|
|
+ jle .Lcbc_dec_tail_collected
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ lea 1($rnds_,$rnds_),$rounds
|
|
|
|
|
|
+ lea 0x10($out),$out
|
|
|
|
|
|
+.Lcbc_dec_tail:
|
|
|
|
|
|
movups ($inp),$inout0
|
|
|
|
|
|
- cmp \$0x10,$len
|
|
|
|
|
|
movaps $inout0,$in0
|
|
|
|
|
|
+ cmp \$0x10,$len
|
|
|
|
|
|
jbe .Lcbc_dec_one
|
|
|
|
|
|
+
|
|
|
|
|
|
movups 0x10($inp),$inout1
|
|
|
|
|
|
- cmp \$0x20,$len
|
|
|
|
|
|
movaps $inout1,$in1
|
|
|
|
|
|
+ cmp \$0x20,$len
|
|
|
|
|
|
jbe .Lcbc_dec_two
|
|
|
|
|
|
+
|
|
|
|
|
|
movups 0x20($inp),$inout2
|
|
|
|
|
|
- cmp \$0x30,$len
|
|
|
|
|
|
movaps $inout2,$in2
|
|
|
|
|
|
+ cmp \$0x30,$len
|
|
|
|
|
|
jbe .Lcbc_dec_three
|
|
|
|
|
|
+
|
|
|
|
|
|
movups 0x30($inp),$inout3
|
|
|
|
|
|
- call _aesni_decrypt4
|
|
|
|
|
|
- pxor $iv,$inout0
|
|
|
|
|
|
- movups 0x30($inp),$iv
|
|
|
|
|
|
- pxor $in0,$inout1
|
|
|
|
|
|
+ cmp \$0x40,$len
|
|
|
|
|
|
+ jbe .Lcbc_dec_four
|
|
|
|
|
|
+
|
|
|
|
|
|
+ movups 0x40($inp),$inout4
|
|
|
|
|
|
+ cmp \$0x50,$len
|
|
|
|
|
|
+ jbe .Lcbc_dec_five
|
|
|
|
|
|
+
|
|
|
|
|
|
+ movups 0x50($inp),$inout5
|
|
|
|
|
|
+ cmp \$0x60,$len
|
|
|
|
|
|
+ jbe .Lcbc_dec_six
|
|
|
|
|
|
+
|
|
|
|
|
|
+ movups 0x60($inp),$inout6
|
|
|
|
|
|
+ movaps $iv,$reserved(%rsp) # save IV
|
|
|
|
|
|
+ call _aesni_decrypt8
|
|
|
|
|
|
+ movups ($inp),$rndkey1
|
|
|
|
|
|
+ movups 0x10($inp),$rndkey0
|
|
|
|
|
|
+ xorps $reserved(%rsp),$inout0 # ^= IV
|
|
|
|
|
|
+ xorps $rndkey1,$inout1
|
|
|
|
|
|
+ movups 0x20($inp),$rndkey1
|
|
|
|
|
|
+ xorps $rndkey0,$inout2
|
|
|
|
|
|
+ movups 0x30($inp),$rndkey0
|
|
|
|
|
|
+ xorps $rndkey1,$inout3
|
|
|
|
|
|
+ movups 0x40($inp),$rndkey1
|
|
|
|
|
|
+ xorps $rndkey0,$inout4
|
|
|
|
|
|
+ movups 0x50($inp),$rndkey0
|
|
|
|
|
|
+ xorps $rndkey1,$inout5
|
|
|
|
|
|
+ movups 0x60($inp),$iv # IV
|
|
|
|
|
|
+ xorps $rndkey0,$inout6
|
|
|
|
|
|
movups $inout0,($out)
|
|
|
|
|
|
- pxor $in1,$inout2
|
|
|
|
|
|
movups $inout1,0x10($out)
|
|
|
|
|
|
- pxor $in2,$inout3
|
|
|
|
|
|
movups $inout2,0x20($out)
|
|
|
|
|
|
- movaps $inout3,$inout0
|
|
|
|
|
|
- lea 0x30($out),$out
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ movups $inout5,0x50($out)
|
|
|
|
|
|
+ lea 0x60($out),$out
|
|
|
|
|
|
+ movaps $inout6,$inout0
|
|
|
|
|
|
+ sub \$0x70,$len
|
|
|
|
|
|
jmp .Lcbc_dec_tail_collected
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lcbc_dec_one:
|
|
|
|
|
|
___
|
|
|
|
|
|
&aesni_generate1("dec",$key,$rounds);
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
- pxor $iv,$inout0
|
|
|
|
|
|
+ xorps $iv,$inout0
|
|
|
|
|
|
movaps $in0,$iv
|
|
|
|
|
|
+ sub \$0x10,$len
|
|
|
|
|
|
jmp .Lcbc_dec_tail_collected
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lcbc_dec_two:
|
|
|
|
|
|
+ xorps $inout2,$inout2
|
|
|
|
|
|
call _aesni_decrypt3
|
|
|
|
|
|
- pxor $iv,$inout0
|
|
|
|
|
|
- pxor $in0,$inout1
|
|
|
|
|
|
+ xorps $iv,$inout0
|
|
|
|
|
|
+ xorps $in0,$inout1
|
|
|
|
|
|
movups $inout0,($out)
|
|
|
|
|
|
movaps $in1,$iv
|
|
|
|
|
|
movaps $inout1,$inout0
|
|
|
|
|
|
lea 0x10($out),$out
|
|
|
|
|
|
+ sub \$0x20,$len
|
|
|
|
|
|
jmp .Lcbc_dec_tail_collected
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lcbc_dec_three:
|
|
|
|
|
|
call _aesni_decrypt3
|
|
|
|
|
|
- pxor $iv,$inout0
|
|
|
|
|
|
- pxor $in0,$inout1
|
|
|
|
|
|
+ xorps $iv,$inout0
|
|
|
|
|
|
+ xorps $in0,$inout1
|
|
|
|
|
|
movups $inout0,($out)
|
|
|
|
|
|
- pxor $in1,$inout2
|
|
|
|
|
|
+ xorps $in1,$inout2
|
|
|
|
|
|
movups $inout1,0x10($out)
|
|
|
|
|
|
movaps $in2,$iv
|
|
|
|
|
|
movaps $inout2,$inout0
|
|
|
|
|
|
lea 0x20($out),$out
|
|
|
|
|
|
+ sub \$0x30,$len
|
|
|
|
|
|
+ jmp .Lcbc_dec_tail_collected
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lcbc_dec_four:
|
|
|
|
|
|
+ call _aesni_decrypt4
|
|
|
|
|
|
+ xorps $iv,$inout0
|
|
|
|
|
|
+ movups 0x30($inp),$iv
|
|
|
|
|
|
+ xorps $in0,$inout1
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ xorps $in1,$inout2
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ xorps $in2,$inout3
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movaps $inout3,$inout0
|
|
|
|
|
|
+ lea 0x30($out),$out
|
|
|
|
|
|
+ sub \$0x40,$len
|
|
|
|
|
|
+ jmp .Lcbc_dec_tail_collected
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lcbc_dec_five:
|
|
|
|
|
|
+ xorps $inout5,$inout5
|
|
|
|
|
|
+ call _aesni_decrypt6
|
|
|
|
|
|
+ movups 0x10($inp),$rndkey1
|
|
|
|
|
|
+ movups 0x20($inp),$rndkey0
|
|
|
|
|
|
+ xorps $iv,$inout0
|
|
|
|
|
|
+ xorps $in0,$inout1
|
|
|
|
|
|
+ xorps $rndkey1,$inout2
|
|
|
|
|
|
+ movups 0x30($inp),$rndkey1
|
|
|
|
|
|
+ xorps $rndkey0,$inout3
|
|
|
|
|
|
+ movups 0x40($inp),$iv
|
|
|
|
|
|
+ xorps $rndkey1,$inout4
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ lea 0x40($out),$out
|
|
|
|
|
|
+ movaps $inout4,$inout0
|
|
|
|
|
|
+ sub \$0x50,$len
|
|
|
|
|
|
+ jmp .Lcbc_dec_tail_collected
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lcbc_dec_six:
|
|
|
|
|
|
+ call _aesni_decrypt6
|
|
|
|
|
|
+ movups 0x10($inp),$rndkey1
|
|
|
|
|
|
+ movups 0x20($inp),$rndkey0
|
|
|
|
|
|
+ xorps $iv,$inout0
|
|
|
|
|
|
+ xorps $in0,$inout1
|
|
|
|
|
|
+ xorps $rndkey1,$inout2
|
|
|
|
|
|
+ movups 0x30($inp),$rndkey1
|
|
|
|
|
|
+ xorps $rndkey0,$inout3
|
|
|
|
|
|
+ movups 0x40($inp),$rndkey0
|
|
|
|
|
|
+ xorps $rndkey1,$inout4
|
|
|
|
|
|
+ movups 0x50($inp),$iv
|
|
|
|
|
|
+ xorps $rndkey0,$inout5
|
|
|
|
|
|
+ movups $inout0,($out)
|
|
|
|
|
|
+ movups $inout1,0x10($out)
|
|
|
|
|
|
+ movups $inout2,0x20($out)
|
|
|
|
|
|
+ movups $inout3,0x30($out)
|
|
|
|
|
|
+ movups $inout4,0x40($out)
|
|
|
|
|
|
+ lea 0x50($out),$out
|
|
|
|
|
|
+ movaps $inout5,$inout0
|
|
|
|
|
|
+ sub \$0x60,$len
|
|
|
|
|
|
jmp .Lcbc_dec_tail_collected
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lcbc_dec_tail_collected:
|
|
|
|
|
|
@@ -523,10 +1114,12 @@ $code.=<<___;
|
|
|
|
|
|
jnz .Lcbc_dec_tail_partial
|
|
|
|
|
|
movups $inout0,($out)
|
|
|
|
|
|
jmp .Lcbc_dec_ret
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
.Lcbc_dec_tail_partial:
|
|
|
|
|
|
movaps $inout0,$reserved(%rsp)
|
|
|
|
|
|
+ mov \$16,%rcx
|
|
|
|
|
|
mov $out,%rdi
|
|
|
|
|
|
- mov $len,%rcx
|
|
|
|
|
|
+ sub $len,%rcx
|
|
|
|
|
|
lea $reserved(%rsp),%rsi
|
|
|
|
|
|
.long 0x9066A4F3 # rep movsb
|
|
|
|
|
|
|
|
|
|
|
|
@@ -544,7 +1137,7 @@ $code.=<<___;
|
|
|
|
|
|
ret
|
|
|
|
|
|
.size ${PREFIX}_cbc_encrypt,.-${PREFIX}_cbc_encrypt
|
|
|
|
|
|
___
|
|
|
|
|
|
-
|
|
|
|
|
|
+}
|
|
|
|
|
|
# int $PREFIX_set_[en|de]crypt_key (const unsigned char *userKey,
|
|
|
|
|
|
# int bits, AES_KEY *key)
|
|
|
|
|
|
{ my ($inp,$bits,$key) = @_4args;
|
|
|
|
|
|
@@ -556,7 +1149,7 @@ $code.=<<___;
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
${PREFIX}_set_decrypt_key:
|
|
|
|
|
|
.byte 0x48,0x83,0xEC,0x08 # sub rsp,8
|
|
|
|
|
|
- call _aesni_set_encrypt_key
|
|
|
|
|
|
+ call __aesni_set_encrypt_key
|
|
|
|
|
|
shl \$4,$bits # rounds-1 after _aesni_set_encrypt_key
|
|
|
|
|
|
test %eax,%eax
|
|
|
|
|
|
jnz .Ldec_key_ret
|
|
|
|
|
|
@@ -576,9 +1169,9 @@ ${PREFIX}_set_decrypt_key:
|
|
|
|
|
|
aesimc %xmm1,%xmm1
|
|
|
|
|
|
lea 16($key),$key
|
|
|
|
|
|
lea -16($inp),$inp
|
|
|
|
|
|
- cmp $key,$inp
|
|
|
|
|
|
$movkey %xmm0,16($inp)
|
|
|
|
|
|
$movkey %xmm1,-16($key)
|
|
|
|
|
|
+ cmp $key,$inp
|
|
|
|
|
|
ja .Ldec_key_inverse
|
|
|
|
|
|
|
|
|
|
|
|
$movkey ($key),%xmm0 # inverse middle
|
|
|
|
|
|
@@ -605,16 +1198,16 @@ $code.=<<___;
|
|
|
|
|
|
.type ${PREFIX}_set_encrypt_key,\@abi-omnipotent
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
${PREFIX}_set_encrypt_key:
|
|
|
|
|
|
-_aesni_set_encrypt_key:
|
|
|
|
|
|
+__aesni_set_encrypt_key:
|
|
|
|
|
|
.byte 0x48,0x83,0xEC,0x08 # sub rsp,8
|
|
|
|
|
|
- test $inp,$inp
|
|
|
|
|
|
mov \$-1,%rax
|
|
|
|
|
|
+ test $inp,$inp
|
|
|
|
|
|
jz .Lenc_key_ret
|
|
|
|
|
|
test $key,$key
|
|
|
|
|
|
jz .Lenc_key_ret
|
|
|
|
|
|
|
|
|
|
|
|
movups ($inp),%xmm0 # pull first 128 bits of *userKey
|
|
|
|
|
|
- pxor %xmm4,%xmm4 # low dword of xmm4 is assumed 0
|
|
|
|
|
|
+ xorps %xmm4,%xmm4 # low dword of xmm4 is assumed 0
|
|
|
|
|
|
lea 16($key),%rax
|
|
|
|
|
|
cmp \$256,$bits
|
|
|
|
|
|
je .L14rounds
|
|
|
|
|
|
@@ -729,11 +1322,11 @@ _aesni_set_encrypt_key:
|
|
|
|
|
|
lea 16(%rax),%rax
|
|
|
|
|
|
.Lkey_expansion_128_cold:
|
|
|
|
|
|
shufps \$0b00010000,%xmm0,%xmm4
|
|
|
|
|
|
- pxor %xmm4, %xmm0
|
|
|
|
|
|
+ xorps %xmm4, %xmm0
|
|
|
|
|
|
shufps \$0b10001100,%xmm0,%xmm4
|
|
|
|
|
|
- pxor %xmm4, %xmm0
|
|
|
|
|
|
- pshufd \$0b11111111,%xmm1,%xmm1 # critical path
|
|
|
|
|
|
- pxor %xmm1,%xmm0
|
|
|
|
|
|
+ xorps %xmm4, %xmm0
|
|
|
|
|
|
+ shufps \$0b11111111,%xmm1,%xmm1 # critical path
|
|
|
|
|
|
+ xorps %xmm1,%xmm0
|
|
|
|
|
|
ret
|
|
|
|
|
|
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
@@ -744,11 +1337,11 @@ _aesni_set_encrypt_key:
|
|
|
|
|
|
movaps %xmm2, %xmm5
|
|
|
|
|
|
.Lkey_expansion_192b_warm:
|
|
|
|
|
|
shufps \$0b00010000,%xmm0,%xmm4
|
|
|
|
|
|
- movaps %xmm2,%xmm3
|
|
|
|
|
|
- pxor %xmm4,%xmm0
|
|
|
|
|
|
+ movdqa %xmm2,%xmm3
|
|
|
|
|
|
+ xorps %xmm4,%xmm0
|
|
|
|
|
|
shufps \$0b10001100,%xmm0,%xmm4
|
|
|
|
|
|
pslldq \$4,%xmm3
|
|
|
|
|
|
- pxor %xmm4,%xmm0
|
|
|
|
|
|
+ xorps %xmm4,%xmm0
|
|
|
|
|
|
pshufd \$0b01010101,%xmm1,%xmm1 # critical path
|
|
|
|
|
|
pxor %xmm3,%xmm2
|
|
|
|
|
|
pxor %xmm1,%xmm0
|
|
|
|
|
|
@@ -772,11 +1365,11 @@ _aesni_set_encrypt_key:
|
|
|
|
|
|
lea 16(%rax),%rax
|
|
|
|
|
|
.Lkey_expansion_256a_cold:
|
|
|
|
|
|
shufps \$0b00010000,%xmm0,%xmm4
|
|
|
|
|
|
- pxor %xmm4,%xmm0
|
|
|
|
|
|
+ xorps %xmm4,%xmm0
|
|
|
|
|
|
shufps \$0b10001100,%xmm0,%xmm4
|
|
|
|
|
|
- pxor %xmm4,%xmm0
|
|
|
|
|
|
- pshufd \$0b11111111,%xmm1,%xmm1 # critical path
|
|
|
|
|
|
- pxor %xmm1,%xmm0
|
|
|
|
|
|
+ xorps %xmm4,%xmm0
|
|
|
|
|
|
+ shufps \$0b11111111,%xmm1,%xmm1 # critical path
|
|
|
|
|
|
+ xorps %xmm1,%xmm0
|
|
|
|
|
|
ret
|
|
|
|
|
|
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
@@ -785,17 +1378,28 @@ _aesni_set_encrypt_key:
|
|
|
|
|
|
lea 16(%rax),%rax
|
|
|
|
|
|
|
|
|
|
|
|
shufps \$0b00010000,%xmm2,%xmm4
|
|
|
|
|
|
- pxor %xmm4,%xmm2
|
|
|
|
|
|
+ xorps %xmm4,%xmm2
|
|
|
|
|
|
shufps \$0b10001100,%xmm2,%xmm4
|
|
|
|
|
|
- pxor %xmm4,%xmm2
|
|
|
|
|
|
- pshufd \$0b10101010,%xmm1,%xmm1 # critical path
|
|
|
|
|
|
- pxor %xmm1,%xmm2
|
|
|
|
|
|
+ xorps %xmm4,%xmm2
|
|
|
|
|
|
+ shufps \$0b10101010,%xmm1,%xmm1 # critical path
|
|
|
|
|
|
+ xorps %xmm1,%xmm2
|
|
|
|
|
|
ret
|
|
|
|
|
|
.size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key
|
|
|
|
|
|
+.size __aesni_set_encrypt_key,.-__aesni_set_encrypt_key
|
|
|
|
|
|
___
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
+.align 64
|
|
|
|
|
|
+.Lbswap_mask:
|
|
|
|
|
|
+ .byte 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
|
|
|
|
|
|
+.Lincrement32:
|
|
|
|
|
|
+ .long 6,6,6,0
|
|
|
|
|
|
+.Lincrement64:
|
|
|
|
|
|
+ .long 1,0,0,0
|
|
|
|
|
|
+.Lxts_magic:
|
|
|
|
|
|
+ .long 0x87,0,1,0
|
|
|
|
|
|
+
|
|
|
|
|
|
.asciz "AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>"
|
|
|
|
|
|
.align 64
|
|
|
|
|
|
___
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/cryptlib.c.intelopts openssl-1.0.0d/crypto/cryptlib.c
|
|
|
|
|
|
--- openssl-1.0.0d/crypto/cryptlib.c.intelopts 2010-11-19 01:11:27.000000000 +0100
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+++ openssl-1.0.0d/crypto/cryptlib.c 2011-08-24 12:50:55.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -662,22 +662,23 @@ const char *CRYPTO_get_lock_name(int typ
|
|
|
|
|
|
defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
|
|
|
|
|
|
|
|
|
|
|
|
unsigned long OPENSSL_ia32cap_P=0;
|
|
|
|
|
|
+unsigned long long OPENSSL_ia32cap_X=0;
|
|
|
|
|
|
unsigned long *OPENSSL_ia32cap_loc(void) { return &OPENSSL_ia32cap_P; }
|
|
|
|
|
|
|
|
|
|
|
|
#if defined(OPENSSL_CPUID_OBJ) && !defined(OPENSSL_NO_ASM) && !defined(I386_ONLY)
|
|
|
|
|
|
#define OPENSSL_CPUID_SETUP
|
|
|
|
|
|
void OPENSSL_cpuid_setup(void)
|
|
|
|
|
|
{ static int trigger=0;
|
|
|
|
|
|
- unsigned long OPENSSL_ia32_cpuid(void);
|
|
|
|
|
|
+ unsigned long long OPENSSL_ia32_cpuid(void);
|
|
|
|
|
|
char *env;
|
|
|
|
|
|
|
|
|
|
|
|
if (trigger) return;
|
|
|
|
|
|
|
|
|
|
|
|
trigger=1;
|
|
|
|
|
|
if ((env=getenv("OPENSSL_ia32cap")))
|
|
|
|
|
|
- OPENSSL_ia32cap_P = strtoul(env,NULL,0)|(1<<10);
|
|
|
|
|
|
+ OPENSSL_ia32cap_X = OPENSSL_ia32cap_P = strtoul(env,NULL,0)|(1<<10);
|
|
|
|
|
|
else
|
|
|
|
|
|
- OPENSSL_ia32cap_P = OPENSSL_ia32_cpuid()|(1<<10);
|
|
|
|
|
|
+ OPENSSL_ia32cap_P = OPENSSL_ia32cap_X = OPENSSL_ia32_cpuid()|(1<<10);
|
|
|
|
|
|
/*
|
|
|
|
|
|
* |(1<<10) sets a reserved bit to signal that variable
|
|
|
|
|
|
* was initialized already... This is to avoid interference
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/engine/eng_aesni.c.intelopts openssl-1.0.0d/crypto/engine/eng_aesni.c
|
2011-11-03 09:18:52 +00:00
|
|
|
|
--- openssl-1.0.0d/crypto/engine/eng_aesni.c.intelopts 2011-08-24 12:50:55.000000000 +0200
|
|
|
|
|
|
+++ openssl-1.0.0d/crypto/engine/eng_aesni.c 2011-08-24 12:50:55.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -157,16 +157,20 @@ typedef unsigned __int64 IA32CAP;
|
|
|
|
|
|
typedef unsigned long long IA32CAP;
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
+extern IA32CAP OPENSSL_ia32cap_X;
|
|
|
|
|
|
+
|
|
|
|
|
|
/* Prepare the ENGINE structure for registration */
|
|
|
|
|
|
static int
|
|
|
|
|
|
aesni_bind_helper(ENGINE *e)
|
|
|
|
|
|
{
|
|
|
|
|
|
int engage;
|
|
|
|
|
|
- if (sizeof(OPENSSL_ia32cap_P) > 4) {
|
|
|
|
|
|
- engage = (OPENSSL_ia32cap_P >> 57) & 1;
|
|
|
|
|
|
- } else {
|
|
|
|
|
|
- IA32CAP OPENSSL_ia32_cpuid(void);
|
|
|
|
|
|
- engage = (OPENSSL_ia32_cpuid() >> 57) & 1;
|
|
|
|
|
|
+ engage = (OPENSSL_ia32cap_X >> 57) & 1;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ /* Disable the AES-NI support if the environment variable
|
|
|
|
|
|
+ * OPENSSL_DISABLE_AES_NI is set to any value
|
|
|
|
|
|
+ */
|
|
|
|
|
|
+ if (getenv("OPENSSL_DISABLE_AES_NI") != NULL) {
|
|
|
|
|
|
+ engage = 0;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/* Register everything or return with an error */
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/fips/fips_standalone_sha1.c.intelopts openssl-1.0.0d/crypto/fips/fips_standalone_sha1.c
|
2011-11-03 09:18:52 +00:00
|
|
|
|
--- openssl-1.0.0d/crypto/fips/fips_standalone_sha1.c.intelopts 2011-08-24 12:50:55.000000000 +0200
|
|
|
|
|
|
+++ openssl-1.0.0d/crypto/fips/fips_standalone_sha1.c 2011-08-24 12:50:55.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -62,6 +62,8 @@ void OPENSSL_cleanse(void *p,size_t len)
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef OPENSSL_FIPS
|
|
|
|
|
|
|
|
|
|
|
|
+unsigned long long OPENSSL_ia32cap_X = 0;
|
|
|
|
|
|
+
|
|
|
|
|
|
static void hmac_init(SHA256_CTX *md_ctx,SHA256_CTX *o_ctx,
|
|
|
|
|
|
const char *key)
|
|
|
|
|
|
{
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/perlasm/x86asm.pl.intelopts openssl-1.0.0d/crypto/perlasm/x86asm.pl
|
|
|
|
|
|
--- openssl-1.0.0d/crypto/perlasm/x86asm.pl.intelopts 2008-12-17 20:56:47.000000000 +0100
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+++ openssl-1.0.0d/crypto/perlasm/x86asm.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
|
|
|
|
|
|
# require 'x86asm.pl';
|
|
|
|
|
|
# &asm_init(<flavor>,"des-586.pl"[,$i386only]);
|
|
|
|
|
|
@@ -80,6 +80,57 @@ sub ::movq
|
|
|
|
|
|
{ &::generic("movq",@_); }
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
+# SSE>2 instructions
|
|
|
|
|
|
+my %regrm = ( "eax"=>0, "ecx"=>1, "edx"=>2, "ebx"=>3,
|
|
|
|
|
|
+ "esp"=>4, "ebp"=>5, "esi"=>6, "edi"=>7 );
|
|
|
|
|
|
+sub ::pextrd
|
|
|
|
|
|
+{ my($dst,$src,$imm)=@_;
|
|
|
|
|
|
+ if ("$dst:$src" =~ /(e[a-dsd][ixp]):xmm([0-7])/)
|
|
|
|
|
|
+ { &::data_byte(0x66,0x0f,0x3a,0x16,0xc0|($2<<3)|$regrm{$1},$imm); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &::generic("pextrd",@_); }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub ::pinsrd
|
|
|
|
|
|
+{ my($dst,$src,$imm)=@_;
|
|
|
|
|
|
+ if ("$dst:$src" =~ /xmm([0-7]):(e[a-dsd][ixp])/)
|
|
|
|
|
|
+ { &::data_byte(0x66,0x0f,0x3a,0x22,0xc0|($1<<3)|$regrm{$2},$imm); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &::generic("pinsrd",@_); }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub ::pshufb
|
|
|
|
|
|
+{ my($dst,$src)=@_;
|
|
|
|
|
|
+ if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
|
|
|
|
|
|
+ { &data_byte(0x66,0x0f,0x38,0x00,0xc0|($1<<3)|$2); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &::generic("pshufb",@_); }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub ::palignr
|
|
|
|
|
|
+{ my($dst,$src,$imm)=@_;
|
|
|
|
|
|
+ if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
|
|
|
|
|
|
+ { &::data_byte(0x66,0x0f,0x3a,0x0f,0xc0|($1<<3)|$2,$imm); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &::generic("palignr",@_); }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub ::pclmulqdq
|
|
|
|
|
|
+{ my($dst,$src,$imm)=@_;
|
|
|
|
|
|
+ if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)
|
|
|
|
|
|
+ { &::data_byte(0x66,0x0f,0x3a,0x44,0xc0|($1<<3)|$2,$imm); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &::generic("pclmulqdq",@_); }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub ::rdrand
|
|
|
|
|
|
+{ my ($dst)=@_;
|
|
|
|
|
|
+ if ($dst =~ /(e[a-dsd][ixp])/)
|
|
|
|
|
|
+ { &::data_byte(0x0f,0xc7,0xf0|$regrm{$dst}); }
|
|
|
|
|
|
+ else
|
|
|
|
|
|
+ { &::generic("rdrand",@_); }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
# label management
|
|
|
|
|
|
$lbdecor="L"; # local label decoration, set by package
|
|
|
|
|
|
$label="000";
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/perlasm/x86gas.pl.intelopts openssl-1.0.0d/crypto/perlasm/x86gas.pl
|
|
|
|
|
|
--- openssl-1.0.0d/crypto/perlasm/x86gas.pl.intelopts 2008-12-17 20:56:47.000000000 +0100
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+++ openssl-1.0.0d/crypto/perlasm/x86gas.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
|
|
|
|
|
|
package x86gas;
|
|
|
|
|
|
|
|
|
|
|
|
@@ -91,6 +91,7 @@ sub ::DWP
|
|
|
|
|
|
}
|
|
|
|
|
|
sub ::QWP { &::DWP(@_); }
|
|
|
|
|
|
sub ::BP { &::DWP(@_); }
|
|
|
|
|
|
+sub ::WP { &::DWP(@_); }
|
|
|
|
|
|
sub ::BC { @_; }
|
|
|
|
|
|
sub ::DWC { @_; }
|
|
|
|
|
|
|
|
|
|
|
|
@@ -161,10 +162,16 @@ sub ::file_end
|
|
|
|
|
|
{ push(@out,"$non_lazy_ptr{$i}:\n.indirect_symbol\t$i\n.long\t0\n"); }
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
+ if (grep {/\b${nmdecor}OPENSSL_ia32cap_X\b/i} @out) {
|
|
|
|
|
|
+ my $tmp=".comm\t${nmdecor}OPENSSL_ia32cap_X,8";
|
|
|
|
|
|
+ if ($::elf) { push (@out,"$tmp,4\n"); }
|
|
|
|
|
|
+ else { push (@out,"$tmp\n"); }
|
|
|
|
|
|
+ }
|
|
|
|
|
|
push(@out,$initseg) if ($initseg);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
sub ::data_byte { push(@out,".byte\t".join(',',@_)."\n"); }
|
|
|
|
|
|
+sub ::data_short{ push(@out,".value\t".join(',',@_)."\n"); }
|
|
|
|
|
|
sub ::data_word { push(@out,".long\t".join(',',@_)."\n"); }
|
|
|
|
|
|
|
|
|
|
|
|
sub ::align
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/perlasm/x86_64-xlate.pl.intelopts openssl-1.0.0d/crypto/perlasm/x86_64-xlate.pl
|
|
|
|
|
|
--- openssl-1.0.0d/crypto/perlasm/x86_64-xlate.pl.intelopts 2010-10-10 23:14:17.000000000 +0200
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+++ openssl-1.0.0d/crypto/perlasm/x86_64-xlate.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
|
|
|
|
|
|
# Ascetic x86_64 AT&T to MASM/NASM assembler translator by <appro>.
|
|
|
|
|
|
#
|
|
|
|
|
|
@@ -121,7 +121,11 @@ my %globals;
|
|
|
|
|
|
$self->{sz} = "b";
|
|
|
|
|
|
} elsif ($self->{op} =~ /call|jmp/) {
|
|
|
|
|
|
$self->{sz} = "";
|
|
|
|
|
|
- } elsif ($self->{op} =~ /^p/ && $' !~ /^(ush|op)/) { # SSEn
|
|
|
|
|
|
+ } elsif ($self->{op} =~ /^p/ && $' !~ /^(ush|op|insrw)/) { # SSEn
|
|
|
|
|
|
+ $self->{sz} = "";
|
|
|
|
|
|
+ } elsif ($self->{op} =~ /^v/) { # VEX
|
|
|
|
|
|
+ $self->{sz} = "";
|
|
|
|
|
|
+ } elsif ($self->{op} =~ /movq/ && $line =~ /%xmm/) {
|
|
|
|
|
|
$self->{sz} = "";
|
|
|
|
|
|
} elsif ($self->{op} =~ /([a-z]{3,})([qlwb])$/) {
|
|
|
|
|
|
$self->{op} = $1;
|
|
|
|
|
|
@@ -246,35 +250,38 @@ my %globals;
|
|
|
|
|
|
$self->{index} =~ s/^[er](.?[0-9xpi])[d]?$/r\1/;
|
|
|
|
|
|
$self->{base} =~ s/^[er](.?[0-9xpi])[d]?$/r\1/;
|
|
|
|
|
|
|
|
|
|
|
|
+ # Solaris /usr/ccs/bin/as can't handle multiplications
|
|
|
|
|
|
+ # in $self->{label}, new gas requires sign extension...
|
|
|
|
|
|
+ use integer;
|
|
|
|
|
|
+ $self->{label} =~ s/(?<![\w\$\.])(0x?[0-9a-f]+)/oct($1)/egi;
|
|
|
|
|
|
+ $self->{label} =~ s/([0-9]+\s*[\*\/\%]\s*[0-9]+)/eval($1)/eg;
|
|
|
|
|
|
+ $self->{label} =~ s/([0-9]+)/$1<<32>>32/eg;
|
|
|
|
|
|
+
|
|
|
|
|
|
if ($gas) {
|
|
|
|
|
|
- # Solaris /usr/ccs/bin/as can't handle multiplications
|
|
|
|
|
|
- # in $self->{label}, new gas requires sign extension...
|
|
|
|
|
|
- use integer;
|
|
|
|
|
|
- $self->{label} =~ s/(?<![\w\$\.])(0x?[0-9a-f]+)/oct($1)/egi;
|
|
|
|
|
|
- $self->{label} =~ s/([0-9]+\s*[\*\/\%]\s*[0-9]+)/eval($1)/eg;
|
|
|
|
|
|
- $self->{label} =~ s/([0-9]+)/$1<<32>>32/eg;
|
|
|
|
|
|
$self->{label} =~ s/^___imp_/__imp__/ if ($flavour eq "mingw64");
|
|
|
|
|
|
|
|
|
|
|
|
if (defined($self->{index})) {
|
|
|
|
|
|
- sprintf "%s%s(%%%s,%%%s,%d)",$self->{asterisk},
|
|
|
|
|
|
- $self->{label},$self->{base},
|
|
|
|
|
|
+ sprintf "%s%s(%s,%%%s,%d)",$self->{asterisk},
|
|
|
|
|
|
+ $self->{label},
|
|
|
|
|
|
+ $self->{base}?"%$self->{base}":"",
|
|
|
|
|
|
$self->{index},$self->{scale};
|
|
|
|
|
|
} else {
|
|
|
|
|
|
sprintf "%s%s(%%%s)", $self->{asterisk},$self->{label},$self->{base};
|
|
|
|
|
|
}
|
|
|
|
|
|
} else {
|
|
|
|
|
|
- %szmap = ( b=>"BYTE$PTR", w=>"WORD$PTR", l=>"DWORD$PTR", q=>"QWORD$PTR" );
|
|
|
|
|
|
+ %szmap = ( b=>"BYTE$PTR", w=>"WORD$PTR", l=>"DWORD$PTR",
|
|
|
|
|
|
+ q=>"QWORD$PTR",o=>"OWORD$PTR",x=>"XMMWORD$PTR" );
|
|
|
|
|
|
|
|
|
|
|
|
$self->{label} =~ s/\./\$/g;
|
|
|
|
|
|
$self->{label} =~ s/(?<![\w\$\.])0x([0-9a-f]+)/0$1h/ig;
|
|
|
|
|
|
$self->{label} = "($self->{label})" if ($self->{label} =~ /[\*\+\-\/]/);
|
|
|
|
|
|
- $sz="q" if ($self->{asterisk});
|
|
|
|
|
|
+ $sz="q" if ($self->{asterisk} || opcode->mnemonic() eq "movq");
|
|
|
|
|
|
|
|
|
|
|
|
if (defined($self->{index})) {
|
|
|
|
|
|
- sprintf "%s[%s%s*%d+%s]",$szmap{$sz},
|
|
|
|
|
|
+ sprintf "%s[%s%s*%d%s]",$szmap{$sz},
|
|
|
|
|
|
$self->{label}?"$self->{label}+":"",
|
|
|
|
|
|
$self->{index},$self->{scale},
|
|
|
|
|
|
- $self->{base};
|
|
|
|
|
|
+ $self->{base}?"+$self->{base}":"";
|
|
|
|
|
|
} elsif ($self->{base} eq "rip") {
|
|
|
|
|
|
sprintf "%s[%s]",$szmap{$sz},$self->{label};
|
|
|
|
|
|
} else {
|
|
|
|
|
|
@@ -506,6 +513,11 @@ my %globals;
|
|
|
|
|
|
}
|
|
|
|
|
|
} elsif ($dir =~ /\.(text|data)/) {
|
|
|
|
|
|
$current_segment=".$1";
|
|
|
|
|
|
+ } elsif ($dir =~ /\.hidden/) {
|
|
|
|
|
|
+ if ($flavour eq "macosx") { $self->{value} = ".private_extern\t$prefix$line"; }
|
|
|
|
|
|
+ elsif ($flavour eq "mingw64") { $self->{value} = ""; }
|
|
|
|
|
|
+ } elsif ($dir =~ /\.comm/) {
|
|
|
|
|
|
+ $self->{value} = "$dir\t$prefix$line";
|
|
|
|
|
|
}
|
|
|
|
|
|
$line = "";
|
|
|
|
|
|
return $self;
|
|
|
|
|
|
@@ -613,6 +625,19 @@ my %globals;
|
|
|
|
|
|
.join(",",@str) if (@str);
|
|
|
|
|
|
last;
|
|
|
|
|
|
};
|
|
|
|
|
|
+ /\.comm/ && do { my @str=split(/,\s*/,$line);
|
|
|
|
|
|
+ my $v=undef;
|
|
|
|
|
|
+ if ($nasm) {
|
|
|
|
|
|
+ $v.="common $prefix@str[0] @str[1]";
|
|
|
|
|
|
+ } else {
|
|
|
|
|
|
+ $v="$current_segment\tENDS\n" if ($current_segment);
|
|
|
|
|
|
+ $current_segment = "_DATA";
|
|
|
|
|
|
+ $v.="$current_segment\tSEGMENT\n";
|
|
|
|
|
|
+ $v.="COMM @str[0]:DWORD:".@str[1]/4;
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+ $self->{value} = $v;
|
|
|
|
|
|
+ last;
|
|
|
|
|
|
+ };
|
|
|
|
|
|
}
|
|
|
|
|
|
$line = "";
|
|
|
|
|
|
}
|
|
|
|
|
|
@@ -625,9 +650,133 @@ my %globals;
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
+sub rex {
|
|
|
|
|
|
+ local *opcode=shift;
|
|
|
|
|
|
+ my ($dst,$src,$rex)=@_;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $rex|=0x04 if($dst>=8);
|
|
|
|
|
|
+ $rex|=0x01 if($src>=8);
|
|
|
|
|
|
+ push @opcode,($rex|0x40) if ($rex);
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+# older gas and ml64 don't handle SSE>2 instructions
|
|
|
|
|
|
+my %regrm = ( "%eax"=>0, "%ecx"=>1, "%edx"=>2, "%ebx"=>3,
|
|
|
|
|
|
+ "%esp"=>4, "%ebp"=>5, "%esi"=>6, "%edi"=>7 );
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $movq = sub { # elderly gas can't handle inter-register movq
|
|
|
|
|
|
+ my $arg = shift;
|
|
|
|
|
|
+ my @opcode=(0x66);
|
|
|
|
|
|
+ if ($arg =~ /%xmm([0-9]+),%r(\w+)/) {
|
|
|
|
|
|
+ my ($src,$dst)=($1,$2);
|
|
|
|
|
|
+ if ($dst !~ /[0-9]+/) { $dst = $regrm{"%e$dst"}; }
|
|
|
|
|
|
+ rex(\@opcode,$src,$dst,0x8);
|
|
|
|
|
|
+ push @opcode,0x0f,0x7e;
|
|
|
|
|
|
+ push @opcode,0xc0|(($src&7)<<3)|($dst&7); # ModR/M
|
|
|
|
|
|
+ @opcode;
|
|
|
|
|
|
+ } elsif ($arg =~ /%r(\w+),%xmm([0-9]+)/) {
|
|
|
|
|
|
+ my ($src,$dst)=($2,$1);
|
|
|
|
|
|
+ if ($dst !~ /[0-9]+/) { $dst = $regrm{"%e$dst"}; }
|
|
|
|
|
|
+ rex(\@opcode,$src,$dst,0x8);
|
|
|
|
|
|
+ push @opcode,0x0f,0x6e;
|
|
|
|
|
|
+ push @opcode,0xc0|(($src&7)<<3)|($dst&7); # ModR/M
|
|
|
|
|
|
+ @opcode;
|
|
|
|
|
|
+ } else {
|
|
|
|
|
|
+ ();
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+};
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $pextrd = sub {
|
|
|
|
|
|
+ if (shift =~ /\$([0-9]+),%xmm([0-9]+),(%\w+)/) {
|
|
|
|
|
|
+ my @opcode=(0x66);
|
|
|
|
|
|
+ $imm=$1;
|
|
|
|
|
|
+ $src=$2;
|
|
|
|
|
|
+ $dst=$3;
|
|
|
|
|
|
+ if ($dst =~ /%r([0-9]+)d/) { $dst = $1; }
|
|
|
|
|
|
+ elsif ($dst =~ /%e/) { $dst = $regrm{$dst}; }
|
|
|
|
|
|
+ rex(\@opcode,$src,$dst);
|
|
|
|
|
|
+ push @opcode,0x0f,0x3a,0x16;
|
|
|
|
|
|
+ push @opcode,0xc0|(($src&7)<<3)|($dst&7); # ModR/M
|
|
|
|
|
|
+ push @opcode,$imm;
|
|
|
|
|
|
+ @opcode;
|
|
|
|
|
|
+ } else {
|
|
|
|
|
|
+ ();
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+};
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $pinsrd = sub {
|
|
|
|
|
|
+ if (shift =~ /\$([0-9]+),(%\w+),%xmm([0-9]+)/) {
|
|
|
|
|
|
+ my @opcode=(0x66);
|
|
|
|
|
|
+ $imm=$1;
|
|
|
|
|
|
+ $src=$2;
|
|
|
|
|
|
+ $dst=$3;
|
|
|
|
|
|
+ if ($src =~ /%r([0-9]+)/) { $src = $1; }
|
|
|
|
|
|
+ elsif ($src =~ /%e/) { $src = $regrm{$src}; }
|
|
|
|
|
|
+ rex(\@opcode,$dst,$src);
|
|
|
|
|
|
+ push @opcode,0x0f,0x3a,0x22;
|
|
|
|
|
|
+ push @opcode,0xc0|(($dst&7)<<3)|($src&7); # ModR/M
|
|
|
|
|
|
+ push @opcode,$imm;
|
|
|
|
|
|
+ @opcode;
|
|
|
|
|
|
+ } else {
|
|
|
|
|
|
+ ();
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+};
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $pshufb = sub {
|
|
|
|
|
|
+ if (shift =~ /%xmm([0-9]+),%xmm([0-9]+)/) {
|
|
|
|
|
|
+ my @opcode=(0x66);
|
|
|
|
|
|
+ rex(\@opcode,$2,$1);
|
|
|
|
|
|
+ push @opcode,0x0f,0x38,0x00;
|
|
|
|
|
|
+ push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M
|
|
|
|
|
|
+ @opcode;
|
|
|
|
|
|
+ } else {
|
|
|
|
|
|
+ ();
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+};
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $palignr = sub {
|
|
|
|
|
|
+ if (shift =~ /\$([0-9]+),%xmm([0-9]+),%xmm([0-9]+)/) {
|
|
|
|
|
|
+ my @opcode=(0x66);
|
|
|
|
|
|
+ rex(\@opcode,$3,$2);
|
|
|
|
|
|
+ push @opcode,0x0f,0x3a,0x0f;
|
|
|
|
|
|
+ push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M
|
|
|
|
|
|
+ push @opcode,$1;
|
|
|
|
|
|
+ @opcode;
|
|
|
|
|
|
+ } else {
|
|
|
|
|
|
+ ();
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+};
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $pclmulqdq = sub {
|
|
|
|
|
|
+ if (shift =~ /\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
|
|
|
|
|
|
+ my @opcode=(0x66);
|
|
|
|
|
|
+ rex(\@opcode,$3,$2);
|
|
|
|
|
|
+ push @opcode,0x0f,0x3a,0x44;
|
|
|
|
|
|
+ push @opcode,0xc0|($2&7)|(($3&7)<<3); # ModR/M
|
|
|
|
|
|
+ my $c=$1;
|
|
|
|
|
|
+ push @opcode,$c=~/^0/?oct($c):$c;
|
|
|
|
|
|
+ @opcode;
|
|
|
|
|
|
+ } else {
|
|
|
|
|
|
+ ();
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+};
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $rdrand = sub {
|
|
|
|
|
|
+ if (shift =~ /%[er](\w+)/) {
|
|
|
|
|
|
+ my @opcode=();
|
|
|
|
|
|
+ my $dst=$1;
|
|
|
|
|
|
+ if ($dst !~ /[0-9]+/) { $dst = $regrm{"%e$dst"}; }
|
|
|
|
|
|
+ rex(\@opcode,0,$1,8);
|
|
|
|
|
|
+ push @opcode,0x0f,0xc7,0xf0|($dst&7);
|
|
|
|
|
|
+ @opcode;
|
|
|
|
|
|
+ } else {
|
|
|
|
|
|
+ ();
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+};
|
|
|
|
|
|
+
|
|
|
|
|
|
if ($nasm) {
|
|
|
|
|
|
print <<___;
|
|
|
|
|
|
default rel
|
|
|
|
|
|
+%define XMMWORD
|
|
|
|
|
|
___
|
|
|
|
|
|
} elsif ($masm) {
|
|
|
|
|
|
print <<___;
|
|
|
|
|
|
@@ -644,14 +793,22 @@ while($line=<>) {
|
|
|
|
|
|
|
|
|
|
|
|
undef $label;
|
|
|
|
|
|
undef $opcode;
|
|
|
|
|
|
- undef $sz;
|
|
|
|
|
|
undef @args;
|
|
|
|
|
|
|
|
|
|
|
|
if ($label=label->re(\$line)) { print $label->out(); }
|
|
|
|
|
|
|
|
|
|
|
|
if (directive->re(\$line)) {
|
|
|
|
|
|
printf "%s",directive->out();
|
|
|
|
|
|
- } elsif ($opcode=opcode->re(\$line)) { ARGUMENT: while (1) {
|
|
|
|
|
|
+ } elsif ($opcode=opcode->re(\$line)) {
|
|
|
|
|
|
+ my $asm = eval("\$".$opcode->mnemonic());
|
|
|
|
|
|
+ undef @bytes;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ if ((ref($asm) eq 'CODE') && scalar(@bytes=&$asm($line))) {
|
|
|
|
|
|
+ print $gas?".byte\t":"DB\t",join(',',@bytes),"\n";
|
|
|
|
|
|
+ next;
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+
|
|
|
|
|
|
+ ARGUMENT: while (1) {
|
|
|
|
|
|
my $arg;
|
|
|
|
|
|
|
|
|
|
|
|
if ($arg=register->re(\$line)) { opcode->size($arg->size()); }
|
|
|
|
|
|
@@ -667,19 +824,26 @@ while($line=<>) {
|
|
|
|
|
|
$line =~ s/^,\s*//;
|
|
|
|
|
|
} # ARGUMENT:
|
|
|
|
|
|
|
|
|
|
|
|
- $sz=opcode->size();
|
|
|
|
|
|
-
|
|
|
|
|
|
if ($#args>=0) {
|
|
|
|
|
|
my $insn;
|
|
|
|
|
|
+ my $sz=opcode->size();
|
|
|
|
|
|
+
|
|
|
|
|
|
if ($gas) {
|
|
|
|
|
|
$insn = $opcode->out($#args>=1?$args[$#args]->size():$sz);
|
|
|
|
|
|
+ @args = map($_->out($sz),@args);
|
|
|
|
|
|
+ printf "\t%s\t%s",$insn,join(",",@args);
|
|
|
|
|
|
} else {
|
|
|
|
|
|
$insn = $opcode->out();
|
|
|
|
|
|
- $insn .= $sz if (map($_->out() =~ /x?mm/,@args));
|
|
|
|
|
|
+ foreach (@args) {
|
|
|
|
|
|
+ my $arg = $_->out();
|
|
|
|
|
|
+ # $insn.=$sz compensates for movq, pinsrw, ...
|
|
|
|
|
|
+ if ($arg =~ /^xmm[0-9]+$/) { $insn.=$sz; $sz="x" if(!$sz); last; }
|
|
|
|
|
|
+ if ($arg =~ /^mm[0-9]+$/) { $insn.=$sz; $sz="q" if(!$sz); last; }
|
|
|
|
|
|
+ }
|
|
|
|
|
|
@args = reverse(@args);
|
|
|
|
|
|
undef $sz if ($nasm && $opcode->mnemonic() eq "lea");
|
|
|
|
|
|
+ printf "\t%s\t%s",$insn,join(",",map($_->out($sz),@args));
|
|
|
|
|
|
}
|
|
|
|
|
|
- printf "\t%s\t%s",$insn,join(",",map($_->out($sz),@args));
|
|
|
|
|
|
} else {
|
|
|
|
|
|
printf "\t%s",$opcode->out();
|
|
|
|
|
|
}
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/rc4/asm/rc4-x86_64.pl.intelopts openssl-1.0.0d/crypto/rc4/asm/rc4-x86_64.pl
|
2011-11-03 09:18:52 +00:00
|
|
|
|
--- openssl-1.0.0d/crypto/rc4/asm/rc4-x86_64.pl.intelopts 2011-08-24 12:50:55.000000000 +0200
|
|
|
|
|
|
+++ openssl-1.0.0d/crypto/rc4/asm/rc4-x86_64.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
#
|
|
|
|
|
|
# ====================================================================
|
|
|
|
|
|
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
|
|
|
|
|
|
@@ -7,6 +7,8 @@
|
|
|
|
|
|
# details see http://www.openssl.org/~appro/cryptogams/.
|
|
|
|
|
|
# ====================================================================
|
|
|
|
|
|
#
|
|
|
|
|
|
+# July 2004
|
|
|
|
|
|
+#
|
|
|
|
|
|
# 2.22x RC4 tune-up:-) It should be noted though that my hand [as in
|
|
|
|
|
|
# "hand-coded assembler"] doesn't stand for the whole improvement
|
|
|
|
|
|
# coefficient. It turned out that eliminating RC4_CHAR from config
|
|
|
|
|
|
@@ -19,6 +21,8 @@
|
|
|
|
|
|
# to operate on partial registers, it turned out to be the best bet.
|
|
|
|
|
|
# At least for AMD... How IA32E would perform remains to be seen...
|
|
|
|
|
|
|
|
|
|
|
|
+# November 2004
|
|
|
|
|
|
+#
|
|
|
|
|
|
# As was shown by Marc Bevand reordering of couple of load operations
|
|
|
|
|
|
# results in even higher performance gain of 3.3x:-) At least on
|
|
|
|
|
|
# Opteron... For reference, 1x in this case is RC4_CHAR C-code
|
|
|
|
|
|
@@ -26,6 +30,8 @@
|
|
|
|
|
|
# Latter means that if you want to *estimate* what to expect from
|
|
|
|
|
|
# *your* Opteron, then multiply 54 by 3.3 and clock frequency in GHz.
|
|
|
|
|
|
|
|
|
|
|
|
+# November 2004
|
|
|
|
|
|
+#
|
|
|
|
|
|
# Intel P4 EM64T core was found to run the AMD64 code really slow...
|
|
|
|
|
|
# The only way to achieve comparable performance on P4 was to keep
|
|
|
|
|
|
# RC4_CHAR. Kind of ironic, huh? As it's apparently impossible to
|
|
|
|
|
|
@@ -33,10 +39,14 @@
|
|
|
|
|
|
# on either AMD and Intel platforms, I implement both cases. See
|
|
|
|
|
|
# rc4_skey.c for further details...
|
|
|
|
|
|
|
|
|
|
|
|
+# April 2005
|
|
|
|
|
|
+#
|
|
|
|
|
|
# P4 EM64T core appears to be "allergic" to 64-bit inc/dec. Replacing
|
|
|
|
|
|
# those with add/sub results in 50% performance improvement of folded
|
|
|
|
|
|
# loop...
|
|
|
|
|
|
|
|
|
|
|
|
+# May 2005
|
|
|
|
|
|
+#
|
|
|
|
|
|
# As was shown by Zou Nanhai loop unrolling can improve Intel EM64T
|
|
|
|
|
|
# performance by >30% [unlike P4 32-bit case that is]. But this is
|
|
|
|
|
|
# provided that loads are reordered even more aggressively! Both code
|
|
|
|
|
|
@@ -50,6 +60,8 @@
|
|
|
|
|
|
# is not implemented, then this final RC4_CHAR code-path should be
|
|
|
|
|
|
# preferred, as it provides better *all-round* performance].
|
|
|
|
|
|
|
|
|
|
|
|
+# March 2007
|
|
|
|
|
|
+#
|
|
|
|
|
|
# Intel Core2 was observed to perform poorly on both code paths:-( It
|
|
|
|
|
|
# apparently suffers from some kind of partial register stall, which
|
|
|
|
|
|
# occurs in 64-bit mode only [as virtually identical 32-bit loop was
|
|
|
|
|
|
@@ -58,6 +70,34 @@
|
|
|
|
|
|
# fit for Core2 and therefore the code was modified to skip cloop8 on
|
|
|
|
|
|
# this CPU.
|
|
|
|
|
|
|
|
|
|
|
|
+# May 2010
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Intel Westmere was observed to perform suboptimally. Adding yet
|
|
|
|
|
|
+# another movzb to cloop1 improved performance by almost 50%! Core2
|
|
|
|
|
|
+# performance is improved too, but nominally...
|
|
|
|
|
|
+
|
|
|
|
|
|
+# May 2011
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# The only code path that was not modified is P4-specific one. Non-P4
|
|
|
|
|
|
+# Intel code path optimization is heavily based on submission by Maxim
|
|
|
|
|
|
+# Perminov, Maxim Locktyukhin and Jim Guilford of Intel. I've used
|
|
|
|
|
|
+# some of the ideas even in attempt to optmize the original RC4_INT
|
|
|
|
|
|
+# code path... Current performance in cycles per processed byte (less
|
|
|
|
|
|
+# is better) and improvement coefficients relative to previous
|
|
|
|
|
|
+# version of this module are:
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Opteron 5.3/+0%
|
|
|
|
|
|
+# P4 6.5
|
|
|
|
|
|
+# Core2 6.2/+15%(*)
|
|
|
|
|
|
+# Westmere 4.2/+60%
|
|
|
|
|
|
+# Sandy Bridge 4.2/+120%
|
|
|
|
|
|
+# Atom 9.3/+80%
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# (*) Note that Core2 result is ~15% lower than corresponding result
|
|
|
|
|
|
+# for 32-bit code, meaning that it's possible to improve it,
|
|
|
|
|
|
+# but more than likely at the cost of the others (see rc4-586.pl
|
|
|
|
|
|
+# to get the idea)...
|
|
|
|
|
|
+
|
|
|
|
|
|
$flavour = shift;
|
|
|
|
|
|
$output = shift;
|
|
|
|
|
|
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
|
|
|
|
|
|
@@ -76,13 +116,10 @@ $len="%rsi"; # arg2
|
|
|
|
|
|
$inp="%rdx"; # arg3
|
|
|
|
|
|
$out="%rcx"; # arg4
|
|
|
|
|
|
|
|
|
|
|
|
-@XX=("%r8","%r10");
|
|
|
|
|
|
-@TX=("%r9","%r11");
|
|
|
|
|
|
-$YY="%r12";
|
|
|
|
|
|
-$TY="%r13";
|
|
|
|
|
|
-
|
|
|
|
|
|
+{
|
|
|
|
|
|
$code=<<___;
|
|
|
|
|
|
.text
|
|
|
|
|
|
+.extern OPENSSL_ia32cap_P
|
|
|
|
|
|
|
|
|
|
|
|
.globl RC4
|
|
|
|
|
|
.type RC4,\@function,4
|
|
|
|
|
|
@@ -95,48 +132,173 @@ RC4: or $len,$len
|
|
|
|
|
|
push %r12
|
|
|
|
|
|
push %r13
|
|
|
|
|
|
.Lprologue:
|
|
|
|
|
|
+ mov $len,%r11
|
|
|
|
|
|
+ mov $inp,%r12
|
|
|
|
|
|
+ mov $out,%r13
|
|
|
|
|
|
+___
|
|
|
|
|
|
+my $len="%r11"; # reassign input arguments
|
|
|
|
|
|
+my $inp="%r12";
|
|
|
|
|
|
+my $out="%r13";
|
|
|
|
|
|
+
|
|
|
|
|
|
+my @XX=("%r10","%rsi");
|
|
|
|
|
|
+my @TX=("%rax","%rbx");
|
|
|
|
|
|
+my $YY="%rcx";
|
|
|
|
|
|
+my $TY="%rdx";
|
|
|
|
|
|
|
|
|
|
|
|
- add \$8,$dat
|
|
|
|
|
|
- movl -8($dat),$XX[0]#d
|
|
|
|
|
|
- movl -4($dat),$YY#d
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ xor $XX[0],$XX[0]
|
|
|
|
|
|
+ xor $YY,$YY
|
|
|
|
|
|
+
|
|
|
|
|
|
+ lea 8($dat),$dat
|
|
|
|
|
|
+ mov -8($dat),$XX[0]#b
|
|
|
|
|
|
+ mov -4($dat),$YY#b
|
|
|
|
|
|
cmpl \$-1,256($dat)
|
|
|
|
|
|
je .LRC4_CHAR
|
|
|
|
|
|
+ mov OPENSSL_ia32cap_P(%rip),%r8d
|
|
|
|
|
|
+ xor $TX[1],$TX[1]
|
|
|
|
|
|
inc $XX[0]#b
|
|
|
|
|
|
+ sub $XX[0],$TX[1]
|
|
|
|
|
|
+ sub $inp,$out
|
|
|
|
|
|
movl ($dat,$XX[0],4),$TX[0]#d
|
|
|
|
|
|
- test \$-8,$len
|
|
|
|
|
|
+ test \$-16,$len
|
|
|
|
|
|
jz .Lloop1
|
|
|
|
|
|
- jmp .Lloop8
|
|
|
|
|
|
+ bt \$30,%r8d # Intel CPU?
|
|
|
|
|
|
+ jc .Lintel
|
|
|
|
|
|
+ and \$7,$TX[1]
|
|
|
|
|
|
+ lea 1($XX[0]),$XX[1]
|
|
|
|
|
|
+ jz .Loop8
|
|
|
|
|
|
+ sub $TX[1],$len
|
|
|
|
|
|
+.Loop8_warmup:
|
|
|
|
|
|
+ add $TX[0]#b,$YY#b
|
|
|
|
|
|
+ movl ($dat,$YY,4),$TY#d
|
|
|
|
|
|
+ movl $TX[0]#d,($dat,$YY,4)
|
|
|
|
|
|
+ movl $TY#d,($dat,$XX[0],4)
|
|
|
|
|
|
+ add $TY#b,$TX[0]#b
|
|
|
|
|
|
+ inc $XX[0]#b
|
|
|
|
|
|
+ movl ($dat,$TX[0],4),$TY#d
|
|
|
|
|
|
+ movl ($dat,$XX[0],4),$TX[0]#d
|
|
|
|
|
|
+ xorb ($inp),$TY#b
|
|
|
|
|
|
+ movb $TY#b,($out,$inp)
|
|
|
|
|
|
+ lea 1($inp),$inp
|
|
|
|
|
|
+ dec $TX[1]
|
|
|
|
|
|
+ jnz .Loop8_warmup
|
|
|
|
|
|
+
|
|
|
|
|
|
+ lea 1($XX[0]),$XX[1]
|
|
|
|
|
|
+ jmp .Loop8
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
-.Lloop8:
|
|
|
|
|
|
+.Loop8:
|
|
|
|
|
|
___
|
|
|
|
|
|
for ($i=0;$i<8;$i++) {
|
|
|
|
|
|
+$code.=<<___ if ($i==7);
|
|
|
|
|
|
+ add \$8,$XX[1]#b
|
|
|
|
|
|
+___
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
add $TX[0]#b,$YY#b
|
|
|
|
|
|
- mov $XX[0],$XX[1]
|
|
|
|
|
|
movl ($dat,$YY,4),$TY#d
|
|
|
|
|
|
- ror \$8,%rax # ror is redundant when $i=0
|
|
|
|
|
|
- inc $XX[1]#b
|
|
|
|
|
|
- movl ($dat,$XX[1],4),$TX[1]#d
|
|
|
|
|
|
- cmp $XX[1],$YY
|
|
|
|
|
|
movl $TX[0]#d,($dat,$YY,4)
|
|
|
|
|
|
- cmove $TX[0],$TX[1]
|
|
|
|
|
|
- movl $TY#d,($dat,$XX[0],4)
|
|
|
|
|
|
+ movl `4*($i==7?-1:$i)`($dat,$XX[1],4),$TX[1]#d
|
|
|
|
|
|
+ ror \$8,%r8 # ror is redundant when $i=0
|
|
|
|
|
|
+ movl $TY#d,4*$i($dat,$XX[0],4)
|
|
|
|
|
|
add $TX[0]#b,$TY#b
|
|
|
|
|
|
- movb ($dat,$TY,4),%al
|
|
|
|
|
|
+ movb ($dat,$TY,4),%r8b
|
|
|
|
|
|
___
|
|
|
|
|
|
-push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
|
|
|
|
|
|
+push(@TX,shift(@TX)); #push(@XX,shift(@XX)); # "rotate" registers
|
|
|
|
|
|
}
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
- ror \$8,%rax
|
|
|
|
|
|
+ add \$8,$XX[0]#b
|
|
|
|
|
|
+ ror \$8,%r8
|
|
|
|
|
|
sub \$8,$len
|
|
|
|
|
|
|
|
|
|
|
|
- xor ($inp),%rax
|
|
|
|
|
|
- add \$8,$inp
|
|
|
|
|
|
- mov %rax,($out)
|
|
|
|
|
|
- add \$8,$out
|
|
|
|
|
|
+ xor ($inp),%r8
|
|
|
|
|
|
+ mov %r8,($out,$inp)
|
|
|
|
|
|
+ lea 8($inp),$inp
|
|
|
|
|
|
|
|
|
|
|
|
test \$-8,$len
|
|
|
|
|
|
- jnz .Lloop8
|
|
|
|
|
|
+ jnz .Loop8
|
|
|
|
|
|
+ cmp \$0,$len
|
|
|
|
|
|
+ jne .Lloop1
|
|
|
|
|
|
+ jmp .Lexit
|
|
|
|
|
|
+
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lintel:
|
|
|
|
|
|
+ test \$-32,$len
|
|
|
|
|
|
+ jz .Lloop1
|
|
|
|
|
|
+ and \$15,$TX[1]
|
|
|
|
|
|
+ jz .Loop16_is_hot
|
|
|
|
|
|
+ sub $TX[1],$len
|
|
|
|
|
|
+.Loop16_warmup:
|
|
|
|
|
|
+ add $TX[0]#b,$YY#b
|
|
|
|
|
|
+ movl ($dat,$YY,4),$TY#d
|
|
|
|
|
|
+ movl $TX[0]#d,($dat,$YY,4)
|
|
|
|
|
|
+ movl $TY#d,($dat,$XX[0],4)
|
|
|
|
|
|
+ add $TY#b,$TX[0]#b
|
|
|
|
|
|
+ inc $XX[0]#b
|
|
|
|
|
|
+ movl ($dat,$TX[0],4),$TY#d
|
|
|
|
|
|
+ movl ($dat,$XX[0],4),$TX[0]#d
|
|
|
|
|
|
+ xorb ($inp),$TY#b
|
|
|
|
|
|
+ movb $TY#b,($out,$inp)
|
|
|
|
|
|
+ lea 1($inp),$inp
|
|
|
|
|
|
+ dec $TX[1]
|
|
|
|
|
|
+ jnz .Loop16_warmup
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov $YY,$TX[1]
|
|
|
|
|
|
+ xor $YY,$YY
|
|
|
|
|
|
+ mov $TX[1]#b,$YY#b
|
|
|
|
|
|
+
|
|
|
|
|
|
+.Loop16_is_hot:
|
|
|
|
|
|
+ lea ($dat,$XX[0],4),$XX[1]
|
|
|
|
|
|
+___
|
|
|
|
|
|
+sub RC4_loop {
|
|
|
|
|
|
+ my $i=shift;
|
|
|
|
|
|
+ my $j=$i<0?0:$i;
|
|
|
|
|
|
+ my $xmm="%xmm".($j&1);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $code.=" add \$16,$XX[0]#b\n" if ($i==15);
|
|
|
|
|
|
+ $code.=" movdqu ($inp),%xmm2\n" if ($i==15);
|
|
|
|
|
|
+ $code.=" add $TX[0]#b,$YY#b\n" if ($i<=0);
|
|
|
|
|
|
+ $code.=" movl ($dat,$YY,4),$TY#d\n";
|
|
|
|
|
|
+ $code.=" pxor %xmm0,%xmm2\n" if ($i==0);
|
|
|
|
|
|
+ $code.=" psllq \$8,%xmm1\n" if ($i==0);
|
|
|
|
|
|
+ $code.=" pxor $xmm,$xmm\n" if ($i<=1);
|
|
|
|
|
|
+ $code.=" movl $TX[0]#d,($dat,$YY,4)\n";
|
|
|
|
|
|
+ $code.=" add $TY#b,$TX[0]#b\n";
|
|
|
|
|
|
+ $code.=" movl `4*($j+1)`($XX[1]),$TX[1]#d\n" if ($i<15);
|
|
|
|
|
|
+ $code.=" movz $TX[0]#b,$TX[0]#d\n";
|
|
|
|
|
|
+ $code.=" movl $TY#d,`4*$j`($XX[1])\n";
|
|
|
|
|
|
+ $code.=" pxor %xmm1,%xmm2\n" if ($i==0);
|
|
|
|
|
|
+ $code.=" lea ($dat,$XX[0],4),$XX[1]\n" if ($i==15);
|
|
|
|
|
|
+ $code.=" add $TX[1]#b,$YY#b\n" if ($i<15);
|
|
|
|
|
|
+ $code.=" pinsrw \$`$j>>1`,($dat,$TX[0],4),$xmm\n";
|
|
|
|
|
|
+ $code.=" movdqu %xmm2,($out,$inp)\n" if ($i==0);
|
|
|
|
|
|
+ $code.=" lea 16($inp),$inp\n" if ($i==0);
|
|
|
|
|
|
+ $code.=" movl ($XX[1]),$TX[1]#d\n" if ($i==15);
|
|
|
|
|
|
+}
|
|
|
|
|
|
+ RC4_loop(-1);
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ jmp .Loop16_enter
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Loop16:
|
|
|
|
|
|
+___
|
|
|
|
|
|
+
|
|
|
|
|
|
+for ($i=0;$i<16;$i++) {
|
|
|
|
|
|
+ $code.=".Loop16_enter:\n" if ($i==1);
|
|
|
|
|
|
+ RC4_loop($i);
|
|
|
|
|
|
+ push(@TX,shift(@TX)); # "rotate" registers
|
|
|
|
|
|
+}
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ mov $YY,$TX[1]
|
|
|
|
|
|
+ xor $YY,$YY # keyword to partial register
|
|
|
|
|
|
+ sub \$16,$len
|
|
|
|
|
|
+ mov $TX[1]#b,$YY#b
|
|
|
|
|
|
+ test \$-16,$len
|
|
|
|
|
|
+ jnz .Loop16
|
|
|
|
|
|
+
|
|
|
|
|
|
+ psllq \$8,%xmm1
|
|
|
|
|
|
+ pxor %xmm0,%xmm2
|
|
|
|
|
|
+ pxor %xmm1,%xmm2
|
|
|
|
|
|
+ movdqu %xmm2,($out,$inp)
|
|
|
|
|
|
+ lea 16($inp),$inp
|
|
|
|
|
|
+
|
|
|
|
|
|
cmp \$0,$len
|
|
|
|
|
|
jne .Lloop1
|
|
|
|
|
|
jmp .Lexit
|
|
|
|
|
|
@@ -152,9 +314,8 @@ $code.=<<___;
|
|
|
|
|
|
movl ($dat,$TX[0],4),$TY#d
|
|
|
|
|
|
movl ($dat,$XX[0],4),$TX[0]#d
|
|
|
|
|
|
xorb ($inp),$TY#b
|
|
|
|
|
|
- inc $inp
|
|
|
|
|
|
- movb $TY#b,($out)
|
|
|
|
|
|
- inc $out
|
|
|
|
|
|
+ movb $TY#b,($out,$inp)
|
|
|
|
|
|
+ lea 1($inp),$inp
|
|
|
|
|
|
dec $len
|
|
|
|
|
|
jnz .Lloop1
|
|
|
|
|
|
jmp .Lexit
|
|
|
|
|
|
@@ -165,13 +326,11 @@ $code.=<<___;
|
|
|
|
|
|
movzb ($dat,$XX[0]),$TX[0]#d
|
|
|
|
|
|
test \$-8,$len
|
|
|
|
|
|
jz .Lcloop1
|
|
|
|
|
|
- cmpl \$0,260($dat)
|
|
|
|
|
|
- jnz .Lcloop1
|
|
|
|
|
|
jmp .Lcloop8
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lcloop8:
|
|
|
|
|
|
- mov ($inp),%eax
|
|
|
|
|
|
- mov 4($inp),%ebx
|
|
|
|
|
|
+ mov ($inp),%r8d
|
|
|
|
|
|
+ mov 4($inp),%r9d
|
|
|
|
|
|
___
|
|
|
|
|
|
# unroll 2x4-wise, because 64-bit rotates kill Intel P4...
|
|
|
|
|
|
for ($i=0;$i<4;$i++) {
|
|
|
|
|
|
@@ -188,8 +347,8 @@ $code.=<<___;
|
|
|
|
|
|
mov $TX[0],$TX[1]
|
|
|
|
|
|
.Lcmov$i:
|
|
|
|
|
|
add $TX[0]#b,$TY#b
|
|
|
|
|
|
- xor ($dat,$TY),%al
|
|
|
|
|
|
- ror \$8,%eax
|
|
|
|
|
|
+ xor ($dat,$TY),%r8b
|
|
|
|
|
|
+ ror \$8,%r8d
|
|
|
|
|
|
___
|
|
|
|
|
|
push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
|
|
|
|
|
|
}
|
|
|
|
|
|
@@ -207,16 +366,16 @@ $code.=<<___;
|
|
|
|
|
|
mov $TX[0],$TX[1]
|
|
|
|
|
|
.Lcmov$i:
|
|
|
|
|
|
add $TX[0]#b,$TY#b
|
|
|
|
|
|
- xor ($dat,$TY),%bl
|
|
|
|
|
|
- ror \$8,%ebx
|
|
|
|
|
|
+ xor ($dat,$TY),%r9b
|
|
|
|
|
|
+ ror \$8,%r9d
|
|
|
|
|
|
___
|
|
|
|
|
|
push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
|
|
|
|
|
|
}
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
lea -8($len),$len
|
|
|
|
|
|
- mov %eax,($out)
|
|
|
|
|
|
+ mov %r8d,($out)
|
|
|
|
|
|
lea 8($inp),$inp
|
|
|
|
|
|
- mov %ebx,4($out)
|
|
|
|
|
|
+ mov %r9d,4($out)
|
|
|
|
|
|
lea 8($out),$out
|
|
|
|
|
|
|
|
|
|
|
|
test \$-8,$len
|
|
|
|
|
|
@@ -229,6 +388,7 @@ $code.=<<___;
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lcloop1:
|
|
|
|
|
|
add $TX[0]#b,$YY#b
|
|
|
|
|
|
+ movzb $YY#b,$YY#d
|
|
|
|
|
|
movzb ($dat,$YY),$TY#d
|
|
|
|
|
|
movb $TX[0]#b,($dat,$YY)
|
|
|
|
|
|
movb $TY#b,($dat,$XX[0])
|
|
|
|
|
|
@@ -260,12 +420,12 @@ $code.=<<___;
|
|
|
|
|
|
ret
|
|
|
|
|
|
.size RC4,.-RC4
|
|
|
|
|
|
___
|
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
|
|
|
|
$idx="%r8";
|
|
|
|
|
|
$ido="%r9";
|
|
|
|
|
|
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
-.extern OPENSSL_ia32cap_P
|
|
|
|
|
|
.globl RC4_set_key
|
|
|
|
|
|
.type RC4_set_key,\@function,3
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
@@ -280,12 +440,9 @@ RC4_set_key:
|
|
|
|
|
|
xor %r11,%r11
|
|
|
|
|
|
|
|
|
|
|
|
mov OPENSSL_ia32cap_P(%rip),$idx#d
|
|
|
|
|
|
- bt \$20,$idx#d
|
|
|
|
|
|
- jnc .Lw1stloop
|
|
|
|
|
|
- bt \$30,$idx#d
|
|
|
|
|
|
- setc $ido#b
|
|
|
|
|
|
- mov $ido#d,260($dat)
|
|
|
|
|
|
- jmp .Lc1stloop
|
|
|
|
|
|
+ bt \$20,$idx#d # RC4_CHAR?
|
|
|
|
|
|
+ jc .Lc1stloop
|
|
|
|
|
|
+ jmp .Lw1stloop
|
|
|
|
|
|
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
.Lw1stloop:
|
|
|
|
|
|
@@ -348,18 +505,20 @@ RC4_options:
|
|
|
|
|
|
lea .Lopts(%rip),%rax
|
|
|
|
|
|
mov OPENSSL_ia32cap_P(%rip),%edx
|
|
|
|
|
|
bt \$20,%edx
|
|
|
|
|
|
- jnc .Ldone
|
|
|
|
|
|
- add \$12,%rax
|
|
|
|
|
|
+ jc .L8xchar
|
|
|
|
|
|
bt \$30,%edx
|
|
|
|
|
|
jnc .Ldone
|
|
|
|
|
|
- add \$13,%rax
|
|
|
|
|
|
+ add \$25,%rax
|
|
|
|
|
|
+ ret
|
|
|
|
|
|
+.L8xchar:
|
|
|
|
|
|
+ add \$12,%rax
|
|
|
|
|
|
.Ldone:
|
|
|
|
|
|
ret
|
|
|
|
|
|
.align 64
|
|
|
|
|
|
.Lopts:
|
|
|
|
|
|
.asciz "rc4(8x,int)"
|
|
|
|
|
|
.asciz "rc4(8x,char)"
|
|
|
|
|
|
-.asciz "rc4(1x,char)"
|
|
|
|
|
|
+.asciz "rc4(16x,int)"
|
|
|
|
|
|
.asciz "RC4 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
|
|
|
|
|
|
.align 64
|
|
|
|
|
|
.size RC4_options,.-RC4_options
|
|
|
|
|
|
@@ -497,8 +656,17 @@ key_se_handler:
|
|
|
|
|
|
___
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
-$code =~ s/#([bwd])/$1/gm;
|
|
|
|
|
|
+sub reg_part {
|
|
|
|
|
|
+my ($reg,$conv)=@_;
|
|
|
|
|
|
+ if ($reg =~ /%r[0-9]+/) { $reg .= $conv; }
|
|
|
|
|
|
+ elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; }
|
|
|
|
|
|
+ elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; }
|
|
|
|
|
|
+ elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; }
|
|
|
|
|
|
+ return $reg;
|
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
|
|
|
|
+$code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem;
|
|
|
|
|
|
+$code =~ s/\`([^\`]*)\`/eval $1/gem;
|
|
|
|
|
|
$code =~ s/RC4_set_key/private_RC4_set_key/g if ($ENV{FIPS} ne "");
|
|
|
|
|
|
|
|
|
|
|
|
print $code;
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/rc4/asm/rc4-586.pl.intelopts openssl-1.0.0d/crypto/rc4/asm/rc4-586.pl
|
2011-11-03 09:18:52 +00:00
|
|
|
|
--- openssl-1.0.0d/crypto/rc4/asm/rc4-586.pl.intelopts 2011-08-24 12:50:55.000000000 +0200
|
|
|
|
|
|
+++ openssl-1.0.0d/crypto/rc4/asm/rc4-586.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
|
|
|
|
|
|
# ====================================================================
|
|
|
|
|
|
# [Re]written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
|
|
|
|
|
|
@@ -28,6 +28,33 @@
|
|
|
|
|
|
#
|
|
|
|
|
|
# <appro@fy.chalmers.se>
|
|
|
|
|
|
|
|
|
|
|
|
+# May 2011
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Optimize for Core2 and Westmere [and incidentally Opteron]. Current
|
|
|
|
|
|
+# performance in cycles per processed byte (less is better) is:
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Pentium 10.2 # original numbers
|
|
|
|
|
|
+# Pentium III 7.8(*)
|
|
|
|
|
|
+# Intel P4 7.5
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Opteron 6.1/+20% # new MMX numbers
|
|
|
|
|
|
+# Core2 5.3/+67%(**)
|
|
|
|
|
|
+# Westmere 5.1/+94%(**)
|
|
|
|
|
|
+# Sandy Bridge 5.0/+8%
|
|
|
|
|
|
+# Atom 12.6/+6%
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# (*) PIII can actually deliver 6.6 cycles per byte with MMX code,
|
|
|
|
|
|
+# but this specific code performs poorly on Core2. And vice
|
|
|
|
|
|
+# versa, below MMX/SSE code delivering 5.8/7.1 on Core2 performs
|
|
|
|
|
|
+# poorly on PIII, at 8.0/14.5:-( As PIII is not a "hot" CPU
|
|
|
|
|
|
+# [anymore], I chose to discard PIII-specific code path and opt
|
|
|
|
|
|
+# for original IALU-only code, which is why MMX/SSE code path
|
|
|
|
|
|
+# is guarded by SSE2 bit (see below), not MMX/SSE.
|
|
|
|
|
|
+# (**) Performance vs. block size on Core2 and Westmere had a maximum
|
|
|
|
|
|
+# at ... 64 bytes block size. And it was quite a maximum, 40-60%
|
|
|
|
|
|
+# in comparison to largest 8KB block size. Above improvement
|
|
|
|
|
|
+# coefficients are for the largest block size.
|
|
|
|
|
|
+
|
|
|
|
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|
|
|
|
|
push(@INC,"${dir}","${dir}../../perlasm");
|
|
|
|
|
|
require "x86asm.pl";
|
|
|
|
|
|
@@ -62,6 +89,68 @@ sub RC4_loop {
|
|
|
|
|
|
&$func ($out,&DWP(0,$dat,$ty,4));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
+if ($alt=0) {
|
|
|
|
|
|
+ # >20% faster on Atom and Sandy Bridge[!], 8% faster on Opteron,
|
|
|
|
|
|
+ # but ~40% slower on Core2 and Westmere... Attempt to add movz
|
|
|
|
|
|
+ # brings down Opteron by 25%, Atom and Sandy Bridge by 15%, yet
|
|
|
|
|
|
+ # on Core2 with movz it's almost 20% slower than below alternative
|
|
|
|
|
|
+ # code... Yes, it's a total mess...
|
|
|
|
|
|
+ my @XX=($xx,$out);
|
|
|
|
|
|
+ $RC4_loop_mmx = sub { # SSE actually...
|
|
|
|
|
|
+ my $i=shift;
|
|
|
|
|
|
+ my $j=$i<=0?0:$i>>1;
|
|
|
|
|
|
+ my $mm=$i<=0?"mm0":"mm".($i&1);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &add (&LB($yy),&LB($tx));
|
|
|
|
|
|
+ &lea (@XX[1],&DWP(1,@XX[0]));
|
|
|
|
|
|
+ &pxor ("mm2","mm0") if ($i==0);
|
|
|
|
|
|
+ &psllq ("mm1",8) if ($i==0);
|
|
|
|
|
|
+ &and (@XX[1],0xff);
|
|
|
|
|
|
+ &pxor ("mm0","mm0") if ($i<=0);
|
|
|
|
|
|
+ &mov ($ty,&DWP(0,$dat,$yy,4));
|
|
|
|
|
|
+ &mov (&DWP(0,$dat,$yy,4),$tx);
|
|
|
|
|
|
+ &pxor ("mm1","mm2") if ($i==0);
|
|
|
|
|
|
+ &mov (&DWP(0,$dat,$XX[0],4),$ty);
|
|
|
|
|
|
+ &add (&LB($ty),&LB($tx));
|
|
|
|
|
|
+ &movd (@XX[0],"mm7") if ($i==0);
|
|
|
|
|
|
+ &mov ($tx,&DWP(0,$dat,@XX[1],4));
|
|
|
|
|
|
+ &pxor ("mm1","mm1") if ($i==1);
|
|
|
|
|
|
+ &movq ("mm2",&QWP(0,$inp)) if ($i==1);
|
|
|
|
|
|
+ &movq (&QWP(-8,(@XX[0],$inp)),"mm1") if ($i==0);
|
|
|
|
|
|
+ &pinsrw ($mm,&DWP(0,$dat,$ty,4),$j);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ push (@XX,shift(@XX)) if ($i>=0);
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+} else {
|
|
|
|
|
|
+ # Using pinsrw here improves performane on Intel CPUs by 2-3%, but
|
|
|
|
|
|
+ # brings down AMD by 7%...
|
|
|
|
|
|
+ $RC4_loop_mmx = sub {
|
|
|
|
|
|
+ my $i=shift;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &add (&LB($yy),&LB($tx));
|
|
|
|
|
|
+ &psllq ("mm1",8*(($i-1)&7)) if (abs($i)!=1);
|
|
|
|
|
|
+ &mov ($ty,&DWP(0,$dat,$yy,4));
|
|
|
|
|
|
+ &mov (&DWP(0,$dat,$yy,4),$tx);
|
|
|
|
|
|
+ &mov (&DWP(0,$dat,$xx,4),$ty);
|
|
|
|
|
|
+ &inc ($xx);
|
|
|
|
|
|
+ &add ($ty,$tx);
|
|
|
|
|
|
+ &movz ($xx,&LB($xx)); # (*)
|
|
|
|
|
|
+ &movz ($ty,&LB($ty)); # (*)
|
|
|
|
|
|
+ &pxor ("mm2",$i==1?"mm0":"mm1") if ($i>=0);
|
|
|
|
|
|
+ &movq ("mm0",&QWP(0,$inp)) if ($i<=0);
|
|
|
|
|
|
+ &movq (&QWP(-8,($out,$inp)),"mm2") if ($i==0);
|
|
|
|
|
|
+ &mov ($tx,&DWP(0,$dat,$xx,4));
|
|
|
|
|
|
+ &movd ($i>0?"mm1":"mm2",&DWP(0,$dat,$ty,4));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ # (*) This is the key to Core2 and Westmere performance.
|
|
|
|
|
|
+ # Whithout movz out-of-order execution logic confuses
|
|
|
|
|
|
+ # itself and fails to reorder loads and stores. Problem
|
|
|
|
|
|
+ # appears to be fixed in Sandy Bridge...
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+&external_label("OPENSSL_ia32cap_P");
|
|
|
|
|
|
+
|
|
|
|
|
|
# void RC4(RC4_KEY *key,size_t len,const unsigned char *inp,unsigned char *out);
|
|
|
|
|
|
&function_begin("RC4");
|
|
|
|
|
|
&mov ($dat,&wparam(0)); # load key schedule pointer
|
|
|
|
|
|
@@ -94,11 +183,56 @@ sub RC4_loop {
|
|
|
|
|
|
&and ($ty,-4); # how many 4-byte chunks?
|
|
|
|
|
|
&jz (&label("loop1"));
|
|
|
|
|
|
|
|
|
|
|
|
+ &test ($ty,-8);
|
|
|
|
|
|
+ &mov (&wparam(3),$out); # $out as accumulator in these loops
|
|
|
|
|
|
+ &jz (&label("go4loop4"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &picmeup($out,"OPENSSL_ia32cap_P");
|
|
|
|
|
|
+ &bt (&DWP(0,$out),26); # check SSE2 bit [could have been MMX]
|
|
|
|
|
|
+ &jnc (&label("go4loop4"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov ($out,&wparam(3)) if (!$alt);
|
|
|
|
|
|
+ &movd ("mm7",&wparam(3)) if ($alt);
|
|
|
|
|
|
+ &and ($ty,-8);
|
|
|
|
|
|
+ &lea ($ty,&DWP(-8,$inp,$ty));
|
|
|
|
|
|
+ &mov (&DWP(-4,$dat),$ty); # save input+(len/8)*8-8
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &$RC4_loop_mmx(-1);
|
|
|
|
|
|
+ &jmp(&label("loop_mmx_enter"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &set_label("loop_mmx",16);
|
|
|
|
|
|
+ &$RC4_loop_mmx(0);
|
|
|
|
|
|
+ &set_label("loop_mmx_enter");
|
|
|
|
|
|
+ for ($i=1;$i<8;$i++) { &$RC4_loop_mmx($i); }
|
|
|
|
|
|
+ &mov ($ty,$yy);
|
|
|
|
|
|
+ &xor ($yy,$yy); # this is second key to Core2
|
|
|
|
|
|
+ &mov (&LB($yy),&LB($ty)); # and Westmere performance...
|
|
|
|
|
|
+ &cmp ($inp,&DWP(-4,$dat));
|
|
|
|
|
|
+ &lea ($inp,&DWP(8,$inp));
|
|
|
|
|
|
+ &jb (&label("loop_mmx"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ if ($alt) {
|
|
|
|
|
|
+ &movd ($out,"mm7");
|
|
|
|
|
|
+ &pxor ("mm2","mm0");
|
|
|
|
|
|
+ &psllq ("mm1",8);
|
|
|
|
|
|
+ &pxor ("mm1","mm2");
|
|
|
|
|
|
+ &movq (&QWP(-8,$out,$inp),"mm1");
|
|
|
|
|
|
+ } else {
|
|
|
|
|
|
+ &psllq ("mm1",56);
|
|
|
|
|
|
+ &pxor ("mm2","mm1");
|
|
|
|
|
|
+ &movq (&QWP(-8,$out,$inp),"mm2");
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+ &emms ();
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &cmp ($inp,&wparam(1)); # compare to input+len
|
|
|
|
|
|
+ &je (&label("done"));
|
|
|
|
|
|
+ &jmp (&label("loop1"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("go4loop4",16);
|
|
|
|
|
|
&lea ($ty,&DWP(-4,$inp,$ty));
|
|
|
|
|
|
&mov (&wparam(2),$ty); # save input+(len/4)*4-4
|
|
|
|
|
|
- &mov (&wparam(3),$out); # $out as accumulator in this loop
|
|
|
|
|
|
|
|
|
|
|
|
- &set_label("loop4",16);
|
|
|
|
|
|
+ &set_label("loop4");
|
|
|
|
|
|
for ($i=0;$i<4;$i++) { RC4_loop($i); }
|
|
|
|
|
|
&ror ($out,8);
|
|
|
|
|
|
&xor ($out,&DWP(0,$inp));
|
|
|
|
|
|
@@ -151,7 +285,7 @@ sub RC4_loop {
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("done");
|
|
|
|
|
|
&dec (&LB($xx));
|
|
|
|
|
|
- &mov (&BP(-4,$dat),&LB($yy)); # save key->y
|
|
|
|
|
|
+ &mov (&DWP(-4,$dat),$yy); # save key->y
|
|
|
|
|
|
&mov (&BP(-8,$dat),&LB($xx)); # save key->x
|
|
|
|
|
|
&set_label("abort");
|
|
|
|
|
|
&function_end("RC4");
|
|
|
|
|
|
@@ -164,12 +298,9 @@ $idi="ebp";
|
|
|
|
|
|
$ido="ecx";
|
|
|
|
|
|
$idx="edx";
|
|
|
|
|
|
|
|
|
|
|
|
-&external_label("OPENSSL_ia32cap_P");
|
|
|
|
|
|
-
|
|
|
|
|
|
$setkeyfunc = "RC4_set_key";
|
|
|
|
|
|
$setkeyfunc = "private_RC4_set_key" if ($ENV{FIPS} ne "");
|
|
|
|
|
|
|
|
|
|
|
|
-
|
|
|
|
|
|
# void RC4_set_key(RC4_KEY *key,int len,const unsigned char *data);
|
|
|
|
|
|
&function_begin($setkeyfunc);
|
|
|
|
|
|
&mov ($out,&wparam(0)); # load key
|
|
|
|
|
|
@@ -258,14 +389,21 @@ $setkeyfunc = "private_RC4_set_key" if (
|
|
|
|
|
|
&blindpop("eax");
|
|
|
|
|
|
&lea ("eax",&DWP(&label("opts")."-".&label("pic_point"),"eax"));
|
|
|
|
|
|
&picmeup("edx","OPENSSL_ia32cap_P");
|
|
|
|
|
|
- &bt (&DWP(0,"edx"),20);
|
|
|
|
|
|
- &jnc (&label("skip"));
|
|
|
|
|
|
- &add ("eax",12);
|
|
|
|
|
|
- &set_label("skip");
|
|
|
|
|
|
+ &mov ("edx",&DWP(0,"edx"));
|
|
|
|
|
|
+ &bt ("edx",20);
|
|
|
|
|
|
+ &jc (&label("1xchar"));
|
|
|
|
|
|
+ &bt ("edx",26);
|
|
|
|
|
|
+ &jnc (&label("ret"));
|
|
|
|
|
|
+ &add ("eax",25);
|
|
|
|
|
|
+ &ret ();
|
|
|
|
|
|
+&set_label("1xchar");
|
|
|
|
|
|
+ &add ("eax",12);
|
|
|
|
|
|
+&set_label("ret");
|
|
|
|
|
|
&ret ();
|
|
|
|
|
|
&set_label("opts",64);
|
|
|
|
|
|
&asciz ("rc4(4x,int)");
|
|
|
|
|
|
&asciz ("rc4(1x,char)");
|
|
|
|
|
|
+&asciz ("rc4(8x,mmx)");
|
|
|
|
|
|
&asciz ("RC4 for x86, CRYPTOGAMS by <appro\@openssl.org>");
|
|
|
|
|
|
&align (64);
|
|
|
|
|
|
&function_end_B("RC4_options");
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/sha/asm/sha1-x86_64.pl.intelopts openssl-1.0.0d/crypto/sha/asm/sha1-x86_64.pl
|
|
|
|
|
|
--- openssl-1.0.0d/crypto/sha/asm/sha1-x86_64.pl.intelopts 2010-01-17 17:58:56.000000000 +0100
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+++ openssl-1.0.0d/crypto/sha/asm/sha1-x86_64.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
#
|
|
|
|
|
|
# ====================================================================
|
|
|
|
|
|
# Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
|
|
|
|
|
|
@@ -16,7 +16,7 @@
|
|
|
|
|
|
# There was suggestion to mechanically translate 32-bit code, but I
|
|
|
|
|
|
# dismissed it, reasoning that x86_64 offers enough register bank
|
|
|
|
|
|
# capacity to fully utilize SHA-1 parallelism. Therefore this fresh
|
|
|
|
|
|
-# implementation:-) However! While 64-bit code does performs better
|
|
|
|
|
|
+# implementation:-) However! While 64-bit code does perform better
|
|
|
|
|
|
# on Opteron, I failed to beat 32-bit assembler on EM64T core. Well,
|
|
|
|
|
|
# x86_64 does offer larger *addressable* bank, but out-of-order core
|
|
|
|
|
|
# reaches for even more registers through dynamic aliasing, and EM64T
|
|
|
|
|
|
@@ -29,6 +29,38 @@
|
|
|
|
|
|
# Xeon P4 +65% +0% 9.9
|
|
|
|
|
|
# Core2 +60% +10% 7.0
|
|
|
|
|
|
|
|
|
|
|
|
+# August 2009.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# The code was revised to minimize code size and to maximize
|
|
|
|
|
|
+# "distance" between instructions producing input to 'lea'
|
|
|
|
|
|
+# instruction and the 'lea' instruction itself, which is essential
|
|
|
|
|
|
+# for Intel Atom core.
|
|
|
|
|
|
+
|
|
|
|
|
|
+# October 2010.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Add SSSE3, Supplemental[!] SSE3, implementation. The idea behind it
|
|
|
|
|
|
+# is to offload message schedule denoted by Wt in NIST specification,
|
|
|
|
|
|
+# or Xupdate in OpenSSL source, to SIMD unit. See sha1-586.pl module
|
|
|
|
|
|
+# for background and implementation details. The only difference from
|
|
|
|
|
|
+# 32-bit code is that 64-bit code doesn't have to spill @X[] elements
|
|
|
|
|
|
+# to free temporary registers.
|
|
|
|
|
|
+
|
|
|
|
|
|
+# April 2011.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Add AVX code path. See sha1-586.pl for further information.
|
|
|
|
|
|
+
|
|
|
|
|
|
+######################################################################
|
|
|
|
|
|
+# Current performance is summarized in following table. Numbers are
|
|
|
|
|
|
+# CPU clock cycles spent to process single byte (less is better).
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# x86_64 SSSE3 AVX
|
|
|
|
|
|
+# P4 9.8 -
|
|
|
|
|
|
+# Opteron 6.6 -
|
|
|
|
|
|
+# Core2 6.7 6.1/+10% -
|
|
|
|
|
|
+# Atom 11.0 9.7/+13% -
|
|
|
|
|
|
+# Westmere 7.1 5.6/+27% -
|
|
|
|
|
|
+# Sandy Bridge 7.9 6.3/+25% 5.2/+51%
|
|
|
|
|
|
+
|
|
|
|
|
|
$flavour = shift;
|
|
|
|
|
|
$output = shift;
|
|
|
|
|
|
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
|
|
|
|
|
|
@@ -40,6 +72,13 @@ $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|
|
|
|
|
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
|
|
|
|
|
|
die "can't locate x86_64-xlate.pl";
|
|
|
|
|
|
|
|
|
|
|
|
+$avx=1 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
|
|
|
|
|
|
+ =~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
|
|
|
|
|
|
+ $1>=2.19);
|
|
|
|
|
|
+$avx=1 if (!$avx && $flavour =~ /nasm/ &&
|
|
|
|
|
|
+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
|
|
|
|
|
|
+ $1>=2.03);
|
|
|
|
|
|
+
|
|
|
|
|
|
open STDOUT,"| $^X $xlate $flavour $output";
|
|
|
|
|
|
|
|
|
|
|
|
$ctx="%rdi"; # 1st arg
|
|
|
|
|
|
@@ -51,196 +90,994 @@ $ctx="%r8";
|
|
|
|
|
|
$inp="%r9";
|
|
|
|
|
|
$num="%r10";
|
|
|
|
|
|
|
|
|
|
|
|
-$xi="%eax";
|
|
|
|
|
|
-$t0="%ebx";
|
|
|
|
|
|
-$t1="%ecx";
|
|
|
|
|
|
-$A="%edx";
|
|
|
|
|
|
-$B="%esi";
|
|
|
|
|
|
-$C="%edi";
|
|
|
|
|
|
-$D="%ebp";
|
|
|
|
|
|
-$E="%r11d";
|
|
|
|
|
|
-$T="%r12d";
|
|
|
|
|
|
-
|
|
|
|
|
|
-@V=($A,$B,$C,$D,$E,$T);
|
|
|
|
|
|
+$t0="%eax";
|
|
|
|
|
|
+$t1="%ebx";
|
|
|
|
|
|
+$t2="%ecx";
|
|
|
|
|
|
+@xi=("%edx","%ebp");
|
|
|
|
|
|
+$A="%esi";
|
|
|
|
|
|
+$B="%edi";
|
|
|
|
|
|
+$C="%r11d";
|
|
|
|
|
|
+$D="%r12d";
|
|
|
|
|
|
+$E="%r13d";
|
|
|
|
|
|
|
|
|
|
|
|
-sub PROLOGUE {
|
|
|
|
|
|
-my $func=shift;
|
|
|
|
|
|
-$code.=<<___;
|
|
|
|
|
|
-.globl $func
|
|
|
|
|
|
-.type $func,\@function,3
|
|
|
|
|
|
-.align 16
|
|
|
|
|
|
-$func:
|
|
|
|
|
|
- push %rbx
|
|
|
|
|
|
- push %rbp
|
|
|
|
|
|
- push %r12
|
|
|
|
|
|
- mov %rsp,%r11
|
|
|
|
|
|
- mov %rdi,$ctx # reassigned argument
|
|
|
|
|
|
- sub \$`8+16*4`,%rsp
|
|
|
|
|
|
- mov %rsi,$inp # reassigned argument
|
|
|
|
|
|
- and \$-64,%rsp
|
|
|
|
|
|
- mov %rdx,$num # reassigned argument
|
|
|
|
|
|
- mov %r11,`16*4`(%rsp)
|
|
|
|
|
|
-.Lprologue:
|
|
|
|
|
|
-
|
|
|
|
|
|
- mov 0($ctx),$A
|
|
|
|
|
|
- mov 4($ctx),$B
|
|
|
|
|
|
- mov 8($ctx),$C
|
|
|
|
|
|
- mov 12($ctx),$D
|
|
|
|
|
|
- mov 16($ctx),$E
|
|
|
|
|
|
-___
|
|
|
|
|
|
-}
|
|
|
|
|
|
-
|
|
|
|
|
|
-sub EPILOGUE {
|
|
|
|
|
|
-my $func=shift;
|
|
|
|
|
|
-$code.=<<___;
|
|
|
|
|
|
- mov `16*4`(%rsp),%rsi
|
|
|
|
|
|
- mov (%rsi),%r12
|
|
|
|
|
|
- mov 8(%rsi),%rbp
|
|
|
|
|
|
- mov 16(%rsi),%rbx
|
|
|
|
|
|
- lea 24(%rsi),%rsp
|
|
|
|
|
|
-.Lepilogue:
|
|
|
|
|
|
- ret
|
|
|
|
|
|
-.size $func,.-$func
|
|
|
|
|
|
-___
|
|
|
|
|
|
-}
|
|
|
|
|
|
+@V=($A,$B,$C,$D,$E);
|
|
|
|
|
|
|
|
|
|
|
|
sub BODY_00_19 {
|
|
|
|
|
|
-my ($i,$a,$b,$c,$d,$e,$f,$host)=@_;
|
|
|
|
|
|
+my ($i,$a,$b,$c,$d,$e)=@_;
|
|
|
|
|
|
my $j=$i+1;
|
|
|
|
|
|
$code.=<<___ if ($i==0);
|
|
|
|
|
|
- mov `4*$i`($inp),$xi
|
|
|
|
|
|
- `"bswap $xi" if(!defined($host))`
|
|
|
|
|
|
- mov $xi,`4*$i`(%rsp)
|
|
|
|
|
|
+ mov `4*$i`($inp),$xi[0]
|
|
|
|
|
|
+ bswap $xi[0]
|
|
|
|
|
|
+ mov $xi[0],`4*$i`(%rsp)
|
|
|
|
|
|
___
|
|
|
|
|
|
$code.=<<___ if ($i<15);
|
|
|
|
|
|
- lea 0x5a827999($xi,$e),$f
|
|
|
|
|
|
mov $c,$t0
|
|
|
|
|
|
- mov `4*$j`($inp),$xi
|
|
|
|
|
|
- mov $a,$e
|
|
|
|
|
|
+ mov `4*$j`($inp),$xi[1]
|
|
|
|
|
|
+ mov $a,$t2
|
|
|
|
|
|
xor $d,$t0
|
|
|
|
|
|
- `"bswap $xi" if(!defined($host))`
|
|
|
|
|
|
- rol \$5,$e
|
|
|
|
|
|
+ bswap $xi[1]
|
|
|
|
|
|
+ rol \$5,$t2
|
|
|
|
|
|
+ lea 0x5a827999($xi[0],$e),$e
|
|
|
|
|
|
and $b,$t0
|
|
|
|
|
|
- mov $xi,`4*$j`(%rsp)
|
|
|
|
|
|
- add $e,$f
|
|
|
|
|
|
+ mov $xi[1],`4*$j`(%rsp)
|
|
|
|
|
|
+ add $t2,$e
|
|
|
|
|
|
xor $d,$t0
|
|
|
|
|
|
rol \$30,$b
|
|
|
|
|
|
- add $t0,$f
|
|
|
|
|
|
+ add $t0,$e
|
|
|
|
|
|
___
|
|
|
|
|
|
$code.=<<___ if ($i>=15);
|
|
|
|
|
|
- lea 0x5a827999($xi,$e),$f
|
|
|
|
|
|
- mov `4*($j%16)`(%rsp),$xi
|
|
|
|
|
|
+ mov `4*($j%16)`(%rsp),$xi[1]
|
|
|
|
|
|
mov $c,$t0
|
|
|
|
|
|
- mov $a,$e
|
|
|
|
|
|
- xor `4*(($j+2)%16)`(%rsp),$xi
|
|
|
|
|
|
+ mov $a,$t2
|
|
|
|
|
|
+ xor `4*(($j+2)%16)`(%rsp),$xi[1]
|
|
|
|
|
|
xor $d,$t0
|
|
|
|
|
|
- rol \$5,$e
|
|
|
|
|
|
- xor `4*(($j+8)%16)`(%rsp),$xi
|
|
|
|
|
|
+ rol \$5,$t2
|
|
|
|
|
|
+ xor `4*(($j+8)%16)`(%rsp),$xi[1]
|
|
|
|
|
|
and $b,$t0
|
|
|
|
|
|
- add $e,$f
|
|
|
|
|
|
- xor `4*(($j+13)%16)`(%rsp),$xi
|
|
|
|
|
|
+ lea 0x5a827999($xi[0],$e),$e
|
|
|
|
|
|
+ xor `4*(($j+13)%16)`(%rsp),$xi[1]
|
|
|
|
|
|
xor $d,$t0
|
|
|
|
|
|
+ rol \$1,$xi[1]
|
|
|
|
|
|
+ add $t2,$e
|
|
|
|
|
|
rol \$30,$b
|
|
|
|
|
|
- add $t0,$f
|
|
|
|
|
|
- rol \$1,$xi
|
|
|
|
|
|
- mov $xi,`4*($j%16)`(%rsp)
|
|
|
|
|
|
+ mov $xi[1],`4*($j%16)`(%rsp)
|
|
|
|
|
|
+ add $t0,$e
|
|
|
|
|
|
___
|
|
|
|
|
|
+unshift(@xi,pop(@xi));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
sub BODY_20_39 {
|
|
|
|
|
|
-my ($i,$a,$b,$c,$d,$e,$f)=@_;
|
|
|
|
|
|
+my ($i,$a,$b,$c,$d,$e)=@_;
|
|
|
|
|
|
my $j=$i+1;
|
|
|
|
|
|
my $K=($i<40)?0x6ed9eba1:0xca62c1d6;
|
|
|
|
|
|
$code.=<<___ if ($i<79);
|
|
|
|
|
|
- lea $K($xi,$e),$f
|
|
|
|
|
|
- mov `4*($j%16)`(%rsp),$xi
|
|
|
|
|
|
+ mov `4*($j%16)`(%rsp),$xi[1]
|
|
|
|
|
|
mov $c,$t0
|
|
|
|
|
|
- mov $a,$e
|
|
|
|
|
|
- xor `4*(($j+2)%16)`(%rsp),$xi
|
|
|
|
|
|
+ mov $a,$t2
|
|
|
|
|
|
+ xor `4*(($j+2)%16)`(%rsp),$xi[1]
|
|
|
|
|
|
xor $b,$t0
|
|
|
|
|
|
- rol \$5,$e
|
|
|
|
|
|
- xor `4*(($j+8)%16)`(%rsp),$xi
|
|
|
|
|
|
+ rol \$5,$t2
|
|
|
|
|
|
+ lea $K($xi[0],$e),$e
|
|
|
|
|
|
+ xor `4*(($j+8)%16)`(%rsp),$xi[1]
|
|
|
|
|
|
xor $d,$t0
|
|
|
|
|
|
- add $e,$f
|
|
|
|
|
|
- xor `4*(($j+13)%16)`(%rsp),$xi
|
|
|
|
|
|
+ add $t2,$e
|
|
|
|
|
|
+ xor `4*(($j+13)%16)`(%rsp),$xi[1]
|
|
|
|
|
|
rol \$30,$b
|
|
|
|
|
|
- add $t0,$f
|
|
|
|
|
|
- rol \$1,$xi
|
|
|
|
|
|
+ add $t0,$e
|
|
|
|
|
|
+ rol \$1,$xi[1]
|
|
|
|
|
|
___
|
|
|
|
|
|
$code.=<<___ if ($i<76);
|
|
|
|
|
|
- mov $xi,`4*($j%16)`(%rsp)
|
|
|
|
|
|
+ mov $xi[1],`4*($j%16)`(%rsp)
|
|
|
|
|
|
___
|
|
|
|
|
|
$code.=<<___ if ($i==79);
|
|
|
|
|
|
- lea $K($xi,$e),$f
|
|
|
|
|
|
mov $c,$t0
|
|
|
|
|
|
- mov $a,$e
|
|
|
|
|
|
+ mov $a,$t2
|
|
|
|
|
|
xor $b,$t0
|
|
|
|
|
|
- rol \$5,$e
|
|
|
|
|
|
+ lea $K($xi[0],$e),$e
|
|
|
|
|
|
+ rol \$5,$t2
|
|
|
|
|
|
xor $d,$t0
|
|
|
|
|
|
- add $e,$f
|
|
|
|
|
|
+ add $t2,$e
|
|
|
|
|
|
rol \$30,$b
|
|
|
|
|
|
- add $t0,$f
|
|
|
|
|
|
+ add $t0,$e
|
|
|
|
|
|
___
|
|
|
|
|
|
+unshift(@xi,pop(@xi));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
sub BODY_40_59 {
|
|
|
|
|
|
-my ($i,$a,$b,$c,$d,$e,$f)=@_;
|
|
|
|
|
|
+my ($i,$a,$b,$c,$d,$e)=@_;
|
|
|
|
|
|
my $j=$i+1;
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
- lea 0x8f1bbcdc($xi,$e),$f
|
|
|
|
|
|
- mov `4*($j%16)`(%rsp),$xi
|
|
|
|
|
|
- mov $b,$t0
|
|
|
|
|
|
- mov $b,$t1
|
|
|
|
|
|
- xor `4*(($j+2)%16)`(%rsp),$xi
|
|
|
|
|
|
- mov $a,$e
|
|
|
|
|
|
- and $c,$t0
|
|
|
|
|
|
- xor `4*(($j+8)%16)`(%rsp),$xi
|
|
|
|
|
|
- or $c,$t1
|
|
|
|
|
|
- rol \$5,$e
|
|
|
|
|
|
- xor `4*(($j+13)%16)`(%rsp),$xi
|
|
|
|
|
|
- and $d,$t1
|
|
|
|
|
|
- add $e,$f
|
|
|
|
|
|
- rol \$1,$xi
|
|
|
|
|
|
- or $t1,$t0
|
|
|
|
|
|
+ mov `4*($j%16)`(%rsp),$xi[1]
|
|
|
|
|
|
+ mov $c,$t0
|
|
|
|
|
|
+ mov $c,$t1
|
|
|
|
|
|
+ xor `4*(($j+2)%16)`(%rsp),$xi[1]
|
|
|
|
|
|
+ and $d,$t0
|
|
|
|
|
|
+ mov $a,$t2
|
|
|
|
|
|
+ xor `4*(($j+8)%16)`(%rsp),$xi[1]
|
|
|
|
|
|
+ xor $d,$t1
|
|
|
|
|
|
+ lea 0x8f1bbcdc($xi[0],$e),$e
|
|
|
|
|
|
+ rol \$5,$t2
|
|
|
|
|
|
+ xor `4*(($j+13)%16)`(%rsp),$xi[1]
|
|
|
|
|
|
+ add $t0,$e
|
|
|
|
|
|
+ and $b,$t1
|
|
|
|
|
|
+ rol \$1,$xi[1]
|
|
|
|
|
|
+ add $t1,$e
|
|
|
|
|
|
rol \$30,$b
|
|
|
|
|
|
- mov $xi,`4*($j%16)`(%rsp)
|
|
|
|
|
|
- add $t0,$f
|
|
|
|
|
|
+ mov $xi[1],`4*($j%16)`(%rsp)
|
|
|
|
|
|
+ add $t2,$e
|
|
|
|
|
|
___
|
|
|
|
|
|
+unshift(@xi,pop(@xi));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
-$code=".text\n";
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+.text
|
|
|
|
|
|
+.extern OPENSSL_ia32cap_X
|
|
|
|
|
|
+
|
|
|
|
|
|
+.globl sha1_block_data_order
|
|
|
|
|
|
+.type sha1_block_data_order,\@function,3
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+sha1_block_data_order:
|
|
|
|
|
|
+ mov OPENSSL_ia32cap_X+0(%rip),%r9d
|
|
|
|
|
|
+ mov OPENSSL_ia32cap_X+4(%rip),%r8d
|
|
|
|
|
|
+ test \$`1<<9`,%r8d # check SSSE3 bit
|
|
|
|
|
|
+ jz .Lialu
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___ if ($avx);
|
|
|
|
|
|
+ and \$`1<<28`,%r8d # mask AVX bit
|
|
|
|
|
|
+ and \$`1<<30`,%r9d # mask "Intel CPU" bit
|
|
|
|
|
|
+ or %r9d,%r8d
|
|
|
|
|
|
+ cmp \$`1<<28|1<<30`,%r8d
|
|
|
|
|
|
+ je _avx_shortcut
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ jmp _ssse3_shortcut
|
|
|
|
|
|
+
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lialu:
|
|
|
|
|
|
+ push %rbx
|
|
|
|
|
|
+ push %rbp
|
|
|
|
|
|
+ push %r12
|
|
|
|
|
|
+ push %r13
|
|
|
|
|
|
+ mov %rsp,%r11
|
|
|
|
|
|
+ mov %rdi,$ctx # reassigned argument
|
|
|
|
|
|
+ sub \$`8+16*4`,%rsp
|
|
|
|
|
|
+ mov %rsi,$inp # reassigned argument
|
|
|
|
|
|
+ and \$-64,%rsp
|
|
|
|
|
|
+ mov %rdx,$num # reassigned argument
|
|
|
|
|
|
+ mov %r11,`16*4`(%rsp)
|
|
|
|
|
|
+.Lprologue:
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov 0($ctx),$A
|
|
|
|
|
|
+ mov 4($ctx),$B
|
|
|
|
|
|
+ mov 8($ctx),$C
|
|
|
|
|
|
+ mov 12($ctx),$D
|
|
|
|
|
|
+ mov 16($ctx),$E
|
|
|
|
|
|
+ jmp .Lloop
|
|
|
|
|
|
|
|
|
|
|
|
-&PROLOGUE("sha1_block_data_order");
|
|
|
|
|
|
-$code.=".align 4\n.Lloop:\n";
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Lloop:
|
|
|
|
|
|
+___
|
|
|
|
|
|
for($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
|
|
|
|
|
|
for(;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
|
|
|
|
|
|
for(;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
|
|
|
|
|
|
for(;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
- add 0($ctx),$E
|
|
|
|
|
|
- add 4($ctx),$T
|
|
|
|
|
|
- add 8($ctx),$A
|
|
|
|
|
|
- add 12($ctx),$B
|
|
|
|
|
|
- add 16($ctx),$C
|
|
|
|
|
|
- mov $E,0($ctx)
|
|
|
|
|
|
- mov $T,4($ctx)
|
|
|
|
|
|
- mov $A,8($ctx)
|
|
|
|
|
|
- mov $B,12($ctx)
|
|
|
|
|
|
- mov $C,16($ctx)
|
|
|
|
|
|
-
|
|
|
|
|
|
- xchg $E,$A # mov $E,$A
|
|
|
|
|
|
- xchg $T,$B # mov $T,$B
|
|
|
|
|
|
- xchg $E,$C # mov $A,$C
|
|
|
|
|
|
- xchg $T,$D # mov $B,$D
|
|
|
|
|
|
- # mov $C,$E
|
|
|
|
|
|
- lea `16*4`($inp),$inp
|
|
|
|
|
|
+ add 0($ctx),$A
|
|
|
|
|
|
+ add 4($ctx),$B
|
|
|
|
|
|
+ add 8($ctx),$C
|
|
|
|
|
|
+ add 12($ctx),$D
|
|
|
|
|
|
+ add 16($ctx),$E
|
|
|
|
|
|
+ mov $A,0($ctx)
|
|
|
|
|
|
+ mov $B,4($ctx)
|
|
|
|
|
|
+ mov $C,8($ctx)
|
|
|
|
|
|
+ mov $D,12($ctx)
|
|
|
|
|
|
+ mov $E,16($ctx)
|
|
|
|
|
|
+
|
|
|
|
|
|
sub \$1,$num
|
|
|
|
|
|
+ lea `16*4`($inp),$inp
|
|
|
|
|
|
jnz .Lloop
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov `16*4`(%rsp),%rsi
|
|
|
|
|
|
+ mov (%rsi),%r13
|
|
|
|
|
|
+ mov 8(%rsi),%r12
|
|
|
|
|
|
+ mov 16(%rsi),%rbp
|
|
|
|
|
|
+ mov 24(%rsi),%rbx
|
|
|
|
|
|
+ lea 32(%rsi),%rsp
|
|
|
|
|
|
+.Lepilogue:
|
|
|
|
|
|
+ ret
|
|
|
|
|
|
+.size sha1_block_data_order,.-sha1_block_data_order
|
|
|
|
|
|
___
|
|
|
|
|
|
-&EPILOGUE("sha1_block_data_order");
|
|
|
|
|
|
+{{{
|
|
|
|
|
|
+my $Xi=4;
|
|
|
|
|
|
+my @X=map("%xmm$_",(4..7,0..3));
|
|
|
|
|
|
+my @Tx=map("%xmm$_",(8..10));
|
|
|
|
|
|
+my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization
|
|
|
|
|
|
+my @T=("%esi","%edi");
|
|
|
|
|
|
+my $j=0;
|
|
|
|
|
|
+my $K_XX_XX="%r11";
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $_rol=sub { &rol(@_) };
|
|
|
|
|
|
+my $_ror=sub { &ror(@_) };
|
|
|
|
|
|
+
|
|
|
|
|
|
$code.=<<___;
|
|
|
|
|
|
-.asciz "SHA1 block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
|
|
|
|
|
|
+.type sha1_block_data_order_ssse3,\@function,3
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
+sha1_block_data_order_ssse3:
|
|
|
|
|
|
+_ssse3_shortcut:
|
|
|
|
|
|
+ push %rbx
|
|
|
|
|
|
+ push %rbp
|
|
|
|
|
|
+ push %r12
|
|
|
|
|
|
+ lea `-64-($win64?5*16:0)`(%rsp),%rsp
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___ if ($win64);
|
|
|
|
|
|
+ movaps %xmm6,64+0(%rsp)
|
|
|
|
|
|
+ movaps %xmm7,64+16(%rsp)
|
|
|
|
|
|
+ movaps %xmm8,64+32(%rsp)
|
|
|
|
|
|
+ movaps %xmm9,64+48(%rsp)
|
|
|
|
|
|
+ movaps %xmm10,64+64(%rsp)
|
|
|
|
|
|
+.Lprologue_ssse3:
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ mov %rdi,$ctx # reassigned argument
|
|
|
|
|
|
+ mov %rsi,$inp # reassigned argument
|
|
|
|
|
|
+ mov %rdx,$num # reassigned argument
|
|
|
|
|
|
+
|
|
|
|
|
|
+ shl \$6,$num
|
|
|
|
|
|
+ add $inp,$num
|
|
|
|
|
|
+ lea K_XX_XX(%rip),$K_XX_XX
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov 0($ctx),$A # load context
|
|
|
|
|
|
+ mov 4($ctx),$B
|
|
|
|
|
|
+ mov 8($ctx),$C
|
|
|
|
|
|
+ mov 12($ctx),$D
|
|
|
|
|
|
+ mov $B,@T[0] # magic seed
|
|
|
|
|
|
+ mov 16($ctx),$E
|
|
|
|
|
|
+
|
|
|
|
|
|
+ movdqa 64($K_XX_XX),@X[2] # pbswap mask
|
|
|
|
|
|
+ movdqa 0($K_XX_XX),@Tx[1] # K_00_19
|
|
|
|
|
|
+ movdqu 0($inp),@X[-4&7] # load input to %xmm[0-3]
|
|
|
|
|
|
+ movdqu 16($inp),@X[-3&7]
|
|
|
|
|
|
+ movdqu 32($inp),@X[-2&7]
|
|
|
|
|
|
+ movdqu 48($inp),@X[-1&7]
|
|
|
|
|
|
+ pshufb @X[2],@X[-4&7] # byte swap
|
|
|
|
|
|
+ add \$64,$inp
|
|
|
|
|
|
+ pshufb @X[2],@X[-3&7]
|
|
|
|
|
|
+ pshufb @X[2],@X[-2&7]
|
|
|
|
|
|
+ pshufb @X[2],@X[-1&7]
|
|
|
|
|
|
+ paddd @Tx[1],@X[-4&7] # add K_00_19
|
|
|
|
|
|
+ paddd @Tx[1],@X[-3&7]
|
|
|
|
|
|
+ paddd @Tx[1],@X[-2&7]
|
|
|
|
|
|
+ movdqa @X[-4&7],0(%rsp) # X[]+K xfer to IALU
|
|
|
|
|
|
+ psubd @Tx[1],@X[-4&7] # restore X[]
|
|
|
|
|
|
+ movdqa @X[-3&7],16(%rsp)
|
|
|
|
|
|
+ psubd @Tx[1],@X[-3&7]
|
|
|
|
|
|
+ movdqa @X[-2&7],32(%rsp)
|
|
|
|
|
|
+ psubd @Tx[1],@X[-2&7]
|
|
|
|
|
|
+ jmp .Loop_ssse3
|
|
|
|
|
|
+___
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm
|
|
|
|
|
|
+{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
|
|
|
|
|
|
+ my $arg = pop;
|
|
|
|
|
|
+ $arg = "\$$arg" if ($arg*1 eq $arg);
|
|
|
|
|
|
+ $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n";
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 40 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@X[0],@X[-3&7]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (@Tx[0],@X[-1&7]);
|
|
|
|
|
|
+ &palignr(@X[0],@X[-4&7],8); # compose "X[-14]" in "X[0]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &paddd (@Tx[1],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &psrldq (@Tx[0],4); # "X[-3]", 3 dwords
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@Tx[2],@X[0]);
|
|
|
|
|
|
+ &movdqa (@Tx[0],@X[0]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pslldq (@Tx[2],12); # "X[0]"<<96, extract one dword
|
|
|
|
|
|
+ &paddd (@X[0],@X[0]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &psrld (@Tx[0],31);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (@Tx[1],@Tx[2]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &psrld (@Tx[2],30);
|
|
|
|
|
|
+ &por (@X[0],@Tx[0]); # "X[0]"<<<=1
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pslld (@Tx[1],2);
|
|
|
|
|
|
+ &pxor (@X[0],@Tx[2]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)"); # K_XX_XX
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@X[0],@Tx[1]); # "X[0]"^=("X[0]">>96)<<<2
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions [if any]
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi++; push(@X,shift(@X)); # "rotate" X[]
|
|
|
|
|
|
+ push(@Tx,shift(@Tx));
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xupdate_ssse3_32_79()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@Tx[0],@X[-1&7]) if ($Xi==8);
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]"
|
|
|
|
|
|
+ &palignr(@Tx[0],@X[-2&7],8); # compose "X[-6]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/);
|
|
|
|
|
|
+ if ($Xi%5) {
|
|
|
|
|
|
+ &movdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX...
|
|
|
|
|
|
+ } else { # ... or load next one
|
|
|
|
|
|
+ &movdqa (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)");
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+ &paddd (@Tx[1],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@X[0],@Tx[0]); # "X[0]"^="X[-6]"
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@Tx[0],@X[0]);
|
|
|
|
|
|
+ &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pslld (@X[0],2);
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &psrld (@Tx[0],30);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &por (@X[0],@Tx[0]); # "X[0]"<<<=2
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (@Tx[1],@X[0]) if ($Xi<19);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi++; push(@X,shift(@X)); # "rotate" X[]
|
|
|
|
|
|
+ push(@Tx,shift(@Tx));
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xuplast_ssse3_80()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &paddd (@Tx[1],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &cmp ($inp,$num);
|
|
|
|
|
|
+ &je (".Ldone_ssse3");
|
|
|
|
|
|
+
|
|
|
|
|
|
+ unshift(@Tx,pop(@Tx));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@X[2],"64($K_XX_XX)"); # pbswap mask
|
|
|
|
|
|
+ &movdqa (@Tx[1],"0($K_XX_XX)"); # K_00_19
|
|
|
|
|
|
+ &movdqu (@X[-4&7],"0($inp)"); # load input
|
|
|
|
|
|
+ &movdqu (@X[-3&7],"16($inp)");
|
|
|
|
|
|
+ &movdqu (@X[-2&7],"32($inp)");
|
|
|
|
|
|
+ &movdqu (@X[-1&7],"48($inp)");
|
|
|
|
|
|
+ &pshufb (@X[-4&7],@X[2]); # byte swap
|
|
|
|
|
|
+ &add ($inp,64);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi=0;
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xloop_ssse3()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &pshufb (@X[($Xi-3)&7],@X[2]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &paddd (@X[($Xi-4)&7],@Tx[1]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (eval(16*$Xi)."(%rsp)",@X[($Xi-4)&7]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &psubd (@X[($Xi-4)&7],@Tx[1]);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; }
|
|
|
|
|
|
+ $Xi++;
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xtail_ssse3()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub body_00_19 () {
|
|
|
|
|
|
+ (
|
|
|
|
|
|
+ '($a,$b,$c,$d,$e)=@V;'.
|
|
|
|
|
|
+ '&add ($e,eval(4*($j&15))."(%rsp)");', # X[]+K xfer
|
|
|
|
|
|
+ '&xor ($c,$d);',
|
|
|
|
|
|
+ '&mov (@T[1],$a);', # $b in next round
|
|
|
|
|
|
+ '&$_rol ($a,5);',
|
|
|
|
|
|
+ '&and (@T[0],$c);', # ($b&($c^$d))
|
|
|
|
|
|
+ '&xor ($c,$d);', # restore $c
|
|
|
|
|
|
+ '&xor (@T[0],$d);',
|
|
|
|
|
|
+ '&add ($e,$a);',
|
|
|
|
|
|
+ '&$_ror ($b,$j?7:2);', # $b>>>2
|
|
|
|
|
|
+ '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
|
|
|
|
|
|
+ );
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub body_20_39 () {
|
|
|
|
|
|
+ (
|
|
|
|
|
|
+ '($a,$b,$c,$d,$e)=@V;'.
|
|
|
|
|
|
+ '&add ($e,eval(4*($j++&15))."(%rsp)");', # X[]+K xfer
|
|
|
|
|
|
+ '&xor (@T[0],$d);', # ($b^$d)
|
|
|
|
|
|
+ '&mov (@T[1],$a);', # $b in next round
|
|
|
|
|
|
+ '&$_rol ($a,5);',
|
|
|
|
|
|
+ '&xor (@T[0],$c);', # ($b^$d^$c)
|
|
|
|
|
|
+ '&add ($e,$a);',
|
|
|
|
|
|
+ '&$_ror ($b,7);', # $b>>>2
|
|
|
|
|
|
+ '&add ($e,@T[0]);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));'
|
|
|
|
|
|
+ );
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub body_40_59 () {
|
|
|
|
|
|
+ (
|
|
|
|
|
|
+ '($a,$b,$c,$d,$e)=@V;'.
|
|
|
|
|
|
+ '&mov (@T[1],$c);',
|
|
|
|
|
|
+ '&xor ($c,$d);',
|
|
|
|
|
|
+ '&add ($e,eval(4*($j++&15))."(%rsp)");', # X[]+K xfer
|
|
|
|
|
|
+ '&and (@T[1],$d);',
|
|
|
|
|
|
+ '&and (@T[0],$c);', # ($b&($c^$d))
|
|
|
|
|
|
+ '&$_ror ($b,7);', # $b>>>2
|
|
|
|
|
|
+ '&add ($e,@T[1]);',
|
|
|
|
|
|
+ '&mov (@T[1],$a);', # $b in next round
|
|
|
|
|
|
+ '&$_rol ($a,5);',
|
|
|
|
|
|
+ '&add ($e,@T[0]);',
|
|
|
|
|
|
+ '&xor ($c,$d);', # restore $c
|
|
|
|
|
|
+ '&add ($e,$a);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));'
|
|
|
|
|
|
+ );
|
|
|
|
|
|
+}
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Loop_ssse3:
|
|
|
|
|
|
+___
|
|
|
|
|
|
+ &Xupdate_ssse3_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xuplast_ssse3_80(\&body_20_39); # can jump to "done"
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $saved_j=$j; @saved_V=@V;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &Xloop_ssse3(\&body_20_39);
|
|
|
|
|
|
+ &Xloop_ssse3(\&body_20_39);
|
|
|
|
|
|
+ &Xloop_ssse3(\&body_20_39);
|
|
|
|
|
|
+
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ add 0($ctx),$A # update context
|
|
|
|
|
|
+ add 4($ctx),@T[0]
|
|
|
|
|
|
+ add 8($ctx),$C
|
|
|
|
|
|
+ add 12($ctx),$D
|
|
|
|
|
|
+ mov $A,0($ctx)
|
|
|
|
|
|
+ add 16($ctx),$E
|
|
|
|
|
|
+ mov @T[0],4($ctx)
|
|
|
|
|
|
+ mov @T[0],$B # magic seed
|
|
|
|
|
|
+ mov $C,8($ctx)
|
|
|
|
|
|
+ mov $D,12($ctx)
|
|
|
|
|
|
+ mov $E,16($ctx)
|
|
|
|
|
|
+ jmp .Loop_ssse3
|
|
|
|
|
|
+
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Ldone_ssse3:
|
|
|
|
|
|
+___
|
|
|
|
|
|
+ $j=$saved_j; @V=@saved_V;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &Xtail_ssse3(\&body_20_39);
|
|
|
|
|
|
+ &Xtail_ssse3(\&body_20_39);
|
|
|
|
|
|
+ &Xtail_ssse3(\&body_20_39);
|
|
|
|
|
|
+
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ add 0($ctx),$A # update context
|
|
|
|
|
|
+ add 4($ctx),@T[0]
|
|
|
|
|
|
+ add 8($ctx),$C
|
|
|
|
|
|
+ mov $A,0($ctx)
|
|
|
|
|
|
+ add 12($ctx),$D
|
|
|
|
|
|
+ mov @T[0],4($ctx)
|
|
|
|
|
|
+ add 16($ctx),$E
|
|
|
|
|
|
+ mov $C,8($ctx)
|
|
|
|
|
|
+ mov $D,12($ctx)
|
|
|
|
|
|
+ mov $E,16($ctx)
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___ if ($win64);
|
|
|
|
|
|
+ movaps 64+0(%rsp),%xmm6
|
|
|
|
|
|
+ movaps 64+16(%rsp),%xmm7
|
|
|
|
|
|
+ movaps 64+32(%rsp),%xmm8
|
|
|
|
|
|
+ movaps 64+48(%rsp),%xmm9
|
|
|
|
|
|
+ movaps 64+64(%rsp),%xmm10
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ lea `64+($win64?6*16:0)`(%rsp),%rsi
|
|
|
|
|
|
+ mov 0(%rsi),%r12
|
|
|
|
|
|
+ mov 8(%rsi),%rbp
|
|
|
|
|
|
+ mov 16(%rsi),%rbx
|
|
|
|
|
|
+ lea 24(%rsi),%rsp
|
|
|
|
|
|
+.Lepilogue_ssse3:
|
|
|
|
|
|
+ ret
|
|
|
|
|
|
+.size sha1_block_data_order_ssse3,.-sha1_block_data_order_ssse3
|
|
|
|
|
|
+___
|
|
|
|
|
|
+
|
|
|
|
|
|
+if ($avx) {
|
|
|
|
|
|
+my $Xi=4;
|
|
|
|
|
|
+my @X=map("%xmm$_",(4..7,0..3));
|
|
|
|
|
|
+my @Tx=map("%xmm$_",(8..10));
|
|
|
|
|
|
+my @V=($A,$B,$C,$D,$E)=("%eax","%ebx","%ecx","%edx","%ebp"); # size optimization
|
|
|
|
|
|
+my @T=("%esi","%edi");
|
|
|
|
|
|
+my $j=0;
|
|
|
|
|
|
+my $K_XX_XX="%r11";
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $_rol=sub { &shld(@_[0],@_) };
|
|
|
|
|
|
+my $_ror=sub { &shrd(@_[0],@_) };
|
|
|
|
|
|
+
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+.type sha1_block_data_order_avx,\@function,3
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+sha1_block_data_order_avx:
|
|
|
|
|
|
+_avx_shortcut:
|
|
|
|
|
|
+ push %rbx
|
|
|
|
|
|
+ push %rbp
|
|
|
|
|
|
+ push %r12
|
|
|
|
|
|
+ lea `-64-($win64?5*16:0)`(%rsp),%rsp
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___ if ($win64);
|
|
|
|
|
|
+ movaps %xmm6,64+0(%rsp)
|
|
|
|
|
|
+ movaps %xmm7,64+16(%rsp)
|
|
|
|
|
|
+ movaps %xmm8,64+32(%rsp)
|
|
|
|
|
|
+ movaps %xmm9,64+48(%rsp)
|
|
|
|
|
|
+ movaps %xmm10,64+64(%rsp)
|
|
|
|
|
|
+.Lprologue_avx:
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ mov %rdi,$ctx # reassigned argument
|
|
|
|
|
|
+ mov %rsi,$inp # reassigned argument
|
|
|
|
|
|
+ mov %rdx,$num # reassigned argument
|
|
|
|
|
|
+ vzeroall
|
|
|
|
|
|
+
|
|
|
|
|
|
+ shl \$6,$num
|
|
|
|
|
|
+ add $inp,$num
|
|
|
|
|
|
+ lea K_XX_XX(%rip),$K_XX_XX
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov 0($ctx),$A # load context
|
|
|
|
|
|
+ mov 4($ctx),$B
|
|
|
|
|
|
+ mov 8($ctx),$C
|
|
|
|
|
|
+ mov 12($ctx),$D
|
|
|
|
|
|
+ mov $B,@T[0] # magic seed
|
|
|
|
|
|
+ mov 16($ctx),$E
|
|
|
|
|
|
+
|
|
|
|
|
|
+ vmovdqa 64($K_XX_XX),@X[2] # pbswap mask
|
|
|
|
|
|
+ vmovdqa 0($K_XX_XX),@Tx[1] # K_00_19
|
|
|
|
|
|
+ vmovdqu 0($inp),@X[-4&7] # load input to %xmm[0-3]
|
|
|
|
|
|
+ vmovdqu 16($inp),@X[-3&7]
|
|
|
|
|
|
+ vmovdqu 32($inp),@X[-2&7]
|
|
|
|
|
|
+ vmovdqu 48($inp),@X[-1&7]
|
|
|
|
|
|
+ vpshufb @X[2],@X[-4&7],@X[-4&7] # byte swap
|
|
|
|
|
|
+ add \$64,$inp
|
|
|
|
|
|
+ vpshufb @X[2],@X[-3&7],@X[-3&7]
|
|
|
|
|
|
+ vpshufb @X[2],@X[-2&7],@X[-2&7]
|
|
|
|
|
|
+ vpshufb @X[2],@X[-1&7],@X[-1&7]
|
|
|
|
|
|
+ vpaddd @Tx[1],@X[-4&7],@X[0] # add K_00_19
|
|
|
|
|
|
+ vpaddd @Tx[1],@X[-3&7],@X[1]
|
|
|
|
|
|
+ vpaddd @Tx[1],@X[-2&7],@X[2]
|
|
|
|
|
|
+ vmovdqa @X[0],0(%rsp) # X[]+K xfer to IALU
|
|
|
|
|
|
+ vmovdqa @X[1],16(%rsp)
|
|
|
|
|
|
+ vmovdqa @X[2],32(%rsp)
|
|
|
|
|
|
+ jmp .Loop_avx
|
|
|
|
|
|
+___
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 40 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpaddd (@Tx[1],@Tx[1],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpsrldq(@Tx[0],@X[-1&7],4); # "X[-3]", 3 dwords
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@Tx[0],@Tx[0],@X[-2&7]); # "X[-3]"^"X[-8]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-3]"^"X[-8]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpsrld (@Tx[0],@X[0],31);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpslldq(@Tx[2],@X[0],12); # "X[0]"<<96, extract one dword
|
|
|
|
|
|
+ &vpaddd (@X[0],@X[0],@X[0]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpsrld (@Tx[1],@Tx[2],30);
|
|
|
|
|
|
+ &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=1
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpslld (@Tx[2],@Tx[2],2);
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@Tx[1]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@Tx[2]); # "X[0]"^=("X[0]">>96)<<<2
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vmovdqa (@Tx[2],eval(16*(($Xi)/5))."($K_XX_XX)"); # K_XX_XX
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions [if any]
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi++; push(@X,shift(@X)); # "rotate" X[]
|
|
|
|
|
|
+ push(@Tx,shift(@Tx));
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xupdate_avx_32_79()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpalignr(@Tx[0],@X[-1&7],@X[-2&7],8); # compose "X[-6]"
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]"
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)) if (@insns[0] !~ /&ro[rl]/);
|
|
|
|
|
|
+ if ($Xi%5) {
|
|
|
|
|
|
+ &vmovdqa (@Tx[2],@Tx[1]);# "perpetuate" K_XX_XX...
|
|
|
|
|
|
+ } else { # ... or load next one
|
|
|
|
|
|
+ &vmovdqa (@Tx[2],eval(16*($Xi/5))."($K_XX_XX)");
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+ &vpaddd (@Tx[1],@Tx[1],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@Tx[0]); # "X[0]"^="X[-6]"
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpsrld (@Tx[0],@X[0],30);
|
|
|
|
|
|
+ &vmovdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpslld (@X[0],@X[0],2);
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpor (@X[0],@X[0],@Tx[0]); # "X[0]"<<<=2
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vmovdqa (@Tx[1],@X[0]) if ($Xi<19);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi++; push(@X,shift(@X)); # "rotate" X[]
|
|
|
|
|
|
+ push(@Tx,shift(@Tx));
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xuplast_avx_80()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpaddd (@Tx[1],@Tx[1],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (eval(16*(($Xi-1)&3))."(%rsp)",@Tx[1]); # X[]+K xfer IALU
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &cmp ($inp,$num);
|
|
|
|
|
|
+ &je (".Ldone_avx");
|
|
|
|
|
|
+
|
|
|
|
|
|
+ unshift(@Tx,pop(@Tx));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vmovdqa(@X[2],"64($K_XX_XX)"); # pbswap mask
|
|
|
|
|
|
+ &vmovdqa(@Tx[1],"0($K_XX_XX)"); # K_00_19
|
|
|
|
|
|
+ &vmovdqu(@X[-4&7],"0($inp)"); # load input
|
|
|
|
|
|
+ &vmovdqu(@X[-3&7],"16($inp)");
|
|
|
|
|
|
+ &vmovdqu(@X[-2&7],"32($inp)");
|
|
|
|
|
|
+ &vmovdqu(@X[-1&7],"48($inp)");
|
|
|
|
|
|
+ &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap
|
|
|
|
|
|
+ &add ($inp,64);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi=0;
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xloop_avx()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpshufb(@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],@Tx[1]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vmovdqa(eval(16*$Xi)."(%rsp)",@X[$Xi&7]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; }
|
|
|
|
|
|
+ $Xi++;
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xtail_avx()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Loop_avx:
|
|
|
|
|
|
+___
|
|
|
|
|
|
+ &Xupdate_avx_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xuplast_avx_80(\&body_20_39); # can jump to "done"
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $saved_j=$j; @saved_V=@V;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &Xloop_avx(\&body_20_39);
|
|
|
|
|
|
+ &Xloop_avx(\&body_20_39);
|
|
|
|
|
|
+ &Xloop_avx(\&body_20_39);
|
|
|
|
|
|
+
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ add 0($ctx),$A # update context
|
|
|
|
|
|
+ add 4($ctx),@T[0]
|
|
|
|
|
|
+ add 8($ctx),$C
|
|
|
|
|
|
+ add 12($ctx),$D
|
|
|
|
|
|
+ mov $A,0($ctx)
|
|
|
|
|
|
+ add 16($ctx),$E
|
|
|
|
|
|
+ mov @T[0],4($ctx)
|
|
|
|
|
|
+ mov @T[0],$B # magic seed
|
|
|
|
|
|
+ mov $C,8($ctx)
|
|
|
|
|
|
+ mov $D,12($ctx)
|
|
|
|
|
|
+ mov $E,16($ctx)
|
|
|
|
|
|
+ jmp .Loop_avx
|
|
|
|
|
|
+
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+.Ldone_avx:
|
|
|
|
|
|
+___
|
|
|
|
|
|
+ $j=$saved_j; @V=@saved_V;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &Xtail_avx(\&body_20_39);
|
|
|
|
|
|
+ &Xtail_avx(\&body_20_39);
|
|
|
|
|
|
+ &Xtail_avx(\&body_20_39);
|
|
|
|
|
|
+
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ vzeroall
|
|
|
|
|
|
+
|
|
|
|
|
|
+ add 0($ctx),$A # update context
|
|
|
|
|
|
+ add 4($ctx),@T[0]
|
|
|
|
|
|
+ add 8($ctx),$C
|
|
|
|
|
|
+ mov $A,0($ctx)
|
|
|
|
|
|
+ add 12($ctx),$D
|
|
|
|
|
|
+ mov @T[0],4($ctx)
|
|
|
|
|
|
+ add 16($ctx),$E
|
|
|
|
|
|
+ mov $C,8($ctx)
|
|
|
|
|
|
+ mov $D,12($ctx)
|
|
|
|
|
|
+ mov $E,16($ctx)
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___ if ($win64);
|
|
|
|
|
|
+ movaps 64+0(%rsp),%xmm6
|
|
|
|
|
|
+ movaps 64+16(%rsp),%xmm7
|
|
|
|
|
|
+ movaps 64+32(%rsp),%xmm8
|
|
|
|
|
|
+ movaps 64+48(%rsp),%xmm9
|
|
|
|
|
|
+ movaps 64+64(%rsp),%xmm10
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+ lea `64+($win64?6*16:0)`(%rsp),%rsi
|
|
|
|
|
|
+ mov 0(%rsi),%r12
|
|
|
|
|
|
+ mov 8(%rsi),%rbp
|
|
|
|
|
|
+ mov 16(%rsi),%rbx
|
|
|
|
|
|
+ lea 24(%rsi),%rsp
|
|
|
|
|
|
+.Lepilogue_avx:
|
|
|
|
|
|
+ ret
|
|
|
|
|
|
+.size sha1_block_data_order_avx,.-sha1_block_data_order_avx
|
|
|
|
|
|
+___
|
|
|
|
|
|
+}
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+.align 64
|
|
|
|
|
|
+K_XX_XX:
|
|
|
|
|
|
+.long 0x5a827999,0x5a827999,0x5a827999,0x5a827999 # K_00_19
|
|
|
|
|
|
+.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 # K_20_39
|
|
|
|
|
|
+.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc # K_40_59
|
|
|
|
|
|
+.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 # K_60_79
|
|
|
|
|
|
+.long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f # pbswap mask
|
|
|
|
|
|
+___
|
|
|
|
|
|
+}}}
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
+.asciz "SHA1 block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
|
|
|
|
|
|
+.align 64
|
|
|
|
|
|
___
|
|
|
|
|
|
|
|
|
|
|
|
# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
|
|
|
|
|
|
@@ -272,25 +1109,73 @@ se_handler:
|
|
|
|
|
|
|
|
|
|
|
|
lea .Lprologue(%rip),%r10
|
|
|
|
|
|
cmp %r10,%rbx # context->Rip<.Lprologue
|
|
|
|
|
|
- jb .Lin_prologue
|
|
|
|
|
|
+ jb .Lcommon_seh_tail
|
|
|
|
|
|
|
|
|
|
|
|
mov 152($context),%rax # pull context->Rsp
|
|
|
|
|
|
|
|
|
|
|
|
lea .Lepilogue(%rip),%r10
|
|
|
|
|
|
cmp %r10,%rbx # context->Rip>=.Lepilogue
|
|
|
|
|
|
- jae .Lin_prologue
|
|
|
|
|
|
+ jae .Lcommon_seh_tail
|
|
|
|
|
|
|
|
|
|
|
|
mov `16*4`(%rax),%rax # pull saved stack pointer
|
|
|
|
|
|
- lea 24(%rax),%rax
|
|
|
|
|
|
+ lea 32(%rax),%rax
|
|
|
|
|
|
|
|
|
|
|
|
mov -8(%rax),%rbx
|
|
|
|
|
|
mov -16(%rax),%rbp
|
|
|
|
|
|
mov -24(%rax),%r12
|
|
|
|
|
|
+ mov -32(%rax),%r13
|
|
|
|
|
|
mov %rbx,144($context) # restore context->Rbx
|
|
|
|
|
|
mov %rbp,160($context) # restore context->Rbp
|
|
|
|
|
|
mov %r12,216($context) # restore context->R12
|
|
|
|
|
|
+ mov %r13,224($context) # restore context->R13
|
|
|
|
|
|
+
|
|
|
|
|
|
+ jmp .Lcommon_seh_tail
|
|
|
|
|
|
+.size se_handler,.-se_handler
|
|
|
|
|
|
+
|
|
|
|
|
|
+.type ssse3_handler,\@abi-omnipotent
|
|
|
|
|
|
+.align 16
|
|
|
|
|
|
+ssse3_handler:
|
|
|
|
|
|
+ push %rsi
|
|
|
|
|
|
+ push %rdi
|
|
|
|
|
|
+ push %rbx
|
|
|
|
|
|
+ push %rbp
|
|
|
|
|
|
+ push %r12
|
|
|
|
|
|
+ push %r13
|
|
|
|
|
|
+ push %r14
|
|
|
|
|
|
+ push %r15
|
|
|
|
|
|
+ pushfq
|
|
|
|
|
|
+ sub \$64,%rsp
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov 120($context),%rax # pull context->Rax
|
|
|
|
|
|
+ mov 248($context),%rbx # pull context->Rip
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov 8($disp),%rsi # disp->ImageBase
|
|
|
|
|
|
+ mov 56($disp),%r11 # disp->HandlerData
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov 0(%r11),%r10d # HandlerData[0]
|
|
|
|
|
|
+ lea (%rsi,%r10),%r10 # prologue label
|
|
|
|
|
|
+ cmp %r10,%rbx # context->Rip<prologue label
|
|
|
|
|
|
+ jb .Lcommon_seh_tail
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov 152($context),%rax # pull context->Rsp
|
|
|
|
|
|
|
|
|
|
|
|
-.Lin_prologue:
|
|
|
|
|
|
+ mov 4(%r11),%r10d # HandlerData[1]
|
|
|
|
|
|
+ lea (%rsi,%r10),%r10 # epilogue label
|
|
|
|
|
|
+ cmp %r10,%rbx # context->Rip>=epilogue label
|
|
|
|
|
|
+ jae .Lcommon_seh_tail
|
|
|
|
|
|
+
|
|
|
|
|
|
+ lea 64(%rax),%rsi
|
|
|
|
|
|
+ lea 512($context),%rdi # &context.Xmm6
|
|
|
|
|
|
+ mov \$10,%ecx
|
|
|
|
|
|
+ .long 0xa548f3fc # cld; rep movsq
|
|
|
|
|
|
+ lea 24+5*16(%rax),%rax # adjust stack pointer
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov -8(%rax),%rbx
|
|
|
|
|
|
+ mov -16(%rax),%rbp
|
|
|
|
|
|
+ mov %rbx,144($context) # restore context->Rbx
|
|
|
|
|
|
+ mov %rbp,160($context) # restore context->Rbp
|
|
|
|
|
|
+
|
|
|
|
|
|
+.Lcommon_seh_tail:
|
|
|
|
|
|
mov 8(%rax),%rdi
|
|
|
|
|
|
mov 16(%rax),%rsi
|
|
|
|
|
|
mov %rax,152($context) # restore context->Rsp
|
|
|
|
|
|
@@ -328,19 +1213,38 @@ se_handler:
|
|
|
|
|
|
pop %rdi
|
|
|
|
|
|
pop %rsi
|
|
|
|
|
|
ret
|
|
|
|
|
|
-.size se_handler,.-se_handler
|
|
|
|
|
|
+.size ssse3_handler,.-ssse3_handler
|
|
|
|
|
|
|
|
|
|
|
|
.section .pdata
|
|
|
|
|
|
.align 4
|
|
|
|
|
|
.rva .LSEH_begin_sha1_block_data_order
|
|
|
|
|
|
.rva .LSEH_end_sha1_block_data_order
|
|
|
|
|
|
.rva .LSEH_info_sha1_block_data_order
|
|
|
|
|
|
-
|
|
|
|
|
|
+ .rva .LSEH_begin_sha1_block_data_order_ssse3
|
|
|
|
|
|
+ .rva .LSEH_end_sha1_block_data_order_ssse3
|
|
|
|
|
|
+ .rva .LSEH_info_sha1_block_data_order_ssse3
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___ if ($avx);
|
|
|
|
|
|
+ .rva .LSEH_begin_sha1_block_data_order_avx
|
|
|
|
|
|
+ .rva .LSEH_end_sha1_block_data_order_avx
|
|
|
|
|
|
+ .rva .LSEH_info_sha1_block_data_order_avx
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___;
|
|
|
|
|
|
.section .xdata
|
|
|
|
|
|
.align 8
|
|
|
|
|
|
.LSEH_info_sha1_block_data_order:
|
|
|
|
|
|
.byte 9,0,0,0
|
|
|
|
|
|
.rva se_handler
|
|
|
|
|
|
+.LSEH_info_sha1_block_data_order_ssse3:
|
|
|
|
|
|
+ .byte 9,0,0,0
|
|
|
|
|
|
+ .rva ssse3_handler
|
|
|
|
|
|
+ .rva .Lprologue_ssse3,.Lepilogue_ssse3 # HandlerData[]
|
|
|
|
|
|
+___
|
|
|
|
|
|
+$code.=<<___ if ($avx);
|
|
|
|
|
|
+.LSEH_info_sha1_block_data_order_avx:
|
|
|
|
|
|
+ .byte 9,0,0,0
|
|
|
|
|
|
+ .rva ssse3_handler
|
|
|
|
|
|
+ .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[]
|
|
|
|
|
|
___
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/sha/asm/sha1-586.pl.intelopts openssl-1.0.0d/crypto/sha/asm/sha1-586.pl
|
|
|
|
|
|
--- openssl-1.0.0d/crypto/sha/asm/sha1-586.pl.intelopts 2008-07-17 11:50:56.000000000 +0200
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+++ openssl-1.0.0d/crypto/sha/asm/sha1-586.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
|
|
|
|
|
|
# ====================================================================
|
|
|
|
|
|
# [Re]written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
|
|
|
|
|
|
@@ -12,6 +12,8 @@
|
|
|
|
|
|
# commentary below], and in 2006 the rest was rewritten in order to
|
|
|
|
|
|
# gain freedom to liberate licensing terms.
|
|
|
|
|
|
|
|
|
|
|
|
+# January, September 2004.
|
|
|
|
|
|
+#
|
|
|
|
|
|
# It was noted that Intel IA-32 C compiler generates code which
|
|
|
|
|
|
# performs ~30% *faster* on P4 CPU than original *hand-coded*
|
|
|
|
|
|
# SHA1 assembler implementation. To address this problem (and
|
|
|
|
|
|
@@ -31,12 +33,92 @@
|
|
|
|
|
|
# ----------------------------------------------------------------
|
|
|
|
|
|
# <appro@fy.chalmers.se>
|
|
|
|
|
|
|
|
|
|
|
|
+# August 2009.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# George Spelvin has tipped that F_40_59(b,c,d) can be rewritten as
|
|
|
|
|
|
+# '(c&d) + (b&(c^d))', which allows to accumulate partial results
|
|
|
|
|
|
+# and lighten "pressure" on scratch registers. This resulted in
|
|
|
|
|
|
+# >12% performance improvement on contemporary AMD cores (with no
|
|
|
|
|
|
+# degradation on other CPUs:-). Also, the code was revised to maximize
|
|
|
|
|
|
+# "distance" between instructions producing input to 'lea' instruction
|
|
|
|
|
|
+# and the 'lea' instruction itself, which is essential for Intel Atom
|
|
|
|
|
|
+# core and resulted in ~15% improvement.
|
|
|
|
|
|
+
|
|
|
|
|
|
+# October 2010.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Add SSSE3, Supplemental[!] SSE3, implementation. The idea behind it
|
|
|
|
|
|
+# is to offload message schedule denoted by Wt in NIST specification,
|
|
|
|
|
|
+# or Xupdate in OpenSSL source, to SIMD unit. The idea is not novel,
|
|
|
|
|
|
+# and in SSE2 context was first explored by Dean Gaudet in 2004, see
|
|
|
|
|
|
+# http://arctic.org/~dean/crypto/sha1.html. Since then several things
|
|
|
|
|
|
+# have changed that made it interesting again:
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# a) XMM units became faster and wider;
|
|
|
|
|
|
+# b) instruction set became more versatile;
|
|
|
|
|
|
+# c) an important observation was made by Max Locktykhin, which made
|
|
|
|
|
|
+# it possible to reduce amount of instructions required to perform
|
|
|
|
|
|
+# the operation in question, for further details see
|
|
|
|
|
|
+# http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/.
|
|
|
|
|
|
+
|
|
|
|
|
|
+# April 2011.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Add AVX code path, probably most controversial... The thing is that
|
|
|
|
|
|
+# switch to AVX alone improves performance by as little as 4% in
|
|
|
|
|
|
+# comparison to SSSE3 code path. But below result doesn't look like
|
|
|
|
|
|
+# 4% improvement... Trouble is that Sandy Bridge decodes 'ro[rl]' as
|
|
|
|
|
|
+# pair of µ-ops, and it's the additional µ-ops, two per round, that
|
|
|
|
|
|
+# make it run slower than Core2 and Westmere. But 'sh[rl]d' is decoded
|
|
|
|
|
|
+# as single µ-op by Sandy Bridge and it's replacing 'ro[rl]' with
|
|
|
|
|
|
+# equivalent 'sh[rl]d' that is responsible for the impressive 5.1
|
|
|
|
|
|
+# cycles per processed byte. But 'sh[rl]d' is not something that used
|
|
|
|
|
|
+# to be fast, nor does it appear to be fast in upcoming Bulldozer
|
|
|
|
|
|
+# [according to its optimization manual]. Which is why AVX code path
|
|
|
|
|
|
+# is guarded by *both* AVX and synthetic bit denoting Intel CPUs.
|
|
|
|
|
|
+# One can argue that it's unfair to AMD, but without 'sh[rl]d' it
|
|
|
|
|
|
+# makes no sense to keep the AVX code path. If somebody feels that
|
|
|
|
|
|
+# strongly, it's probably more appropriate to discuss possibility of
|
|
|
|
|
|
+# using vector rotate XOP on AMD...
|
|
|
|
|
|
+
|
|
|
|
|
|
+######################################################################
|
|
|
|
|
|
+# Current performance is summarized in following table. Numbers are
|
|
|
|
|
|
+# CPU clock cycles spent to process single byte (less is better).
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# x86 SSSE3 AVX
|
|
|
|
|
|
+# Pentium 15.7 -
|
|
|
|
|
|
+# PIII 11.5 -
|
|
|
|
|
|
+# P4 10.6 -
|
|
|
|
|
|
+# AMD K8 7.1 -
|
|
|
|
|
|
+# Core2 7.3 6.1/+20% -
|
|
|
|
|
|
+# Atom 12.5 9.5(*)/+32% -
|
|
|
|
|
|
+# Westmere 7.3 5.6/+30% -
|
|
|
|
|
|
+# Sandy Bridge 8.8 6.2/+40% 5.1(**)/+70%
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# (*) Loop is 1056 instructions long and expected result is ~8.25.
|
|
|
|
|
|
+# It remains mystery [to me] why ILP is limited to 1.7.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# (**) As per above comment, the result is for AVX *plus* sh[rl]d.
|
|
|
|
|
|
+
|
|
|
|
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|
|
|
|
|
push(@INC,"${dir}","${dir}../../perlasm");
|
|
|
|
|
|
require "x86asm.pl";
|
|
|
|
|
|
|
|
|
|
|
|
&asm_init($ARGV[0],"sha1-586.pl",$ARGV[$#ARGV] eq "386");
|
|
|
|
|
|
|
|
|
|
|
|
+$xmm=1; $ymm=0;
|
|
|
|
|
|
+for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
|
|
|
|
|
|
+
|
|
|
|
|
|
+$ymm=1 if ($xmm &&
|
|
|
|
|
|
+ `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
|
|
|
|
|
|
+ =~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
|
|
|
|
|
|
+ $1>=2.19); # first version supporting AVX
|
|
|
|
|
|
+
|
|
|
|
|
|
+$ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
|
|
|
|
|
|
+ `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
|
|
|
|
|
|
+ $1>=2.03); # first version supporting AVX
|
|
|
|
|
|
+
|
|
|
|
|
|
+&external_label("OPENSSL_ia32cap_X") if ($xmm);
|
|
|
|
|
|
+
|
|
|
|
|
|
+
|
|
|
|
|
|
$A="eax";
|
|
|
|
|
|
$B="ebx";
|
|
|
|
|
|
$C="ecx";
|
|
|
|
|
|
@@ -47,6 +129,10 @@ $tmp1="ebp";
|
|
|
|
|
|
|
|
|
|
|
|
@V=($A,$B,$C,$D,$E,$T);
|
|
|
|
|
|
|
|
|
|
|
|
+$alt=0; # 1 denotes alternative IALU implementation, which performs
|
|
|
|
|
|
+ # 8% *worse* on P4, same on Westmere and Atom, 2% better on
|
|
|
|
|
|
+ # Sandy Bridge...
|
|
|
|
|
|
+
|
|
|
|
|
|
sub BODY_00_15
|
|
|
|
|
|
{
|
|
|
|
|
|
local($n,$a,$b,$c,$d,$e,$f)=@_;
|
|
|
|
|
|
@@ -59,16 +145,18 @@ sub BODY_00_15
|
|
|
|
|
|
&rotl($tmp1,5); # tmp1=ROTATE(a,5)
|
|
|
|
|
|
&xor($f,$d);
|
|
|
|
|
|
&add($tmp1,$e); # tmp1+=e;
|
|
|
|
|
|
- &and($f,$b);
|
|
|
|
|
|
- &mov($e,&swtmp($n%16)); # e becomes volatile and is loaded
|
|
|
|
|
|
+ &mov($e,&swtmp($n%16)); # e becomes volatile and is loaded
|
|
|
|
|
|
# with xi, also note that e becomes
|
|
|
|
|
|
# f in next round...
|
|
|
|
|
|
- &xor($f,$d); # f holds F_00_19(b,c,d)
|
|
|
|
|
|
+ &and($f,$b);
|
|
|
|
|
|
&rotr($b,2); # b=ROTATE(b,30)
|
|
|
|
|
|
- &lea($tmp1,&DWP(0x5a827999,$tmp1,$e)); # tmp1+=K_00_19+xi
|
|
|
|
|
|
+ &xor($f,$d); # f holds F_00_19(b,c,d)
|
|
|
|
|
|
+ &lea($tmp1,&DWP(0x5a827999,$tmp1,$e)); # tmp1+=K_00_19+xi
|
|
|
|
|
|
|
|
|
|
|
|
- if ($n==15) { &add($f,$tmp1); } # f+=tmp1
|
|
|
|
|
|
+ if ($n==15) { &mov($e,&swtmp(($n+1)%16));# pre-fetch f for next round
|
|
|
|
|
|
+ &add($f,$tmp1); } # f+=tmp1
|
|
|
|
|
|
else { &add($tmp1,$f); } # f becomes a in next round
|
|
|
|
|
|
+ &mov($tmp1,$a) if ($alt && $n==15);
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
sub BODY_16_19
|
|
|
|
|
|
@@ -77,22 +165,41 @@ sub BODY_16_19
|
|
|
|
|
|
|
|
|
|
|
|
&comment("16_19 $n");
|
|
|
|
|
|
|
|
|
|
|
|
- &mov($f,&swtmp($n%16)); # f to hold Xupdate(xi,xa,xb,xc,xd)
|
|
|
|
|
|
- &mov($tmp1,$c); # tmp1 to hold F_00_19(b,c,d)
|
|
|
|
|
|
- &xor($f,&swtmp(($n+2)%16));
|
|
|
|
|
|
- &xor($tmp1,$d);
|
|
|
|
|
|
- &xor($f,&swtmp(($n+8)%16));
|
|
|
|
|
|
- &and($tmp1,$b); # tmp1 holds F_00_19(b,c,d)
|
|
|
|
|
|
- &rotr($b,2); # b=ROTATE(b,30)
|
|
|
|
|
|
+if ($alt) {
|
|
|
|
|
|
+ &xor($c,$d);
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd)
|
|
|
|
|
|
+ &and($tmp1,$c); # tmp1 to hold F_00_19(b,c,d), b&=c^d
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+8)%16));
|
|
|
|
|
|
+ &xor($tmp1,$d); # tmp1=F_00_19(b,c,d)
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd
|
|
|
|
|
|
+ &rotl($f,1); # f=ROTATE(f,1)
|
|
|
|
|
|
+ &add($e,$tmp1); # e+=F_00_19(b,c,d)
|
|
|
|
|
|
+ &xor($c,$d); # restore $c
|
|
|
|
|
|
+ &mov($tmp1,$a); # b in next round
|
|
|
|
|
|
+ &rotr($b,$n==16?2:7); # b=ROTATE(b,30)
|
|
|
|
|
|
+ &mov(&swtmp($n%16),$f); # xi=f
|
|
|
|
|
|
+ &rotl($a,5); # ROTATE(a,5)
|
|
|
|
|
|
+ &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e
|
|
|
|
|
|
+ &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round
|
|
|
|
|
|
+ &add($f,$a); # f+=ROTATE(a,5)
|
|
|
|
|
|
+} else {
|
|
|
|
|
|
+ &mov($tmp1,$c); # tmp1 to hold F_00_19(b,c,d)
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd)
|
|
|
|
|
|
+ &xor($tmp1,$d);
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+8)%16));
|
|
|
|
|
|
+ &and($tmp1,$b);
|
|
|
|
|
|
&xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd
|
|
|
|
|
|
&rotl($f,1); # f=ROTATE(f,1)
|
|
|
|
|
|
&xor($tmp1,$d); # tmp1=F_00_19(b,c,d)
|
|
|
|
|
|
- &mov(&swtmp($n%16),$f); # xi=f
|
|
|
|
|
|
- &lea($f,&DWP(0x5a827999,$f,$e));# f+=K_00_19+e
|
|
|
|
|
|
- &mov($e,$a); # e becomes volatile
|
|
|
|
|
|
- &rotl($e,5); # e=ROTATE(a,5)
|
|
|
|
|
|
- &add($f,$tmp1); # f+=F_00_19(b,c,d)
|
|
|
|
|
|
- &add($f,$e); # f+=ROTATE(a,5)
|
|
|
|
|
|
+ &add($e,$tmp1); # e+=F_00_19(b,c,d)
|
|
|
|
|
|
+ &mov($tmp1,$a);
|
|
|
|
|
|
+ &rotr($b,2); # b=ROTATE(b,30)
|
|
|
|
|
|
+ &mov(&swtmp($n%16),$f); # xi=f
|
|
|
|
|
|
+ &rotl($tmp1,5); # ROTATE(a,5)
|
|
|
|
|
|
+ &lea($f,&DWP(0x5a827999,$f,$e));# f+=F_00_19(b,c,d)+e
|
|
|
|
|
|
+ &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round
|
|
|
|
|
|
+ &add($f,$tmp1); # f+=ROTATE(a,5)
|
|
|
|
|
|
+}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
sub BODY_20_39
|
|
|
|
|
|
@@ -102,21 +209,41 @@ sub BODY_20_39
|
|
|
|
|
|
|
|
|
|
|
|
&comment("20_39 $n");
|
|
|
|
|
|
|
|
|
|
|
|
+if ($alt) {
|
|
|
|
|
|
+ &xor($tmp1,$c); # tmp1 to hold F_20_39(b,c,d), b^=c
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd)
|
|
|
|
|
|
+ &xor($tmp1,$d); # tmp1 holds F_20_39(b,c,d)
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+8)%16));
|
|
|
|
|
|
+ &add($e,$tmp1); # e+=F_20_39(b,c,d)
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd
|
|
|
|
|
|
+ &rotl($f,1); # f=ROTATE(f,1)
|
|
|
|
|
|
+ &mov($tmp1,$a); # b in next round
|
|
|
|
|
|
+ &rotr($b,7); # b=ROTATE(b,30)
|
|
|
|
|
|
+ &mov(&swtmp($n%16),$f) if($n<77);# xi=f
|
|
|
|
|
|
+ &rotl($a,5); # ROTATE(a,5)
|
|
|
|
|
|
+ &xor($b,$c) if($n==39);# warm up for BODY_40_59
|
|
|
|
|
|
+ &and($tmp1,$b) if($n==39);
|
|
|
|
|
|
+ &lea($f,&DWP($K,$f,$e)); # f+=e+K_XX_YY
|
|
|
|
|
|
+ &mov($e,&swtmp(($n+1)%16)) if($n<79);# pre-fetch f for next round
|
|
|
|
|
|
+ &add($f,$a); # f+=ROTATE(a,5)
|
|
|
|
|
|
+ &rotr($a,5) if ($n==79);
|
|
|
|
|
|
+} else {
|
|
|
|
|
|
&mov($tmp1,$b); # tmp1 to hold F_20_39(b,c,d)
|
|
|
|
|
|
- &mov($f,&swtmp($n%16)); # f to hold Xupdate(xi,xa,xb,xc,xd)
|
|
|
|
|
|
- &rotr($b,2); # b=ROTATE(b,30)
|
|
|
|
|
|
- &xor($f,&swtmp(($n+2)%16));
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd)
|
|
|
|
|
|
&xor($tmp1,$c);
|
|
|
|
|
|
&xor($f,&swtmp(($n+8)%16));
|
|
|
|
|
|
&xor($tmp1,$d); # tmp1 holds F_20_39(b,c,d)
|
|
|
|
|
|
&xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd
|
|
|
|
|
|
&rotl($f,1); # f=ROTATE(f,1)
|
|
|
|
|
|
- &add($tmp1,$e);
|
|
|
|
|
|
- &mov(&swtmp($n%16),$f); # xi=f
|
|
|
|
|
|
- &mov($e,$a); # e becomes volatile
|
|
|
|
|
|
- &rotl($e,5); # e=ROTATE(a,5)
|
|
|
|
|
|
- &lea($f,&DWP($K,$f,$tmp1)); # f+=K_20_39+e
|
|
|
|
|
|
- &add($f,$e); # f+=ROTATE(a,5)
|
|
|
|
|
|
+ &add($e,$tmp1); # e+=F_20_39(b,c,d)
|
|
|
|
|
|
+ &rotr($b,2); # b=ROTATE(b,30)
|
|
|
|
|
|
+ &mov($tmp1,$a);
|
|
|
|
|
|
+ &rotl($tmp1,5); # ROTATE(a,5)
|
|
|
|
|
|
+ &mov(&swtmp($n%16),$f) if($n<77);# xi=f
|
|
|
|
|
|
+ &lea($f,&DWP($K,$f,$e)); # f+=e+K_XX_YY
|
|
|
|
|
|
+ &mov($e,&swtmp(($n+1)%16)) if($n<79);# pre-fetch f for next round
|
|
|
|
|
|
+ &add($f,$tmp1); # f+=ROTATE(a,5)
|
|
|
|
|
|
+}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
sub BODY_40_59
|
|
|
|
|
|
@@ -125,41 +252,86 @@ sub BODY_40_59
|
|
|
|
|
|
|
|
|
|
|
|
&comment("40_59 $n");
|
|
|
|
|
|
|
|
|
|
|
|
- &mov($f,&swtmp($n%16)); # f to hold Xupdate(xi,xa,xb,xc,xd)
|
|
|
|
|
|
- &mov($tmp1,&swtmp(($n+2)%16));
|
|
|
|
|
|
- &xor($f,$tmp1);
|
|
|
|
|
|
- &mov($tmp1,&swtmp(($n+8)%16));
|
|
|
|
|
|
- &xor($f,$tmp1);
|
|
|
|
|
|
- &mov($tmp1,&swtmp(($n+13)%16));
|
|
|
|
|
|
- &xor($f,$tmp1); # f holds xa^xb^xc^xd
|
|
|
|
|
|
- &mov($tmp1,$b); # tmp1 to hold F_40_59(b,c,d)
|
|
|
|
|
|
+if ($alt) {
|
|
|
|
|
|
+ &add($e,$tmp1); # e+=b&(c^d)
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd)
|
|
|
|
|
|
+ &mov($tmp1,$d);
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+8)%16));
|
|
|
|
|
|
+ &xor($c,$d); # restore $c
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd
|
|
|
|
|
|
&rotl($f,1); # f=ROTATE(f,1)
|
|
|
|
|
|
- &or($tmp1,$c);
|
|
|
|
|
|
- &mov(&swtmp($n%16),$f); # xi=f
|
|
|
|
|
|
- &and($tmp1,$d);
|
|
|
|
|
|
- &lea($f,&DWP(0x8f1bbcdc,$f,$e));# f+=K_40_59+e
|
|
|
|
|
|
- &mov($e,$b); # e becomes volatile and is used
|
|
|
|
|
|
- # to calculate F_40_59(b,c,d)
|
|
|
|
|
|
+ &and($tmp1,$c);
|
|
|
|
|
|
+ &rotr($b,7); # b=ROTATE(b,30)
|
|
|
|
|
|
+ &add($e,$tmp1); # e+=c&d
|
|
|
|
|
|
+ &mov($tmp1,$a); # b in next round
|
|
|
|
|
|
+ &mov(&swtmp($n%16),$f); # xi=f
|
|
|
|
|
|
+ &rotl($a,5); # ROTATE(a,5)
|
|
|
|
|
|
+ &xor($b,$c) if ($n<59);
|
|
|
|
|
|
+ &and($tmp1,$b) if ($n<59);# tmp1 to hold F_40_59(b,c,d)
|
|
|
|
|
|
+ &lea($f,&DWP(0x8f1bbcdc,$f,$e));# f+=K_40_59+e+(b&(c^d))
|
|
|
|
|
|
+ &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round
|
|
|
|
|
|
+ &add($f,$a); # f+=ROTATE(a,5)
|
|
|
|
|
|
+} else {
|
|
|
|
|
|
+ &mov($tmp1,$c); # tmp1 to hold F_40_59(b,c,d)
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+2)%16)); # f to hold Xupdate(xi,xa,xb,xc,xd)
|
|
|
|
|
|
+ &xor($tmp1,$d);
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+8)%16));
|
|
|
|
|
|
+ &and($tmp1,$b);
|
|
|
|
|
|
+ &xor($f,&swtmp(($n+13)%16)); # f holds xa^xb^xc^xd
|
|
|
|
|
|
+ &rotl($f,1); # f=ROTATE(f,1)
|
|
|
|
|
|
+ &add($tmp1,$e); # b&(c^d)+=e
|
|
|
|
|
|
&rotr($b,2); # b=ROTATE(b,30)
|
|
|
|
|
|
- &and($e,$c);
|
|
|
|
|
|
- &or($tmp1,$e); # tmp1 holds F_40_59(b,c,d)
|
|
|
|
|
|
- &mov($e,$a);
|
|
|
|
|
|
- &rotl($e,5); # e=ROTATE(a,5)
|
|
|
|
|
|
- &add($f,$tmp1); # f+=tmp1;
|
|
|
|
|
|
+ &mov($e,$a); # e becomes volatile
|
|
|
|
|
|
+ &rotl($e,5); # ROTATE(a,5)
|
|
|
|
|
|
+ &mov(&swtmp($n%16),$f); # xi=f
|
|
|
|
|
|
+ &lea($f,&DWP(0x8f1bbcdc,$f,$tmp1));# f+=K_40_59+e+(b&(c^d))
|
|
|
|
|
|
+ &mov($tmp1,$c);
|
|
|
|
|
|
&add($f,$e); # f+=ROTATE(a,5)
|
|
|
|
|
|
+ &and($tmp1,$d);
|
|
|
|
|
|
+ &mov($e,&swtmp(($n+1)%16)); # pre-fetch f for next round
|
|
|
|
|
|
+ &add($f,$tmp1); # f+=c&d
|
|
|
|
|
|
+}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
&function_begin("sha1_block_data_order");
|
|
|
|
|
|
+if ($xmm) {
|
|
|
|
|
|
+ &static_label("ssse3_shortcut");
|
|
|
|
|
|
+ &static_label("avx_shortcut") if ($ymm);
|
|
|
|
|
|
+ &static_label("K_XX_XX");
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &call (&label("pic_point")); # make it PIC!
|
|
|
|
|
|
+ &set_label("pic_point");
|
|
|
|
|
|
+ &blindpop($tmp1);
|
|
|
|
|
|
+ &picmeup($T,"OPENSSL_ia32cap_X",$tmp1,&label("pic_point"));
|
|
|
|
|
|
+ &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov ($A,&DWP(0,$T));
|
|
|
|
|
|
+ &mov ($D,&DWP(4,$T));
|
|
|
|
|
|
+ &test ($D,1<<9); # check SSSE3 bit
|
|
|
|
|
|
+ &jz (&label("x86"));
|
|
|
|
|
|
+ &test ($A,1<<24); # check FXSR bit
|
|
|
|
|
|
+ &jz (&label("x86"));
|
|
|
|
|
|
+ if ($ymm) {
|
|
|
|
|
|
+ &and ($D,1<<28); # mask AVX bit
|
|
|
|
|
|
+ &and ($A,1<<30); # mask "Intel CPU" bit
|
|
|
|
|
|
+ &or ($A,$D);
|
|
|
|
|
|
+ &cmp ($A,1<<28|1<<30);
|
|
|
|
|
|
+ &je (&label("avx_shortcut"));
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+ &jmp (&label("ssse3_shortcut"));
|
|
|
|
|
|
+ &set_label("x86",16);
|
|
|
|
|
|
+}
|
|
|
|
|
|
&mov($tmp1,&wparam(0)); # SHA_CTX *c
|
|
|
|
|
|
&mov($T,&wparam(1)); # const void *input
|
|
|
|
|
|
&mov($A,&wparam(2)); # size_t num
|
|
|
|
|
|
- &stack_push(16); # allocate X[16]
|
|
|
|
|
|
+ &stack_push(16+3); # allocate X[16]
|
|
|
|
|
|
&shl($A,6);
|
|
|
|
|
|
&add($A,$T);
|
|
|
|
|
|
&mov(&wparam(2),$A); # pointer beyond the end of input
|
|
|
|
|
|
&mov($E,&DWP(16,$tmp1));# pre-load E
|
|
|
|
|
|
+ &jmp(&label("loop"));
|
|
|
|
|
|
|
|
|
|
|
|
- &set_label("loop",16);
|
|
|
|
|
|
+&set_label("loop",16);
|
|
|
|
|
|
|
|
|
|
|
|
# copy input chunk to X, but reversing byte order!
|
|
|
|
|
|
for ($i=0; $i<16; $i+=4)
|
|
|
|
|
|
@@ -213,8 +385,845 @@ sub BODY_40_59
|
|
|
|
|
|
&mov(&DWP(16,$tmp1),$C);
|
|
|
|
|
|
&jb(&label("loop"));
|
|
|
|
|
|
|
|
|
|
|
|
- &stack_pop(16);
|
|
|
|
|
|
+ &stack_pop(16+3);
|
|
|
|
|
|
&function_end("sha1_block_data_order");
|
|
|
|
|
|
+
|
|
|
|
|
|
+if ($xmm) {
|
|
|
|
|
|
+######################################################################
|
|
|
|
|
|
+# The SSSE3 implementation.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# %xmm[0-7] are used as ring @X[] buffer containing quadruples of last
|
|
|
|
|
|
+# 32 elements of the message schedule or Xupdate outputs. First 4
|
|
|
|
|
|
+# quadruples are simply byte-swapped input, next 4 are calculated
|
|
|
|
|
|
+# according to method originally suggested by Dean Gaudet (modulo
|
|
|
|
|
|
+# being implemented in SSSE3). Once 8 quadruples or 32 elements are
|
|
|
|
|
|
+# collected, it switches to routine proposed by Max Locktyukhin.
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Calculations inevitably require temporary reqisters, and there are
|
|
|
|
|
|
+# no %xmm registers left to spare. For this reason part of the ring
|
|
|
|
|
|
+# buffer, X[2..4] to be specific, is offloaded to 3 quadriples ring
|
|
|
|
|
|
+# buffer on the stack. Keep in mind that X[2] is alias X[-6], X[3] -
|
|
|
|
|
|
+# X[-5], and X[4] - X[-4]...
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Another notable optimization is aggressive stack frame compression
|
|
|
|
|
|
+# aiming to minimize amount of 9-byte instructions...
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Yet another notable optimization is "jumping" $B variable. It means
|
|
|
|
|
|
+# that there is no register permanently allocated for $B value. This
|
|
|
|
|
|
+# allowed to eliminate one instruction from body_20_39...
|
|
|
|
|
|
+#
|
|
|
|
|
|
+my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded
|
|
|
|
|
|
+my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4
|
|
|
|
|
|
+my @V=($A,$B,$C,$D,$E);
|
|
|
|
|
|
+my $j=0; # hash round
|
|
|
|
|
|
+my @T=($T,$tmp1);
|
|
|
|
|
|
+my $inp;
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $_rol=sub { &rol(@_) };
|
|
|
|
|
|
+my $_ror=sub { &ror(@_) };
|
|
|
|
|
|
+
|
|
|
|
|
|
+&function_begin("_sha1_block_data_order_ssse3");
|
|
|
|
|
|
+ &call (&label("pic_point")); # make it PIC!
|
|
|
|
|
|
+ &set_label("pic_point");
|
|
|
|
|
|
+ &blindpop($tmp1);
|
|
|
|
|
|
+ &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1));
|
|
|
|
|
|
+&set_label("ssse3_shortcut");
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@X[3],&QWP(0,$tmp1)); # K_00_19
|
|
|
|
|
|
+ &movdqa (@X[4],&QWP(16,$tmp1)); # K_20_39
|
|
|
|
|
|
+ &movdqa (@X[5],&QWP(32,$tmp1)); # K_40_59
|
|
|
|
|
|
+ &movdqa (@X[6],&QWP(48,$tmp1)); # K_60_79
|
|
|
|
|
|
+ &movdqa (@X[2],&QWP(64,$tmp1)); # pbswap mask
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov ($E,&wparam(0)); # load argument block
|
|
|
|
|
|
+ &mov ($inp=@T[1],&wparam(1));
|
|
|
|
|
|
+ &mov ($D,&wparam(2));
|
|
|
|
|
|
+ &mov (@T[0],"esp");
|
|
|
|
|
|
+
|
|
|
|
|
|
+ # stack frame layout
|
|
|
|
|
|
+ #
|
|
|
|
|
|
+ # +0 X[0]+K X[1]+K X[2]+K X[3]+K # XMM->IALU xfer area
|
|
|
|
|
|
+ # X[4]+K X[5]+K X[6]+K X[7]+K
|
|
|
|
|
|
+ # X[8]+K X[9]+K X[10]+K X[11]+K
|
|
|
|
|
|
+ # X[12]+K X[13]+K X[14]+K X[15]+K
|
|
|
|
|
|
+ #
|
|
|
|
|
|
+ # +64 X[0] X[1] X[2] X[3] # XMM->XMM backtrace area
|
|
|
|
|
|
+ # X[4] X[5] X[6] X[7]
|
|
|
|
|
|
+ # X[8] X[9] X[10] X[11] # even borrowed for K_00_19
|
|
|
|
|
|
+ #
|
|
|
|
|
|
+ # +112 K_20_39 K_20_39 K_20_39 K_20_39 # constants
|
|
|
|
|
|
+ # K_40_59 K_40_59 K_40_59 K_40_59
|
|
|
|
|
|
+ # K_60_79 K_60_79 K_60_79 K_60_79
|
|
|
|
|
|
+ # K_00_19 K_00_19 K_00_19 K_00_19
|
|
|
|
|
|
+ # pbswap mask
|
|
|
|
|
|
+ #
|
|
|
|
|
|
+ # +192 ctx # argument block
|
|
|
|
|
|
+ # +196 inp
|
|
|
|
|
|
+ # +200 end
|
|
|
|
|
|
+ # +204 esp
|
|
|
|
|
|
+ &sub ("esp",208);
|
|
|
|
|
|
+ &and ("esp",-64);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (&QWP(112+0,"esp"),@X[4]); # copy constants
|
|
|
|
|
|
+ &movdqa (&QWP(112+16,"esp"),@X[5]);
|
|
|
|
|
|
+ &movdqa (&QWP(112+32,"esp"),@X[6]);
|
|
|
|
|
|
+ &shl ($D,6); # len*64
|
|
|
|
|
|
+ &movdqa (&QWP(112+48,"esp"),@X[3]);
|
|
|
|
|
|
+ &add ($D,$inp); # end of input
|
|
|
|
|
|
+ &movdqa (&QWP(112+64,"esp"),@X[2]);
|
|
|
|
|
|
+ &add ($inp,64);
|
|
|
|
|
|
+ &mov (&DWP(192+0,"esp"),$E); # save argument block
|
|
|
|
|
|
+ &mov (&DWP(192+4,"esp"),$inp);
|
|
|
|
|
|
+ &mov (&DWP(192+8,"esp"),$D);
|
|
|
|
|
|
+ &mov (&DWP(192+12,"esp"),@T[0]); # save original %esp
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov ($A,&DWP(0,$E)); # load context
|
|
|
|
|
|
+ &mov ($B,&DWP(4,$E));
|
|
|
|
|
|
+ &mov ($C,&DWP(8,$E));
|
|
|
|
|
|
+ &mov ($D,&DWP(12,$E));
|
|
|
|
|
|
+ &mov ($E,&DWP(16,$E));
|
|
|
|
|
|
+ &mov (@T[0],$B); # magic seed
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqu (@X[-4&7],&QWP(-64,$inp)); # load input to %xmm[0-3]
|
|
|
|
|
|
+ &movdqu (@X[-3&7],&QWP(-48,$inp));
|
|
|
|
|
|
+ &movdqu (@X[-2&7],&QWP(-32,$inp));
|
|
|
|
|
|
+ &movdqu (@X[-1&7],&QWP(-16,$inp));
|
|
|
|
|
|
+ &pshufb (@X[-4&7],@X[2]); # byte swap
|
|
|
|
|
|
+ &pshufb (@X[-3&7],@X[2]);
|
|
|
|
|
|
+ &pshufb (@X[-2&7],@X[2]);
|
|
|
|
|
|
+ &movdqa (&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot
|
|
|
|
|
|
+ &pshufb (@X[-1&7],@X[2]);
|
|
|
|
|
|
+ &paddd (@X[-4&7],@X[3]); # add K_00_19
|
|
|
|
|
|
+ &paddd (@X[-3&7],@X[3]);
|
|
|
|
|
|
+ &paddd (@X[-2&7],@X[3]);
|
|
|
|
|
|
+ &movdqa (&QWP(0,"esp"),@X[-4&7]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ &psubd (@X[-4&7],@X[3]); # restore X[]
|
|
|
|
|
|
+ &movdqa (&QWP(0+16,"esp"),@X[-3&7]);
|
|
|
|
|
|
+ &psubd (@X[-3&7],@X[3]);
|
|
|
|
|
|
+ &movdqa (&QWP(0+32,"esp"),@X[-2&7]);
|
|
|
|
|
|
+ &psubd (@X[-2&7],@X[3]);
|
|
|
|
|
|
+ &movdqa (@X[0],@X[-3&7]);
|
|
|
|
|
|
+ &jmp (&label("loop"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+######################################################################
|
|
|
|
|
|
+# SSE instruction sequence is first broken to groups of indepentent
|
|
|
|
|
|
+# instructions, independent in respect to their inputs and shifter
|
|
|
|
|
|
+# (not all architectures have more than one). Then IALU instructions
|
|
|
|
|
|
+# are "knitted in" between the SSE groups. Distance is maintained for
|
|
|
|
|
|
+# SSE latency of 2 in hope that it fits better upcoming AMD Bulldozer
|
|
|
|
|
|
+# [which allegedly also implements SSSE3]...
|
|
|
|
|
|
+#
|
|
|
|
|
|
+# Temporary registers usage. X[2] is volatile at the entry and at the
|
|
|
|
|
|
+# end is restored from backtrace ring buffer. X[3] is expected to
|
|
|
|
|
|
+# contain current K_XX_XX constant and is used to caclulate X[-1]+K
|
|
|
|
|
|
+# from previous round, it becomes volatile the moment the value is
|
|
|
|
|
|
+# saved to stack for transfer to IALU. X[4] becomes volatile whenever
|
|
|
|
|
|
+# X[-4] is accumulated and offloaded to backtrace ring buffer, at the
|
|
|
|
|
|
+# end it is loaded with next K_XX_XX [which becomes X[3] in next
|
|
|
|
|
|
+# round]...
|
|
|
|
|
|
+#
|
|
|
|
|
|
+sub Xupdate_ssse3_16_31() # recall that $Xi starts wtih 4
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 40 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &palignr(@X[0],@X[-4&7],8); # compose "X[-14]" in "X[0]"
|
|
|
|
|
|
+ &movdqa (@X[2],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &paddd (@X[3],@X[-1&7]);
|
|
|
|
|
|
+ &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &psrldq (@X[2],4); # "X[-3]", 3 dwords
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &pxor (@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@X[2],@X[-2&7]); # "X[-3]"^"X[-8]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@X[0],@X[2]); # "X[0]"^="X[-3]"^"X[-8]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@X[4],@X[0]);
|
|
|
|
|
|
+ &movdqa (@X[2],@X[0]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pslldq (@X[4],12); # "X[0]"<<96, extract one dword
|
|
|
|
|
|
+ &paddd (@X[0],@X[0]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &psrld (@X[2],31);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (@X[3],@X[4]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &psrld (@X[4],30);
|
|
|
|
|
|
+ &por (@X[0],@X[2]); # "X[0]"<<<=1
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5); # restore X[] from backtrace buffer
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pslld (@X[3],2);
|
|
|
|
|
|
+ &pxor (@X[0],@X[4]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (@X[4],&QWP(112-16+16*(($Xi)/5),"esp")); # K_XX_XX
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@X[0],@X[3]); # "X[0]"^=("X[0]"<<96)<<<2
|
|
|
|
|
|
+ &movdqa (@X[1],@X[-2&7]) if ($Xi<7);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions [if any]
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi++; push(@X,shift(@X)); # "rotate" X[]
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xupdate_ssse3_32_79()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@X[2],@X[-1&7]) if ($Xi==8);
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ &pxor (@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]"
|
|
|
|
|
|
+ &palignr(@X[2],@X[-2&7],8); # compose "X[-6]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@X[0],@X[-7&7]); # "X[0]"^="X[-28]"
|
|
|
|
|
|
+ &movdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]); # save X[] to backtrace buffer
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ if ($Xi%5) {
|
|
|
|
|
|
+ &movdqa (@X[4],@X[3]); # "perpetuate" K_XX_XX...
|
|
|
|
|
|
+ } else { # ... or load next one
|
|
|
|
|
|
+ &movdqa (@X[4],&QWP(112-16+16*($Xi/5),"esp"));
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+ &paddd (@X[3],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pxor (@X[0],@X[2]); # "X[0]"^="X[-6]"
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@X[2],@X[0]);
|
|
|
|
|
|
+ &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &pslld (@X[0],2);
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &psrld (@X[2],30);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &por (@X[0],@X[2]); # "X[0]"<<<=2
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19); # restore X[] from backtrace buffer
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ &movdqa (@X[3],@X[0]) if ($Xi<19);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi++; push(@X,shift(@X)); # "rotate" X[]
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xuplast_ssse3_80()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &paddd (@X[3],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer IALU
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov ($inp=@T[1],&DWP(192+4,"esp"));
|
|
|
|
|
|
+ &cmp ($inp,&DWP(192+8,"esp"));
|
|
|
|
|
|
+ &je (&label("done"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &movdqa (@X[3],&QWP(112+48,"esp")); # K_00_19
|
|
|
|
|
|
+ &movdqa (@X[2],&QWP(112+64,"esp")); # pbswap mask
|
|
|
|
|
|
+ &movdqu (@X[-4&7],&QWP(0,$inp)); # load input
|
|
|
|
|
|
+ &movdqu (@X[-3&7],&QWP(16,$inp));
|
|
|
|
|
|
+ &movdqu (@X[-2&7],&QWP(32,$inp));
|
|
|
|
|
|
+ &movdqu (@X[-1&7],&QWP(48,$inp));
|
|
|
|
|
|
+ &add ($inp,64);
|
|
|
|
|
|
+ &pshufb (@X[-4&7],@X[2]); # byte swap
|
|
|
|
|
|
+ &mov (&DWP(192+4,"esp"),$inp);
|
|
|
|
|
|
+ &movdqa (&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi=0;
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xloop_ssse3()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &pshufb (@X[($Xi-3)&7],@X[2]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &paddd (@X[($Xi-4)&7],@X[3]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &movdqa (&QWP(0+16*$Xi,"esp"),@X[($Xi-4)&7]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &psubd (@X[($Xi-4)&7],@X[3]);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; }
|
|
|
|
|
|
+ $Xi++;
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xtail_ssse3()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub body_00_19 () {
|
|
|
|
|
|
+ (
|
|
|
|
|
|
+ '($a,$b,$c,$d,$e)=@V;'.
|
|
|
|
|
|
+ '&add ($e,&DWP(4*($j&15),"esp"));', # X[]+K xfer
|
|
|
|
|
|
+ '&xor ($c,$d);',
|
|
|
|
|
|
+ '&mov (@T[1],$a);', # $b in next round
|
|
|
|
|
|
+ '&$_rol ($a,5);',
|
|
|
|
|
|
+ '&and (@T[0],$c);', # ($b&($c^$d))
|
|
|
|
|
|
+ '&xor ($c,$d);', # restore $c
|
|
|
|
|
|
+ '&xor (@T[0],$d);',
|
|
|
|
|
|
+ '&add ($e,$a);',
|
|
|
|
|
|
+ '&$_ror ($b,$j?7:2);', # $b>>>2
|
|
|
|
|
|
+ '&add ($e,@T[0]);' .'$j++; unshift(@V,pop(@V)); unshift(@T,pop(@T));'
|
|
|
|
|
|
+ );
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub body_20_39 () {
|
|
|
|
|
|
+ (
|
|
|
|
|
|
+ '($a,$b,$c,$d,$e)=@V;'.
|
|
|
|
|
|
+ '&add ($e,&DWP(4*($j++&15),"esp"));', # X[]+K xfer
|
|
|
|
|
|
+ '&xor (@T[0],$d);', # ($b^$d)
|
|
|
|
|
|
+ '&mov (@T[1],$a);', # $b in next round
|
|
|
|
|
|
+ '&$_rol ($a,5);',
|
|
|
|
|
|
+ '&xor (@T[0],$c);', # ($b^$d^$c)
|
|
|
|
|
|
+ '&add ($e,$a);',
|
|
|
|
|
|
+ '&$_ror ($b,7);', # $b>>>2
|
|
|
|
|
|
+ '&add ($e,@T[0]);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));'
|
|
|
|
|
|
+ );
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub body_40_59 () {
|
|
|
|
|
|
+ (
|
|
|
|
|
|
+ '($a,$b,$c,$d,$e)=@V;'.
|
|
|
|
|
|
+ '&mov (@T[1],$c);',
|
|
|
|
|
|
+ '&xor ($c,$d);',
|
|
|
|
|
|
+ '&add ($e,&DWP(4*($j++&15),"esp"));', # X[]+K xfer
|
|
|
|
|
|
+ '&and (@T[1],$d);',
|
|
|
|
|
|
+ '&and (@T[0],$c);', # ($b&($c^$d))
|
|
|
|
|
|
+ '&$_ror ($b,7);', # $b>>>2
|
|
|
|
|
|
+ '&add ($e,@T[1]);',
|
|
|
|
|
|
+ '&mov (@T[1],$a);', # $b in next round
|
|
|
|
|
|
+ '&$_rol ($a,5);',
|
|
|
|
|
|
+ '&add ($e,@T[0]);',
|
|
|
|
|
|
+ '&xor ($c,$d);', # restore $c
|
|
|
|
|
|
+ '&add ($e,$a);' .'unshift(@V,pop(@V)); unshift(@T,pop(@T));'
|
|
|
|
|
|
+ );
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("loop",16);
|
|
|
|
|
|
+ &Xupdate_ssse3_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_ssse3_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xuplast_ssse3_80(\&body_20_39); # can jump to "done"
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $saved_j=$j; @saved_V=@V;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &Xloop_ssse3(\&body_20_39);
|
|
|
|
|
|
+ &Xloop_ssse3(\&body_20_39);
|
|
|
|
|
|
+ &Xloop_ssse3(\&body_20_39);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov (@T[1],&DWP(192,"esp")); # update context
|
|
|
|
|
|
+ &add ($A,&DWP(0,@T[1]));
|
|
|
|
|
|
+ &add (@T[0],&DWP(4,@T[1])); # $b
|
|
|
|
|
|
+ &add ($C,&DWP(8,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(0,@T[1]),$A);
|
|
|
|
|
|
+ &add ($D,&DWP(12,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(4,@T[1]),@T[0]);
|
|
|
|
|
|
+ &add ($E,&DWP(16,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(8,@T[1]),$C);
|
|
|
|
|
|
+ &mov ($B,@T[0]);
|
|
|
|
|
|
+ &mov (&DWP(12,@T[1]),$D);
|
|
|
|
|
|
+ &mov (&DWP(16,@T[1]),$E);
|
|
|
|
|
|
+ &movdqa (@X[0],@X[-3&7]);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &jmp (&label("loop"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("done",16); $j=$saved_j; @V=@saved_V;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &Xtail_ssse3(\&body_20_39);
|
|
|
|
|
|
+ &Xtail_ssse3(\&body_20_39);
|
|
|
|
|
|
+ &Xtail_ssse3(\&body_20_39);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov (@T[1],&DWP(192,"esp")); # update context
|
|
|
|
|
|
+ &add ($A,&DWP(0,@T[1]));
|
|
|
|
|
|
+ &mov ("esp",&DWP(192+12,"esp")); # restore %esp
|
|
|
|
|
|
+ &add (@T[0],&DWP(4,@T[1])); # $b
|
|
|
|
|
|
+ &add ($C,&DWP(8,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(0,@T[1]),$A);
|
|
|
|
|
|
+ &add ($D,&DWP(12,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(4,@T[1]),@T[0]);
|
|
|
|
|
|
+ &add ($E,&DWP(16,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(8,@T[1]),$C);
|
|
|
|
|
|
+ &mov (&DWP(12,@T[1]),$D);
|
|
|
|
|
|
+ &mov (&DWP(16,@T[1]),$E);
|
|
|
|
|
|
+
|
|
|
|
|
|
+&function_end("_sha1_block_data_order_ssse3");
|
|
|
|
|
|
+
|
|
|
|
|
|
+if ($ymm) {
|
|
|
|
|
|
+my $Xi=4; # 4xSIMD Xupdate round, start pre-seeded
|
|
|
|
|
|
+my @X=map("xmm$_",(4..7,0..3)); # pre-seeded for $Xi=4
|
|
|
|
|
|
+my @V=($A,$B,$C,$D,$E);
|
|
|
|
|
|
+my $j=0; # hash round
|
|
|
|
|
|
+my @T=($T,$tmp1);
|
|
|
|
|
|
+my $inp;
|
|
|
|
|
|
+
|
|
|
|
|
|
+my $_rol=sub { &shld(@_[0],@_) };
|
|
|
|
|
|
+my $_ror=sub { &shrd(@_[0],@_) };
|
|
|
|
|
|
+
|
|
|
|
|
|
+&function_begin("_sha1_block_data_order_avx");
|
|
|
|
|
|
+ &call (&label("pic_point")); # make it PIC!
|
|
|
|
|
|
+ &set_label("pic_point");
|
|
|
|
|
|
+ &blindpop($tmp1);
|
|
|
|
|
|
+ &lea ($tmp1,&DWP(&label("K_XX_XX")."-".&label("pic_point"),$tmp1));
|
|
|
|
|
|
+&set_label("avx_shortcut");
|
|
|
|
|
|
+ &vzeroall();
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vmovdqa(@X[3],&QWP(0,$tmp1)); # K_00_19
|
|
|
|
|
|
+ &vmovdqa(@X[4],&QWP(16,$tmp1)); # K_20_39
|
|
|
|
|
|
+ &vmovdqa(@X[5],&QWP(32,$tmp1)); # K_40_59
|
|
|
|
|
|
+ &vmovdqa(@X[6],&QWP(48,$tmp1)); # K_60_79
|
|
|
|
|
|
+ &vmovdqa(@X[2],&QWP(64,$tmp1)); # pbswap mask
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov ($E,&wparam(0)); # load argument block
|
|
|
|
|
|
+ &mov ($inp=@T[1],&wparam(1));
|
|
|
|
|
|
+ &mov ($D,&wparam(2));
|
|
|
|
|
|
+ &mov (@T[0],"esp");
|
|
|
|
|
|
+
|
|
|
|
|
|
+ # stack frame layout
|
|
|
|
|
|
+ #
|
|
|
|
|
|
+ # +0 X[0]+K X[1]+K X[2]+K X[3]+K # XMM->IALU xfer area
|
|
|
|
|
|
+ # X[4]+K X[5]+K X[6]+K X[7]+K
|
|
|
|
|
|
+ # X[8]+K X[9]+K X[10]+K X[11]+K
|
|
|
|
|
|
+ # X[12]+K X[13]+K X[14]+K X[15]+K
|
|
|
|
|
|
+ #
|
|
|
|
|
|
+ # +64 X[0] X[1] X[2] X[3] # XMM->XMM backtrace area
|
|
|
|
|
|
+ # X[4] X[5] X[6] X[7]
|
|
|
|
|
|
+ # X[8] X[9] X[10] X[11] # even borrowed for K_00_19
|
|
|
|
|
|
+ #
|
|
|
|
|
|
+ # +112 K_20_39 K_20_39 K_20_39 K_20_39 # constants
|
|
|
|
|
|
+ # K_40_59 K_40_59 K_40_59 K_40_59
|
|
|
|
|
|
+ # K_60_79 K_60_79 K_60_79 K_60_79
|
|
|
|
|
|
+ # K_00_19 K_00_19 K_00_19 K_00_19
|
|
|
|
|
|
+ # pbswap mask
|
|
|
|
|
|
+ #
|
|
|
|
|
|
+ # +192 ctx # argument block
|
|
|
|
|
|
+ # +196 inp
|
|
|
|
|
|
+ # +200 end
|
|
|
|
|
|
+ # +204 esp
|
|
|
|
|
|
+ &sub ("esp",208);
|
|
|
|
|
|
+ &and ("esp",-64);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vmovdqa(&QWP(112+0,"esp"),@X[4]); # copy constants
|
|
|
|
|
|
+ &vmovdqa(&QWP(112+16,"esp"),@X[5]);
|
|
|
|
|
|
+ &vmovdqa(&QWP(112+32,"esp"),@X[6]);
|
|
|
|
|
|
+ &shl ($D,6); # len*64
|
|
|
|
|
|
+ &vmovdqa(&QWP(112+48,"esp"),@X[3]);
|
|
|
|
|
|
+ &add ($D,$inp); # end of input
|
|
|
|
|
|
+ &vmovdqa(&QWP(112+64,"esp"),@X[2]);
|
|
|
|
|
|
+ &add ($inp,64);
|
|
|
|
|
|
+ &mov (&DWP(192+0,"esp"),$E); # save argument block
|
|
|
|
|
|
+ &mov (&DWP(192+4,"esp"),$inp);
|
|
|
|
|
|
+ &mov (&DWP(192+8,"esp"),$D);
|
|
|
|
|
|
+ &mov (&DWP(192+12,"esp"),@T[0]); # save original %esp
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov ($A,&DWP(0,$E)); # load context
|
|
|
|
|
|
+ &mov ($B,&DWP(4,$E));
|
|
|
|
|
|
+ &mov ($C,&DWP(8,$E));
|
|
|
|
|
|
+ &mov ($D,&DWP(12,$E));
|
|
|
|
|
|
+ &mov ($E,&DWP(16,$E));
|
|
|
|
|
|
+ &mov (@T[0],$B); # magic seed
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vmovdqu(@X[-4&7],&QWP(-64,$inp)); # load input to %xmm[0-3]
|
|
|
|
|
|
+ &vmovdqu(@X[-3&7],&QWP(-48,$inp));
|
|
|
|
|
|
+ &vmovdqu(@X[-2&7],&QWP(-32,$inp));
|
|
|
|
|
|
+ &vmovdqu(@X[-1&7],&QWP(-16,$inp));
|
|
|
|
|
|
+ &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap
|
|
|
|
|
|
+ &vpshufb(@X[-3&7],@X[-3&7],@X[2]);
|
|
|
|
|
|
+ &vpshufb(@X[-2&7],@X[-2&7],@X[2]);
|
|
|
|
|
|
+ &vmovdqa(&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot
|
|
|
|
|
|
+ &vpshufb(@X[-1&7],@X[-1&7],@X[2]);
|
|
|
|
|
|
+ &vpaddd (@X[0],@X[-4&7],@X[3]); # add K_00_19
|
|
|
|
|
|
+ &vpaddd (@X[1],@X[-3&7],@X[3]);
|
|
|
|
|
|
+ &vpaddd (@X[2],@X[-2&7],@X[3]);
|
|
|
|
|
|
+ &vmovdqa(&QWP(0,"esp"),@X[0]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ &vmovdqa(&QWP(0+16,"esp"),@X[1]);
|
|
|
|
|
|
+ &vmovdqa(&QWP(0+32,"esp"),@X[2]);
|
|
|
|
|
|
+ &jmp (&label("loop"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xupdate_avx_16_31() # recall that $Xi starts wtih 4
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 40 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpalignr(@X[0],@X[-3&7],@X[-4&7],8); # compose "X[-14]" in "X[0]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpaddd (@X[3],@X[3],@X[-1&7]);
|
|
|
|
|
|
+ &vmovdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]);# save X[] to backtrace buffer
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpsrldq(@X[2],@X[-1&7],4); # "X[-3]", 3 dwords
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"^="X[-16]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@X[2],@X[2],@X[-2&7]); # "X[-3]"^"X[-8]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[2]); # "X[0]"^="X[-3]"^"X[-8]"
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpsrld (@X[2],@X[0],31);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpslldq(@X[4],@X[0],12); # "X[0]"<<96, extract one dword
|
|
|
|
|
|
+ &vpaddd (@X[0],@X[0],@X[0]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpsrld (@X[3],@X[4],30);
|
|
|
|
|
|
+ &vpor (@X[0],@X[0],@X[2]); # "X[0]"<<<=1
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpslld (@X[4],@X[4],2);
|
|
|
|
|
|
+ &vmovdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if ($Xi>5); # restore X[] from backtrace buffer
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[3]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[4]); # "X[0]"^=("X[0]"<<96)<<<2
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vmovdqa (@X[4],&QWP(112-16+16*(($Xi)/5),"esp")); # K_XX_XX
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions [if any]
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi++; push(@X,shift(@X)); # "rotate" X[]
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xupdate_avx_32_79()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 to 48 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpalignr(@X[2],@X[-1&7],@X[-2&7],8); # compose "X[-6]"
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[-4&7]); # "X[0]"="X[-32]"^"X[-16]"
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[-7&7]); # "X[0]"^="X[-28]"
|
|
|
|
|
|
+ &vmovdqa (&QWP(64+16*(($Xi-4)%3),"esp"),@X[-4&7]); # save X[] to backtrace buffer
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ if ($Xi%5) {
|
|
|
|
|
|
+ &vmovdqa (@X[4],@X[3]); # "perpetuate" K_XX_XX...
|
|
|
|
|
|
+ } else { # ... or load next one
|
|
|
|
|
|
+ &vmovdqa (@X[4],&QWP(112-16+16*($Xi/5),"esp"));
|
|
|
|
|
|
+ }
|
|
|
|
|
|
+ &vpaddd (@X[3],@X[3],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpxor (@X[0],@X[0],@X[2]); # "X[0]"^="X[-6]"
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpsrld (@X[2],@X[0],30);
|
|
|
|
|
|
+ &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpslld (@X[0],@X[0],2);
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vpor (@X[0],@X[0],@X[2]); # "X[0]"<<<=2
|
|
|
|
|
|
+ eval(shift(@insns)); # body_20_39
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vmovdqa (@X[2],&QWP(64+16*(($Xi-6)%3),"esp")) if($Xi<19); # restore X[] from backtrace buffer
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # rol
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns)); # ror
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi++; push(@X,shift(@X)); # "rotate" X[]
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xuplast_avx_80()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpaddd (@X[3],@X[3],@X[-1&7]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vmovdqa (&QWP(0+16*(($Xi-1)&3),"esp"),@X[3]); # X[]+K xfer IALU
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; } # remaining instructions
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov ($inp=@T[1],&DWP(192+4,"esp"));
|
|
|
|
|
|
+ &cmp ($inp,&DWP(192+8,"esp"));
|
|
|
|
|
|
+ &je (&label("done"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vmovdqa(@X[3],&QWP(112+48,"esp")); # K_00_19
|
|
|
|
|
|
+ &vmovdqa(@X[2],&QWP(112+64,"esp")); # pbswap mask
|
|
|
|
|
|
+ &vmovdqu(@X[-4&7],&QWP(0,$inp)); # load input
|
|
|
|
|
|
+ &vmovdqu(@X[-3&7],&QWP(16,$inp));
|
|
|
|
|
|
+ &vmovdqu(@X[-2&7],&QWP(32,$inp));
|
|
|
|
|
|
+ &vmovdqu(@X[-1&7],&QWP(48,$inp));
|
|
|
|
|
|
+ &add ($inp,64);
|
|
|
|
|
|
+ &vpshufb(@X[-4&7],@X[-4&7],@X[2]); # byte swap
|
|
|
|
|
|
+ &mov (&DWP(192+4,"esp"),$inp);
|
|
|
|
|
|
+ &vmovdqa(&QWP(112-16,"esp"),@X[3]); # borrow last backtrace slot
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $Xi=0;
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xloop_avx()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpshufb (@X[($Xi-3)&7],@X[($Xi-3)&7],@X[2]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vpaddd (@X[$Xi&7],@X[($Xi-4)&7],@X[3]);
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ &vmovdqa (&QWP(0+16*$Xi,"esp"),@X[$Xi&7]); # X[]+K xfer to IALU
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+ eval(shift(@insns));
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; }
|
|
|
|
|
|
+ $Xi++;
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+sub Xtail_avx()
|
|
|
|
|
|
+{ use integer;
|
|
|
|
|
|
+ my $body = shift;
|
|
|
|
|
|
+ my @insns = (&$body,&$body,&$body,&$body); # 32 instructions
|
|
|
|
|
|
+ my ($a,$b,$c,$d,$e);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ foreach (@insns) { eval; }
|
|
|
|
|
|
+}
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("loop",16);
|
|
|
|
|
|
+ &Xupdate_avx_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_16_31(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_00_19);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_40_59);
|
|
|
|
|
|
+ &Xupdate_avx_32_79(\&body_20_39);
|
|
|
|
|
|
+ &Xuplast_avx_80(\&body_20_39); # can jump to "done"
|
|
|
|
|
|
+
|
|
|
|
|
|
+ $saved_j=$j; @saved_V=@V;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &Xloop_avx(\&body_20_39);
|
|
|
|
|
|
+ &Xloop_avx(\&body_20_39);
|
|
|
|
|
|
+ &Xloop_avx(\&body_20_39);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov (@T[1],&DWP(192,"esp")); # update context
|
|
|
|
|
|
+ &add ($A,&DWP(0,@T[1]));
|
|
|
|
|
|
+ &add (@T[0],&DWP(4,@T[1])); # $b
|
|
|
|
|
|
+ &add ($C,&DWP(8,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(0,@T[1]),$A);
|
|
|
|
|
|
+ &add ($D,&DWP(12,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(4,@T[1]),@T[0]);
|
|
|
|
|
|
+ &add ($E,&DWP(16,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(8,@T[1]),$C);
|
|
|
|
|
|
+ &mov ($B,@T[0]);
|
|
|
|
|
|
+ &mov (&DWP(12,@T[1]),$D);
|
|
|
|
|
|
+ &mov (&DWP(16,@T[1]),$E);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &jmp (&label("loop"));
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("done",16); $j=$saved_j; @V=@saved_V;
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &Xtail_avx(\&body_20_39);
|
|
|
|
|
|
+ &Xtail_avx(\&body_20_39);
|
|
|
|
|
|
+ &Xtail_avx(\&body_20_39);
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &vzeroall();
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &mov (@T[1],&DWP(192,"esp")); # update context
|
|
|
|
|
|
+ &add ($A,&DWP(0,@T[1]));
|
|
|
|
|
|
+ &mov ("esp",&DWP(192+12,"esp")); # restore %esp
|
|
|
|
|
|
+ &add (@T[0],&DWP(4,@T[1])); # $b
|
|
|
|
|
|
+ &add ($C,&DWP(8,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(0,@T[1]),$A);
|
|
|
|
|
|
+ &add ($D,&DWP(12,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(4,@T[1]),@T[0]);
|
|
|
|
|
|
+ &add ($E,&DWP(16,@T[1]));
|
|
|
|
|
|
+ &mov (&DWP(8,@T[1]),$C);
|
|
|
|
|
|
+ &mov (&DWP(12,@T[1]),$D);
|
|
|
|
|
|
+ &mov (&DWP(16,@T[1]),$E);
|
|
|
|
|
|
+&function_end("_sha1_block_data_order_avx");
|
|
|
|
|
|
+}
|
|
|
|
|
|
+&set_label("K_XX_XX",64);
|
|
|
|
|
|
+&data_word(0x5a827999,0x5a827999,0x5a827999,0x5a827999); # K_00_19
|
|
|
|
|
|
+&data_word(0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1); # K_20_39
|
|
|
|
|
|
+&data_word(0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc); # K_40_59
|
|
|
|
|
|
+&data_word(0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6); # K_60_79
|
|
|
|
|
|
+&data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f); # pbswap mask
|
|
|
|
|
|
+}
|
|
|
|
|
|
&asciz("SHA1 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>");
|
|
|
|
|
|
|
|
|
|
|
|
&asm_finish();
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/x86cpuid.pl.intelopts openssl-1.0.0d/crypto/x86cpuid.pl
|
|
|
|
|
|
--- openssl-1.0.0d/crypto/x86cpuid.pl.intelopts 2010-02-12 18:02:12.000000000 +0100
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+++ openssl-1.0.0d/crypto/x86cpuid.pl 2011-11-03 09:55:42.000000000 +0100
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
|
|
|
|
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|
|
|
|
|
push(@INC, "${dir}perlasm", "perlasm");
|
|
|
|
|
|
@@ -20,7 +20,7 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA3
|
|
|
|
|
|
&pop ("eax");
|
|
|
|
|
|
&xor ("ecx","eax");
|
|
|
|
|
|
&bt ("ecx",21);
|
|
|
|
|
|
- &jnc (&label("done"));
|
|
|
|
|
|
+ &jnc (&label("generic"));
|
|
|
|
|
|
&xor ("eax","eax");
|
|
|
|
|
|
&cpuid ();
|
|
|
|
|
|
&mov ("edi","eax"); # max value for standard query level
|
|
|
|
|
|
@@ -51,7 +51,14 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA3
|
|
|
|
|
|
# AMD specific
|
|
|
|
|
|
&mov ("eax",0x80000000);
|
|
|
|
|
|
&cpuid ();
|
|
|
|
|
|
- &cmp ("eax",0x80000008);
|
|
|
|
|
|
+ &cmp ("eax",0x80000001);
|
|
|
|
|
|
+ &jb (&label("intel"));
|
|
|
|
|
|
+ &mov ("esi","eax");
|
|
|
|
|
|
+ &mov ("eax",0x80000001);
|
|
|
|
|
|
+ &cpuid ();
|
|
|
|
|
|
+ &or ("ebp","ecx");
|
|
|
|
|
|
+ &and ("ebp",1<<11|1); # isolate XOP bit
|
|
|
|
|
|
+ &cmp ("esi",0x80000008);
|
|
|
|
|
|
&jb (&label("intel"));
|
|
|
|
|
|
|
|
|
|
|
|
&mov ("eax",0x80000008);
|
|
|
|
|
|
@@ -62,13 +69,13 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA3
|
|
|
|
|
|
&mov ("eax",1);
|
|
|
|
|
|
&cpuid ();
|
|
|
|
|
|
&bt ("edx",28);
|
|
|
|
|
|
- &jnc (&label("done"));
|
|
|
|
|
|
+ &jnc (&label("generic"));
|
|
|
|
|
|
&shr ("ebx",16);
|
|
|
|
|
|
&and ("ebx",0xff);
|
|
|
|
|
|
&cmp ("ebx","esi");
|
|
|
|
|
|
- &ja (&label("done"));
|
|
|
|
|
|
+ &ja (&label("generic"));
|
|
|
|
|
|
&and ("edx",0xefffffff); # clear hyper-threading bit
|
|
|
|
|
|
- &jmp (&label("done"));
|
|
|
|
|
|
+ &jmp (&label("generic"));
|
|
|
|
|
|
|
|
|
|
|
|
&set_label("intel");
|
|
|
|
|
|
&cmp ("edi",4);
|
2011-11-03 09:18:52 +00:00
|
|
|
|
@@ -85,27 +92,45 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA3
|
2011-08-24 11:12:33 +00:00
|
|
|
|
&set_label("nocacheinfo");
|
|
|
|
|
|
&mov ("eax",1);
|
|
|
|
|
|
&cpuid ();
|
|
|
|
|
|
+ &and ("edx",0xbfefffff); # force reserved bits #20, #30 to 0
|
|
|
|
|
|
&cmp ("ebp",0);
|
|
|
|
|
|
- &jne (&label("notP4"));
|
|
|
|
|
|
+ &jne (&label("notintel"));
|
|
|
|
|
|
+ &or ("edx",1<<30); # set reserved bit#30 on Intel CPUs
|
|
|
|
|
|
&and (&HB("eax"),15); # familiy ID
|
|
|
|
|
|
&cmp (&HB("eax"),15); # P4?
|
|
|
|
|
|
- &jne (&label("notP4"));
|
|
|
|
|
|
- &or ("edx",1<<20); # use reserved bit to engage RC4_CHAR
|
|
|
|
|
|
-&set_label("notP4");
|
|
|
|
|
|
+ &jne (&label("notintel"));
|
|
|
|
|
|
+ &or ("edx",1<<20); # set reserved bit#20 to engage RC4_CHAR
|
|
|
|
|
|
+&set_label("notintel");
|
|
|
|
|
|
&bt ("edx",28); # test hyper-threading bit
|
|
|
|
|
|
- &jnc (&label("done"));
|
|
|
|
|
|
+ &jnc (&label("generic"));
|
|
|
|
|
|
&and ("edx",0xefffffff);
|
|
|
|
|
|
&cmp ("edi",0);
|
|
|
|
|
|
- &je (&label("done"));
|
|
|
|
|
|
+ &je (&label("generic"));
|
|
|
|
|
|
|
|
|
|
|
|
&or ("edx",0x10000000);
|
|
|
|
|
|
&shr ("ebx",16);
|
|
|
|
|
|
&cmp (&LB("ebx"),1);
|
|
|
|
|
|
- &ja (&label("done"));
|
|
|
|
|
|
+ &ja (&label("generic"));
|
|
|
|
|
|
&and ("edx",0xefffffff); # clear hyper-threading bit if not
|
|
|
|
|
|
+
|
|
|
|
|
|
+&set_label("generic");
|
|
|
|
|
|
+ &and ("ebp",1<<11); # isolate AMD XOP flag
|
|
|
|
|
|
+ &and ("ecx",0xfffff7ff); # force 11th bit to 0
|
|
|
|
|
|
+ &mov ("esi","edx");
|
|
|
|
|
|
+ &or ("ebp","ecx"); # merge AMD XOP flag
|
|
|
|
|
|
+
|
|
|
|
|
|
+ &bt ("ecx",27); # check OSXSAVE bit
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+ &jnc (&label("clear_avx"));
|
|
|
|
|
|
+ &xor ("ecx","ecx"); # XCR0
|
2011-08-24 11:12:33 +00:00
|
|
|
|
+ &data_byte(0x0f,0x01,0xd0); # xgetbv
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+ &and ("eax",6); # isolate XMM and YMM state support
|
2011-08-24 11:12:33 +00:00
|
|
|
|
+ &cmp ("eax",6);
|
|
|
|
|
|
+ &je (&label("done"));
|
|
|
|
|
|
+&set_label("clear_avx");
|
|
|
|
|
|
+ &and ("ebp",0xefffe7ff); # clear AVX, FMA and AMD XOP bits
|
|
|
|
|
|
&set_label("done");
|
|
|
|
|
|
- &mov ("eax","edx");
|
|
|
|
|
|
- &mov ("edx","ecx");
|
|
|
|
|
|
+ &mov ("eax","esi");
|
|
|
|
|
|
+ &mov ("edx","ebp");
|
|
|
|
|
|
&function_end("OPENSSL_ia32_cpuid");
|
|
|
|
|
|
|
|
|
|
|
|
&external_label("OPENSSL_ia32cap_P");
|
2011-11-03 09:18:52 +00:00
|
|
|
|
@@ -199,8 +224,9 @@ for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA3
|
2011-08-24 11:12:33 +00:00
|
|
|
|
&bt (&DWP(0,"ecx"),1);
|
|
|
|
|
|
&jnc (&label("no_x87"));
|
|
|
|
|
|
if ($sse2) {
|
|
|
|
|
|
- &bt (&DWP(0,"ecx"),26);
|
|
|
|
|
|
- &jnc (&label("no_sse2"));
|
|
|
|
|
|
+ &and ("ecx",1<<26|1<<24); # check SSE2 and FXSR bits
|
|
|
|
|
|
+ &cmp ("ecx",1<<26|1<<24);
|
|
|
|
|
|
+ &jne (&label("no_sse2"));
|
|
|
|
|
|
&pxor ("xmm0","xmm0");
|
|
|
|
|
|
&pxor ("xmm1","xmm1");
|
|
|
|
|
|
&pxor ("xmm2","xmm2");
|
|
|
|
|
|
diff -up openssl-1.0.0d/crypto/x86_64cpuid.pl.intelopts openssl-1.0.0d/crypto/x86_64cpuid.pl
|
|
|
|
|
|
--- openssl-1.0.0d/crypto/x86_64cpuid.pl.intelopts 2010-04-14 21:25:09.000000000 +0200
|
2011-11-03 09:18:52 +00:00
|
|
|
|
+++ openssl-1.0.0d/crypto/x86_64cpuid.pl 2011-08-24 12:50:56.000000000 +0200
|
2011-08-24 11:12:33 +00:00
|
|
|
|
@@ -1,4 +1,4 @@
|
|
|
|
|
|
-#!/usr/bin/env perl
|
|
|
|
|
|
+#!/usr/bin/perl
|
|
|
|
|
|
|
|
|
|
|
|
$flavour = shift;
|
|
|
|
|
|
$output = shift;
|
|
|
|
|
|
@@ -7,12 +7,18 @@ if ($flavour =~ /\./) { $output = $flavo
|
|
|
|
|
|
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
|
|
|
|
|
|
|
|
|
|
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|
|
|
|
|
-open STDOUT,"| $^X ${dir}perlasm/x86_64-xlate.pl $flavour $output";
|
|
|
|
|
|
+( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
|
|
|
|
|
|
+( $xlate="${dir}perlasm/x86_64-xlate.pl" and -f $xlate) or
|
|
|
|
|
|
+die "can't locate x86_64-xlate.pl";
|
|
|
|
|
|
+
|
|
|
|
|
|
+open STDOUT,"| $^X $xlate $flavour $output";
|
|
|
|
|
|
+
|
|
|
|
|
|
+($arg1,$arg2,$arg3,$arg4)=$win64?("%rcx","%rdx","%r8", "%r9") : # Win64 order
|
|
|
|
|
|
+ ("%rdi","%rsi","%rdx","%rcx"); # Unix order
|
|
|
|
|
|
|
|
|
|
|
|
-if ($win64) { $arg1="%rcx"; $arg2="%rdx"; }
|
|
|
|
|
|
-else { $arg1="%rdi"; $arg2="%rsi"; }
|
|
|
|
|
|
print<<___;
|
|
|
|
|
|
.extern OPENSSL_cpuid_setup
|
|
|
|
|
|
+.hidden OPENSSL_cpuid_setup
|
|
|
|
|
|
.section .init
|
|
|
|
|
|
call OPENSSL_cpuid_setup
|
|
|
|
|
|
|
|
|
|
|
|
@@ -46,7 +52,7 @@ OPENSSL_rdtsc:
|
|
|
|
|
|
.type OPENSSL_ia32_cpuid,\@abi-omnipotent
|
|
|
|
|
|
.align 16
|
|
|
|
|
|
OPENSSL_ia32_cpuid:
|
|
|
|
|
|
- mov %rbx,%r8
|
|
|
|
|
|
+ mov %rbx,%r8 # save %rbx
|
|
|
|
|
|
|
|
|
|
|
|
xor %eax,%eax
|
|
|
|
|
|
cpuid
|
|
|
|
|
|
@@ -78,7 +84,15 @@ OPENSSL_ia32_cpuid:
|
|
|
|
|
|
# AMD specific
|
|
|
|
|
|
mov \$0x80000000,%eax
|
|
|
|
|
|
cpuid
|
|
|
|
|
|
- cmp \$0x80000008,%eax
|
|
|
|
|
|
+ cmp \$0x80000001,%eax
|
|
|
|
|
|
+ jb .Lintel
|
|
|
|
|
|
+ mov %eax,%r10d
|
|
|
|
|
|
+ mov \$0x80000001,%eax
|
|
|
|
|
|
+ cpuid
|
|
|
|
|
|
+ or %ecx,%r9d
|
|
|
|
|
|
+ and \$0x00000801,%r9d # isolate AMD XOP bit, 1<<11
|
|
|
|
|
|
+
|
|
|
|
|
|
+ cmp \$0x80000008,%r10d
|
|
|
|
|
|
jb .Lintel
|
|
|
|
|
|
|
|
|
|
|
|
mov \$0x80000008,%eax
|
|
|
|
|
|
@@ -89,12 +103,12 @@ OPENSSL_ia32_cpuid:
|
|
|
|
|
|
mov \$1,%eax
|
|
|
|
|
|
cpuid
|
|
|
|
|
|
bt \$28,%edx # test hyper-threading bit
|
|
|
|
|
|
- jnc .Ldone
|
|
|
|
|
|
+ jnc .Lgeneric
|
|
|
|
|
|
shr \$16,%ebx # number of logical processors
|
|
|
|
|
|
cmp %r10b,%bl
|
|
|
|
|
|
- ja .Ldone
|
|
|
|
|
|
+ ja .Lgeneric
|
|
|
|
|
|
and \$0xefffffff,%edx # ~(1<<28)
|
|
|
|
|
|
- jmp .Ldone
|
|
|
|
|
|
+ jmp .Lgeneric
|
|
|
|
|
|
|
|
|
|
|
|
.Lintel:
|
|
|
|
|
|
cmp \$4,%r11d
|
|
|
|
|
|
@@ -111,30 +125,47 @@ OPENSSL_ia32_cpuid:
|
|
|
|
|
|
.Lnocacheinfo:
|
|
|
|
|
|
mov \$1,%eax
|
|
|
|
|
|
cpuid
|
|
|
|
|
|
+ and \$0xbfefffff,%edx # force reserved bits to 0
|
|
|
|
|
|
cmp \$0,%r9d
|
|
|
|
|
|
jne .Lnotintel
|
|
|
|
|
|
- or \$0x00100000,%edx # use reserved 20th bit to engage RC4_CHAR
|
|
|
|
|
|
+ or \$0x40000000,%edx # set reserved bit#30 on Intel CPUs
|
|
|
|
|
|
and \$15,%ah
|
|
|
|
|
|
cmp \$15,%ah # examine Family ID
|
|
|
|
|
|
- je .Lnotintel
|
|
|
|
|
|
- or \$0x40000000,%edx # use reserved bit to skip unrolled loop
|
|
|
|
|
|
+ jne .Lnotintel
|
|
|
|
|
|
+ or \$0x00100000,%edx # set reserved bit#20 to engage RC4_CHAR
|
|
|
|
|
|
.Lnotintel:
|
|
|
|
|
|
bt \$28,%edx # test hyper-threading bit
|
|
|
|
|
|
- jnc .Ldone
|
|
|
|
|
|
+ jnc .Lgeneric
|
|
|
|
|
|
and \$0xefffffff,%edx # ~(1<<28)
|
|
|
|
|
|
cmp \$0,%r10d
|
|
|
|
|
|
- je .Ldone
|
|
|
|
|
|
+ je .Lgeneric
|
|
|
|
|
|
|
|
|
|
|
|
or \$0x10000000,%edx # 1<<28
|
|
|
|
|
|
shr \$16,%ebx
|
|
|
|
|
|
cmp \$1,%bl # see if cache is shared
|
|
|
|
|
|
- ja .Ldone
|
|
|
|
|
|
+ ja .Lgeneric
|
|
|
|
|
|
and \$0xefffffff,%edx # ~(1<<28)
|
|
|
|
|
|
+.Lgeneric:
|
|
|
|
|
|
+ and \$0x00000800,%r9d # isolate AMD XOP flag
|
|
|
|
|
|
+ and \$0xfffff7ff,%ecx
|
|
|
|
|
|
+ or %ecx,%r9d # merge AMD XOP flag
|
|
|
|
|
|
+
|
|
|
|
|
|
+ mov %edx,%r10d # %r9d:%r10d is copy of %ecx:%edx
|
|
|
|
|
|
+ bt \$27,%r9d # check OSXSAVE bit
|
|
|
|
|
|
+ jnc .Lclear_avx
|
|
|
|
|
|
+ xor %ecx,%ecx # XCR0
|
|
|
|
|
|
+ .byte 0x0f,0x01,0xd0 # xgetbv
|
|
|
|
|
|
+ and \$6,%eax # isolate XMM and YMM state support
|
|
|
|
|
|
+ cmp \$6,%eax
|
|
|
|
|
|
+ je .Ldone
|
|
|
|
|
|
+.Lclear_avx:
|
|
|
|
|
|
+ mov \$0xefffe7ff,%eax # ~(1<<28|1<<12|1<<11)
|
|
|
|
|
|
+ and %eax,%r9d # clear AVX, FMA and AMD XOP bits
|
|
|
|
|
|
.Ldone:
|
|
|
|
|
|
- shl \$32,%rcx
|
|
|
|
|
|
- mov %edx,%eax
|
|
|
|
|
|
- mov %r8,%rbx
|
|
|
|
|
|
- or %rcx,%rax
|
|
|
|
|
|
+ shl \$32,%r9
|
|
|
|
|
|
+ mov %r10d,%eax
|
|
|
|
|
|
+ mov %r8,%rbx # restore %rbx
|
|
|
|
|
|
+ or %r9,%rax
|
|
|
|
|
|
ret
|
|
|
|
|
|
.size OPENSSL_ia32_cpuid,.-OPENSSL_ia32_cpuid
|
|
|
|
|
|
|
