// Code generated by command: go run chacha20poly1305_amd64_asm.go -out ../chacha20poly1305_amd64.s -pkg chacha20poly1305. DO NOT EDIT. //go:build gc && !purego #include "textflag.h" // func polyHashADInternal<>() TEXT polyHashADInternal<>(SB), NOSPLIT, $0 XORQ R10, R10 XORQ R11, R11 XORQ R12, R12 CMPQ R9, $0x0d JNE hashADLoop MOVQ (CX), R10 MOVQ 5(CX), R11 SHRQ $0x18, R11 MOVQ $0x00000001, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 RET hashADLoop: // Hash in 16 byte chunks CMPQ R9, $0x10 JB hashADTail ADDQ (CX), R10 ADCQ 8(CX), R11 ADCQ $0x01, R12 LEAQ 16(CX), CX SUBQ $0x10, R9 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 JMP hashADLoop hashADTail: CMPQ R9, $0x00 JE hashADDone // Hash last < 16 byte tail XORQ R13, R13 XORQ R14, R14 XORQ R15, R15 ADDQ R9, CX hashADTailLoop: SHLQ $0x08, R13, R14 SHLQ $0x08, R13 MOVB -1(CX), R15 XORQ R15, R13 DECQ CX DECQ R9 JNE hashADTailLoop ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 hashADDone: RET // func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool // Requires: AVX, AVX2, BMI2, CMOV, SSE2 TEXT ·chacha20Poly1305Open(SB), $288-97 // For aligned stack access MOVQ SP, BP ADDQ $0x20, BP ANDQ $-32, BP MOVQ dst_base+0(FP), DI MOVQ key_base+24(FP), R8 MOVQ src_base+48(FP), SI MOVQ src_len+56(FP), BX MOVQ ad_base+72(FP), CX VZEROUPPER VMOVDQU ·chacha20Constants<>+0(SB), Y0 VBROADCASTI128 16(R8), Y14 VBROADCASTI128 32(R8), Y12 VBROADCASTI128 48(R8), Y4 VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 // Special optimization, for very short buffers CMPQ BX, $0xc0 JBE openAVX2192 CMPQ BX, $0x00000140 JBE openAVX2320 // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream VMOVDQA Y14, 32(BP) VMOVDQA Y12, 64(BP) VMOVDQA Y4, 192(BP) MOVQ $0x0000000a, R9 openAVX2PreparePolyKey: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x0c, Y4, Y4, Y4 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x04, Y4, Y4, Y4 DECQ R9 JNE openAVX2PreparePolyKey VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD 32(BP), Y14, Y14 VPADDD 64(BP), Y12, Y12 VPADDD 192(BP), Y4, Y4 VPERM2I128 $0x02, Y0, Y14, Y3 // Clamp and store poly key VPAND ·polyClampMask<>+0(SB), Y3, Y3 VMOVDQA Y3, (BP) // Stream for the first 64 bytes VPERM2I128 $0x13, Y0, Y14, Y0 VPERM2I128 $0x13, Y12, Y4, Y14 // Hash AD + first 64 bytes MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) XORQ CX, CX openAVX2InitialHash64: ADDQ (SI)(CX*1), R10 ADCQ 8(SI)(CX*1), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 ADDQ $0x10, CX CMPQ CX, $0x40 JNE openAVX2InitialHash64 // Decrypt the first 64 bytes VPXOR (SI), Y0, Y0 VPXOR 32(SI), Y14, Y14 VMOVDQU Y0, (DI) VMOVDQU Y14, 32(DI) LEAQ 64(SI), SI LEAQ 64(DI), DI SUBQ $0x40, BX openAVX2MainLoop: CMPQ BX, $0x00000200 JB openAVX2MainLoopDone // Load state, increment counter blocks, store the incremented counters VMOVDQU ·chacha20Constants<>+0(SB), Y0 VMOVDQA Y0, Y5 VMOVDQA Y0, Y6 VMOVDQA Y0, Y7 VMOVDQA 32(BP), Y14 VMOVDQA Y14, Y9 VMOVDQA Y14, Y10 VMOVDQA Y14, Y11 VMOVDQA 64(BP), Y12 VMOVDQA Y12, Y13 VMOVDQA Y12, Y8 VMOVDQA Y12, Y15 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 VMOVDQA Y4, 96(BP) VMOVDQA Y1, 128(BP) VMOVDQA Y2, 160(BP) VMOVDQA Y3, 192(BP) XORQ CX, CX openAVX2InternalLoop: ADDQ (SI)(CX*1), R10 ADCQ 8(SI)(CX*1), R11 ADCQ $0x01, R12 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y3, Y3 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 VMOVDQA Y15, 224(BP) VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x0c, Y11, Y15 VPSRLD $0x14, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y3, Y3 ADDQ 16(SI)(CX*1), R10 ADCQ 24(SI)(CX*1), R11 ADCQ $0x01, R12 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x07, Y11, Y15 VPSRLD $0x19, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x04, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x0c, Y2, Y2, Y2 VPALIGNR $0x0c, Y3, Y3, Y3 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y3, Y3 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 ADDQ 32(SI)(CX*1), R10 ADCQ 40(SI)(CX*1), R11 ADCQ $0x01, R12 LEAQ 48(CX), CX VMOVDQA Y15, 224(BP) VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x0c, Y11, Y15 VPSRLD $0x14, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x07, Y11, Y15 VPSRLD $0x19, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x0c, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x04, Y2, Y2, Y2 VPALIGNR $0x04, Y3, Y3, Y3 CMPQ CX, $0x000001e0 JNE openAVX2InternalLoop VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 VPADDD 32(BP), Y10, Y10 VPADDD 32(BP), Y11, Y11 VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 VPADDD 64(BP), Y8, Y8 VPADDD 64(BP), Y15, Y15 VPADDD 96(BP), Y4, Y4 VPADDD 128(BP), Y1, Y1 VPADDD 160(BP), Y2, Y2 VPADDD 192(BP), Y3, Y3 VMOVDQA Y15, 224(BP) // We only hashed 480 of the 512 bytes available - hash the remaining 32 here ADDQ 480(SI), R10 ADCQ 488(SI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPERM2I128 $0x02, Y0, Y14, Y15 VPERM2I128 $0x13, Y0, Y14, Y14 VPERM2I128 $0x02, Y12, Y4, Y0 VPERM2I128 $0x13, Y12, Y4, Y12 VPXOR (SI), Y15, Y15 VPXOR 32(SI), Y0, Y0 VPXOR 64(SI), Y14, Y14 VPXOR 96(SI), Y12, Y12 VMOVDQU Y15, (DI) VMOVDQU Y0, 32(DI) VMOVDQU Y14, 64(DI) VMOVDQU Y12, 96(DI) VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 VPXOR 128(SI), Y0, Y0 VPXOR 160(SI), Y14, Y14 VPXOR 192(SI), Y12, Y12 VPXOR 224(SI), Y4, Y4 VMOVDQU Y0, 128(DI) VMOVDQU Y14, 160(DI) VMOVDQU Y12, 192(DI) VMOVDQU Y4, 224(DI) // and here ADDQ 496(SI), R10 ADCQ 504(SI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPERM2I128 $0x02, Y6, Y10, Y0 VPERM2I128 $0x02, Y8, Y2, Y14 VPERM2I128 $0x13, Y6, Y10, Y12 VPERM2I128 $0x13, Y8, Y2, Y4 VPXOR 256(SI), Y0, Y0 VPXOR 288(SI), Y14, Y14 VPXOR 320(SI), Y12, Y12 VPXOR 352(SI), Y4, Y4 VMOVDQU Y0, 256(DI) VMOVDQU Y14, 288(DI) VMOVDQU Y12, 320(DI) VMOVDQU Y4, 352(DI) VPERM2I128 $0x02, Y7, Y11, Y0 VPERM2I128 $0x02, 224(BP), Y3, Y14 VPERM2I128 $0x13, Y7, Y11, Y12 VPERM2I128 $0x13, 224(BP), Y3, Y4 VPXOR 384(SI), Y0, Y0 VPXOR 416(SI), Y14, Y14 VPXOR 448(SI), Y12, Y12 VPXOR 480(SI), Y4, Y4 VMOVDQU Y0, 384(DI) VMOVDQU Y14, 416(DI) VMOVDQU Y12, 448(DI) VMOVDQU Y4, 480(DI) LEAQ 512(SI), SI LEAQ 512(DI), DI SUBQ $0x00000200, BX JMP openAVX2MainLoop openAVX2MainLoopDone: // Handle the various tail sizes efficiently TESTQ BX, BX JE openSSEFinalize CMPQ BX, $0x80 JBE openAVX2Tail128 CMPQ BX, $0x00000100 JBE openAVX2Tail256 CMPQ BX, $0x00000180 JBE openAVX2Tail384 JMP openAVX2Tail512 openSSEFinalize: // Hash in the PT, AAD lengths ADDQ ad_len+80(FP), R10 ADCQ src_len+56(FP), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 // Final reduce MOVQ R10, R13 MOVQ R11, R14 MOVQ R12, R15 SUBQ $-5, R10 SBBQ $-1, R11 SBBQ $0x03, R12 CMOVQCS R13, R10 CMOVQCS R14, R11 CMOVQCS R15, R12 // Add in the "s" part of the key ADDQ 16(BP), R10 ADCQ 24(BP), R11 // Finally, constant time compare to the tag at the end of the message XORQ AX, AX MOVQ $0x00000001, DX XORQ (SI), R10 XORQ 8(SI), R11 ORQ R11, R10 CMOVQEQ DX, AX // Return true iff tags are equal MOVB AX, ret+96(FP) RET openSSETail16: TESTQ BX, BX JE openSSEFinalize // We can safely load the CT from the end, because it is padded with the MAC MOVQ BX, R9 SHLQ $0x04, R9 LEAQ ·andMask<>+0(SB), R13 MOVOU (SI), X12 ADDQ BX, SI PAND -16(R13)(R9*1), X12 MOVO X12, 64(BP) MOVQ X12, R13 MOVQ 72(BP), R14 PXOR X1, X12 // We can only store one byte at a time, since plaintext can be shorter than 16 bytes openSSETail16Store: MOVQ X12, R8 MOVB R8, (DI) PSRLDQ $0x01, X12 INCQ DI DECQ BX JNE openSSETail16Store ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 JMP openSSEFinalize openAVX2192: VMOVDQA Y0, Y5 VMOVDQA Y14, Y9 VMOVDQA Y12, Y13 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VMOVDQA Y0, Y6 VMOVDQA Y14, Y10 VMOVDQA Y12, Y8 VMOVDQA Y4, Y2 VMOVDQA Y1, Y15 MOVQ $0x0000000a, R9 openAVX2192InnerCipherLoop: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 DECQ R9 JNE openAVX2192InnerCipherLoop VPADDD Y6, Y0, Y0 VPADDD Y6, Y5, Y5 VPADDD Y10, Y14, Y14 VPADDD Y10, Y9, Y9 VPADDD Y8, Y12, Y12 VPADDD Y8, Y13, Y13 VPADDD Y2, Y4, Y4 VPADDD Y15, Y1, Y1 VPERM2I128 $0x02, Y0, Y14, Y3 // Clamp and store poly key VPAND ·polyClampMask<>+0(SB), Y3, Y3 VMOVDQA Y3, (BP) // Stream for up to 192 bytes VPERM2I128 $0x13, Y0, Y14, Y0 VPERM2I128 $0x13, Y12, Y4, Y14 VPERM2I128 $0x02, Y5, Y9, Y12 VPERM2I128 $0x02, Y13, Y1, Y4 VPERM2I128 $0x13, Y5, Y9, Y5 VPERM2I128 $0x13, Y13, Y1, Y9 openAVX2ShortOpen: // Hash MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) openAVX2ShortOpenLoop: CMPQ BX, $0x20 JB openAVX2ShortTail32 SUBQ $0x20, BX // Load for hashing ADDQ (SI), R10 ADCQ 8(SI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 ADDQ 16(SI), R10 ADCQ 24(SI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 // Load for decryption VPXOR (SI), Y0, Y0 VMOVDQU Y0, (DI) LEAQ 32(SI), SI LEAQ 32(DI), DI // Shift stream left VMOVDQA Y14, Y0 VMOVDQA Y12, Y14 VMOVDQA Y4, Y12 VMOVDQA Y5, Y4 VMOVDQA Y9, Y5 VMOVDQA Y13, Y9 VMOVDQA Y1, Y13 VMOVDQA Y6, Y1 VMOVDQA Y10, Y6 JMP openAVX2ShortOpenLoop openAVX2ShortTail32: CMPQ BX, $0x10 VMOVDQA X0, X1 JB openAVX2ShortDone SUBQ $0x10, BX // Load for hashing ADDQ (SI), R10 ADCQ 8(SI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 // Load for decryption VPXOR (SI), X0, X12 VMOVDQU X12, (DI) LEAQ 16(SI), SI LEAQ 16(DI), DI VPERM2I128 $0x11, Y0, Y0, Y0 VMOVDQA X0, X1 openAVX2ShortDone: VZEROUPPER JMP openSSETail16 openAVX2320: VMOVDQA Y0, Y5 VMOVDQA Y14, Y9 VMOVDQA Y12, Y13 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VMOVDQA Y0, Y6 VMOVDQA Y14, Y10 VMOVDQA Y12, Y8 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VMOVDQA Y14, Y7 VMOVDQA Y12, Y11 VMOVDQA Y4, Y15 MOVQ $0x0000000a, R9 openAVX2320InnerCipherLoop: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y3 VPSRLD $0x14, Y10, Y10 VPXOR Y3, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 VPXOR Y3, Y10, Y10 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x0c, Y2, Y2, Y2 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y3 VPSRLD $0x14, Y10, Y10 VPXOR Y3, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 VPXOR Y3, Y10, Y10 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x04, Y2, Y2, Y2 DECQ R9 JNE openAVX2320InnerCipherLoop VMOVDQA ·chacha20Constants<>+0(SB), Y3 VPADDD Y3, Y0, Y0 VPADDD Y3, Y5, Y5 VPADDD Y3, Y6, Y6 VPADDD Y7, Y14, Y14 VPADDD Y7, Y9, Y9 VPADDD Y7, Y10, Y10 VPADDD Y11, Y12, Y12 VPADDD Y11, Y13, Y13 VPADDD Y11, Y8, Y8 VMOVDQA ·avx2IncMask<>+0(SB), Y3 VPADDD Y15, Y4, Y4 VPADDD Y3, Y15, Y15 VPADDD Y15, Y1, Y1 VPADDD Y3, Y15, Y15 VPADDD Y15, Y2, Y2 // Clamp and store poly key VPERM2I128 $0x02, Y0, Y14, Y3 VPAND ·polyClampMask<>+0(SB), Y3, Y3 VMOVDQA Y3, (BP) // Stream for up to 320 bytes VPERM2I128 $0x13, Y0, Y14, Y0 VPERM2I128 $0x13, Y12, Y4, Y14 VPERM2I128 $0x02, Y5, Y9, Y12 VPERM2I128 $0x02, Y13, Y1, Y4 VPERM2I128 $0x13, Y5, Y9, Y5 VPERM2I128 $0x13, Y13, Y1, Y9 VPERM2I128 $0x02, Y6, Y10, Y13 VPERM2I128 $0x02, Y8, Y2, Y1 VPERM2I128 $0x13, Y6, Y10, Y6 VPERM2I128 $0x13, Y8, Y2, Y10 JMP openAVX2ShortOpen openAVX2Tail128: // Need to decrypt up to 128 bytes - prepare two blocks VMOVDQA ·chacha20Constants<>+0(SB), Y5 VMOVDQA 32(BP), Y9 VMOVDQA 64(BP), Y13 VMOVDQA 192(BP), Y1 VPADDD ·avx2IncMask<>+0(SB), Y1, Y1 VMOVDQA Y1, Y4 XORQ R9, R9 MOVQ BX, CX ANDQ $-16, CX TESTQ CX, CX JE openAVX2Tail128LoopB openAVX2Tail128LoopA: ADDQ (SI)(R9*1), R10 ADCQ 8(SI)(R9*1), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 openAVX2Tail128LoopB: ADDQ $0x10, R9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x0c, Y1, Y1, Y1 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x04, Y1, Y1, Y1 CMPQ R9, CX JB openAVX2Tail128LoopA CMPQ R9, $0xa0 JNE openAVX2Tail128LoopB VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD 32(BP), Y9, Y9 VPADDD 64(BP), Y13, Y13 VPADDD Y4, Y1, Y1 VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 openAVX2TailLoop: CMPQ BX, $0x20 JB openAVX2Tail SUBQ $0x20, BX // Load for decryption VPXOR (SI), Y0, Y0 VMOVDQU Y0, (DI) LEAQ 32(SI), SI LEAQ 32(DI), DI VMOVDQA Y14, Y0 VMOVDQA Y12, Y14 VMOVDQA Y4, Y12 JMP openAVX2TailLoop openAVX2Tail: CMPQ BX, $0x10 VMOVDQA X0, X1 JB openAVX2TailDone SUBQ $0x10, BX // Load for decryption VPXOR (SI), X0, X12 VMOVDQU X12, (DI) LEAQ 16(SI), SI LEAQ 16(DI), DI VPERM2I128 $0x11, Y0, Y0, Y0 VMOVDQA X0, X1 openAVX2TailDone: VZEROUPPER JMP openSSETail16 openAVX2Tail256: VMOVDQA ·chacha20Constants<>+0(SB), Y0 VMOVDQA Y0, Y5 VMOVDQA 32(BP), Y14 VMOVDQA Y14, Y9 VMOVDQA 64(BP), Y12 VMOVDQA Y12, Y13 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VMOVDQA Y4, Y7 VMOVDQA Y1, Y11 // Compute the number of iterations that will hash data MOVQ BX, 224(BP) MOVQ BX, CX SUBQ $0x80, CX SHRQ $0x04, CX MOVQ $0x0000000a, R9 CMPQ CX, $0x0a CMOVQGT R9, CX MOVQ SI, BX XORQ R9, R9 openAVX2Tail256LoopA: ADDQ (BX), R10 ADCQ 8(BX), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(BX), BX openAVX2Tail256LoopB: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 INCQ R9 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 CMPQ R9, CX JB openAVX2Tail256LoopA CMPQ R9, $0x0a JNE openAVX2Tail256LoopB MOVQ BX, R9 SUBQ SI, BX MOVQ BX, CX MOVQ 224(BP), BX openAVX2Tail256Hash: ADDQ $0x10, CX CMPQ CX, BX JGT openAVX2Tail256HashEnd ADDQ (R9), R10 ADCQ 8(R9), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(R9), R9 JMP openAVX2Tail256Hash openAVX2Tail256HashEnd: VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 VPADDD Y7, Y4, Y4 VPADDD Y11, Y1, Y1 VPERM2I128 $0x02, Y0, Y14, Y6 VPERM2I128 $0x02, Y12, Y4, Y10 VPERM2I128 $0x13, Y0, Y14, Y8 VPERM2I128 $0x13, Y12, Y4, Y2 VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 VPXOR (SI), Y6, Y6 VPXOR 32(SI), Y10, Y10 VPXOR 64(SI), Y8, Y8 VPXOR 96(SI), Y2, Y2 VMOVDQU Y6, (DI) VMOVDQU Y10, 32(DI) VMOVDQU Y8, 64(DI) VMOVDQU Y2, 96(DI) LEAQ 128(SI), SI LEAQ 128(DI), DI SUBQ $0x80, BX JMP openAVX2TailLoop openAVX2Tail384: // Need to decrypt up to 384 bytes - prepare six blocks VMOVDQA ·chacha20Constants<>+0(SB), Y0 VMOVDQA Y0, Y5 VMOVDQA Y0, Y6 VMOVDQA 32(BP), Y14 VMOVDQA Y14, Y9 VMOVDQA Y14, Y10 VMOVDQA 64(BP), Y12 VMOVDQA Y12, Y13 VMOVDQA Y12, Y8 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VMOVDQA Y4, 96(BP) VMOVDQA Y1, 128(BP) VMOVDQA Y2, 160(BP) // Compute the number of iterations that will hash two blocks of data MOVQ BX, 224(BP) MOVQ BX, CX SUBQ $0x00000100, CX SHRQ $0x04, CX ADDQ $0x06, CX MOVQ $0x0000000a, R9 CMPQ CX, $0x0a CMOVQGT R9, CX MOVQ SI, BX XORQ R9, R9 openAVX2Tail384LoopB: ADDQ (BX), R10 ADCQ 8(BX), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(BX), BX openAVX2Tail384LoopA: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y3 VPSRLD $0x14, Y10, Y10 VPXOR Y3, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 VPXOR Y3, Y10, Y10 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x0c, Y2, Y2, Y2 ADDQ (BX), R10 ADCQ 8(BX), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(BX), BX INCQ R9 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y3 VPSRLD $0x14, Y10, Y10 VPXOR Y3, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 VPXOR Y3, Y10, Y10 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x04, Y2, Y2, Y2 CMPQ R9, CX JB openAVX2Tail384LoopB CMPQ R9, $0x0a JNE openAVX2Tail384LoopA MOVQ BX, R9 SUBQ SI, BX MOVQ BX, CX MOVQ 224(BP), BX openAVX2Tail384Hash: ADDQ $0x10, CX CMPQ CX, BX JGT openAVX2Tail384HashEnd ADDQ (R9), R10 ADCQ 8(R9), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(R9), R9 JMP openAVX2Tail384Hash openAVX2Tail384HashEnd: VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 VPADDD 32(BP), Y10, Y10 VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 VPADDD 64(BP), Y8, Y8 VPADDD 96(BP), Y4, Y4 VPADDD 128(BP), Y1, Y1 VPADDD 160(BP), Y2, Y2 VPERM2I128 $0x02, Y0, Y14, Y3 VPERM2I128 $0x02, Y12, Y4, Y7 VPERM2I128 $0x13, Y0, Y14, Y11 VPERM2I128 $0x13, Y12, Y4, Y15 VPXOR (SI), Y3, Y3 VPXOR 32(SI), Y7, Y7 VPXOR 64(SI), Y11, Y11 VPXOR 96(SI), Y15, Y15 VMOVDQU Y3, (DI) VMOVDQU Y7, 32(DI) VMOVDQU Y11, 64(DI) VMOVDQU Y15, 96(DI) VPERM2I128 $0x02, Y5, Y9, Y3 VPERM2I128 $0x02, Y13, Y1, Y7 VPERM2I128 $0x13, Y5, Y9, Y11 VPERM2I128 $0x13, Y13, Y1, Y15 VPXOR 128(SI), Y3, Y3 VPXOR 160(SI), Y7, Y7 VPXOR 192(SI), Y11, Y11 VPXOR 224(SI), Y15, Y15 VMOVDQU Y3, 128(DI) VMOVDQU Y7, 160(DI) VMOVDQU Y11, 192(DI) VMOVDQU Y15, 224(DI) VPERM2I128 $0x02, Y6, Y10, Y0 VPERM2I128 $0x02, Y8, Y2, Y14 VPERM2I128 $0x13, Y6, Y10, Y12 VPERM2I128 $0x13, Y8, Y2, Y4 LEAQ 256(SI), SI LEAQ 256(DI), DI SUBQ $0x00000100, BX JMP openAVX2TailLoop openAVX2Tail512: VMOVDQU ·chacha20Constants<>+0(SB), Y0 VMOVDQA Y0, Y5 VMOVDQA Y0, Y6 VMOVDQA Y0, Y7 VMOVDQA 32(BP), Y14 VMOVDQA Y14, Y9 VMOVDQA Y14, Y10 VMOVDQA Y14, Y11 VMOVDQA 64(BP), Y12 VMOVDQA Y12, Y13 VMOVDQA Y12, Y8 VMOVDQA Y12, Y15 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 VMOVDQA Y4, 96(BP) VMOVDQA Y1, 128(BP) VMOVDQA Y2, 160(BP) VMOVDQA Y3, 192(BP) XORQ CX, CX MOVQ SI, R9 openAVX2Tail512LoopB: ADDQ (R9), R10 ADCQ 8(R9), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(R9), R9 openAVX2Tail512LoopA: VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x0c, Y11, Y15 VPSRLD $0x14, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 ADDQ (R9), R10 ADCQ 8(R9), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x07, Y11, Y15 VPSRLD $0x19, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x04, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x0c, Y2, Y2, Y2 VPALIGNR $0x0c, Y3, Y3, Y3 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 ADDQ 16(R9), R10 ADCQ 24(R9), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 32(R9), R9 VMOVDQA Y15, 224(BP) VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x0c, Y11, Y15 VPSRLD $0x14, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x07, Y11, Y15 VPSRLD $0x19, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x0c, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x04, Y2, Y2, Y2 VPALIGNR $0x04, Y3, Y3, Y3 INCQ CX CMPQ CX, $0x04 JLT openAVX2Tail512LoopB CMPQ CX, $0x0a JNE openAVX2Tail512LoopA MOVQ BX, CX SUBQ $0x00000180, CX ANDQ $-16, CX openAVX2Tail512HashLoop: TESTQ CX, CX JE openAVX2Tail512HashEnd ADDQ (R9), R10 ADCQ 8(R9), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(R9), R9 SUBQ $0x10, CX JMP openAVX2Tail512HashLoop openAVX2Tail512HashEnd: VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 VPADDD 32(BP), Y10, Y10 VPADDD 32(BP), Y11, Y11 VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 VPADDD 64(BP), Y8, Y8 VPADDD 64(BP), Y15, Y15 VPADDD 96(BP), Y4, Y4 VPADDD 128(BP), Y1, Y1 VPADDD 160(BP), Y2, Y2 VPADDD 192(BP), Y3, Y3 VMOVDQA Y15, 224(BP) VPERM2I128 $0x02, Y0, Y14, Y15 VPERM2I128 $0x13, Y0, Y14, Y14 VPERM2I128 $0x02, Y12, Y4, Y0 VPERM2I128 $0x13, Y12, Y4, Y12 VPXOR (SI), Y15, Y15 VPXOR 32(SI), Y0, Y0 VPXOR 64(SI), Y14, Y14 VPXOR 96(SI), Y12, Y12 VMOVDQU Y15, (DI) VMOVDQU Y0, 32(DI) VMOVDQU Y14, 64(DI) VMOVDQU Y12, 96(DI) VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 VPXOR 128(SI), Y0, Y0 VPXOR 160(SI), Y14, Y14 VPXOR 192(SI), Y12, Y12 VPXOR 224(SI), Y4, Y4 VMOVDQU Y0, 128(DI) VMOVDQU Y14, 160(DI) VMOVDQU Y12, 192(DI) VMOVDQU Y4, 224(DI) VPERM2I128 $0x02, Y6, Y10, Y0 VPERM2I128 $0x02, Y8, Y2, Y14 VPERM2I128 $0x13, Y6, Y10, Y12 VPERM2I128 $0x13, Y8, Y2, Y4 VPXOR 256(SI), Y0, Y0 VPXOR 288(SI), Y14, Y14 VPXOR 320(SI), Y12, Y12 VPXOR 352(SI), Y4, Y4 VMOVDQU Y0, 256(DI) VMOVDQU Y14, 288(DI) VMOVDQU Y12, 320(DI) VMOVDQU Y4, 352(DI) VPERM2I128 $0x02, Y7, Y11, Y0 VPERM2I128 $0x02, 224(BP), Y3, Y14 VPERM2I128 $0x13, Y7, Y11, Y12 VPERM2I128 $0x13, 224(BP), Y3, Y4 LEAQ 384(SI), SI LEAQ 384(DI), DI SUBQ $0x00000180, BX JMP openAVX2TailLoop DATA ·chacha20Constants<>+0(SB)/4, $0x61707865 DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32 DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574 DATA ·chacha20Constants<>+16(SB)/4, $0x61707865 DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32 DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574 GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32 DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000 DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000 DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001 DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000 GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32 DATA ·rol16<>+0(SB)/8, $0x0504070601000302 DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a DATA ·rol16<>+16(SB)/8, $0x0504070601000302 DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a GLOBL ·rol16<>(SB), RODATA|NOPTR, $32 DATA ·rol8<>+0(SB)/8, $0x0605040702010003 DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b DATA ·rol8<>+16(SB)/8, $0x0605040702010003 DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b GLOBL ·rol8<>(SB), RODATA|NOPTR, $32 DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32 DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002 DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000 DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002 DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000 GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32 DATA ·andMask<>+0(SB)/8, $0x00000000000000ff DATA ·andMask<>+8(SB)/8, $0x0000000000000000 DATA ·andMask<>+16(SB)/8, $0x000000000000ffff DATA ·andMask<>+24(SB)/8, $0x0000000000000000 DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff DATA ·andMask<>+40(SB)/8, $0x0000000000000000 DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff DATA ·andMask<>+56(SB)/8, $0x0000000000000000 DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff DATA ·andMask<>+72(SB)/8, $0x0000000000000000 DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff DATA ·andMask<>+88(SB)/8, $0x0000000000000000 DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff DATA ·andMask<>+104(SB)/8, $0x0000000000000000 DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff DATA ·andMask<>+120(SB)/8, $0x0000000000000000 DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff DATA ·andMask<>+136(SB)/8, $0x00000000000000ff DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff DATA ·andMask<>+152(SB)/8, $0x000000000000ffff DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff GLOBL ·andMask<>(SB), RODATA|NOPTR, $240 // func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte) // Requires: AVX, AVX2, BMI2, CMOV, SSE2 TEXT ·chacha20Poly1305Seal(SB), $288-96 MOVQ SP, BP ADDQ $0x20, BP ANDQ $-32, BP MOVQ dst_base+0(FP), DI MOVQ key_base+24(FP), R8 MOVQ src_base+48(FP), SI MOVQ src_len+56(FP), BX MOVQ ad_base+72(FP), CX VZEROUPPER VMOVDQU ·chacha20Constants<>+0(SB), Y0 VBROADCASTI128 16(R8), Y14 VBROADCASTI128 32(R8), Y12 VBROADCASTI128 48(R8), Y4 VPADDD ·avx2InitMask<>+0(SB), Y4, Y4 // Special optimizations, for very short buffers CMPQ BX, $0x000000c0 JBE seal192AVX2 CMPQ BX, $0x00000140 JBE seal320AVX2 // For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream VMOVDQA Y0, Y5 VMOVDQA Y0, Y6 VMOVDQA Y0, Y7 VMOVDQA Y14, Y9 VMOVDQA Y14, Y10 VMOVDQA Y14, Y11 VMOVDQA Y14, 32(BP) VMOVDQA Y12, Y13 VMOVDQA Y12, Y8 VMOVDQA Y12, Y15 VMOVDQA Y12, 64(BP) VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VMOVDQA Y4, 96(BP) VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VMOVDQA Y1, 128(BP) VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 VMOVDQA Y2, 160(BP) VMOVDQA Y3, 192(BP) MOVQ $0x0000000a, R9 sealAVX2IntroLoop: VMOVDQA Y15, 224(BP) VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VMOVDQA 224(BP), Y15 VMOVDQA Y13, 224(BP) VPADDD Y11, Y7, Y7 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y3, Y15, Y15 VPXOR Y15, Y11, Y11 VPSLLD $0x0c, Y11, Y13 VPSRLD $0x14, Y11, Y11 VPXOR Y13, Y11, Y11 VPADDD Y11, Y7, Y7 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y3, Y15, Y15 VPXOR Y15, Y11, Y11 VPSLLD $0x07, Y11, Y13 VPSRLD $0x19, Y11, Y11 VPXOR Y13, Y11, Y11 VMOVDQA 224(BP), Y13 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x0c, Y2, Y2, Y2 VPALIGNR $0x04, Y11, Y11, Y11 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x0c, Y3, Y3, Y3 VMOVDQA Y15, 224(BP) VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VMOVDQA 224(BP), Y15 VMOVDQA Y13, 224(BP) VPADDD Y11, Y7, Y7 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y3, Y15, Y15 VPXOR Y15, Y11, Y11 VPSLLD $0x0c, Y11, Y13 VPSRLD $0x14, Y11, Y11 VPXOR Y13, Y11, Y11 VPADDD Y11, Y7, Y7 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y3, Y15, Y15 VPXOR Y15, Y11, Y11 VPSLLD $0x07, Y11, Y13 VPSRLD $0x19, Y11, Y11 VPXOR Y13, Y11, Y11 VMOVDQA 224(BP), Y13 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x04, Y2, Y2, Y2 VPALIGNR $0x0c, Y11, Y11, Y11 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x04, Y3, Y3, Y3 DECQ R9 JNE sealAVX2IntroLoop VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 VPADDD 32(BP), Y10, Y10 VPADDD 32(BP), Y11, Y11 VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 VPADDD 64(BP), Y8, Y8 VPADDD 64(BP), Y15, Y15 VPADDD 96(BP), Y4, Y4 VPADDD 128(BP), Y1, Y1 VPADDD 160(BP), Y2, Y2 VPADDD 192(BP), Y3, Y3 VPERM2I128 $0x13, Y12, Y4, Y12 VPERM2I128 $0x02, Y0, Y14, Y4 VPERM2I128 $0x13, Y0, Y14, Y0 // Clamp and store poly key VPAND ·polyClampMask<>+0(SB), Y4, Y4 VMOVDQA Y4, (BP) // Hash AD MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) // Can store at least 320 bytes VPXOR (SI), Y0, Y0 VPXOR 32(SI), Y12, Y12 VMOVDQU Y0, (DI) VMOVDQU Y12, 32(DI) VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 VPXOR 64(SI), Y0, Y0 VPXOR 96(SI), Y14, Y14 VPXOR 128(SI), Y12, Y12 VPXOR 160(SI), Y4, Y4 VMOVDQU Y0, 64(DI) VMOVDQU Y14, 96(DI) VMOVDQU Y12, 128(DI) VMOVDQU Y4, 160(DI) VPERM2I128 $0x02, Y6, Y10, Y0 VPERM2I128 $0x02, Y8, Y2, Y14 VPERM2I128 $0x13, Y6, Y10, Y12 VPERM2I128 $0x13, Y8, Y2, Y4 VPXOR 192(SI), Y0, Y0 VPXOR 224(SI), Y14, Y14 VPXOR 256(SI), Y12, Y12 VPXOR 288(SI), Y4, Y4 VMOVDQU Y0, 192(DI) VMOVDQU Y14, 224(DI) VMOVDQU Y12, 256(DI) VMOVDQU Y4, 288(DI) MOVQ $0x00000140, CX SUBQ $0x00000140, BX LEAQ 320(SI), SI VPERM2I128 $0x02, Y7, Y11, Y0 VPERM2I128 $0x02, Y15, Y3, Y14 VPERM2I128 $0x13, Y7, Y11, Y12 VPERM2I128 $0x13, Y15, Y3, Y4 CMPQ BX, $0x80 JBE sealAVX2SealHash VPXOR (SI), Y0, Y0 VPXOR 32(SI), Y14, Y14 VPXOR 64(SI), Y12, Y12 VPXOR 96(SI), Y4, Y4 VMOVDQU Y0, 320(DI) VMOVDQU Y14, 352(DI) VMOVDQU Y12, 384(DI) VMOVDQU Y4, 416(DI) SUBQ $0x80, BX LEAQ 128(SI), SI MOVQ $0x00000008, CX MOVQ $0x00000002, R9 CMPQ BX, $0x80 JBE sealAVX2Tail128 CMPQ BX, $0x00000100 JBE sealAVX2Tail256 CMPQ BX, $0x00000180 JBE sealAVX2Tail384 CMPQ BX, $0x00000200 JBE sealAVX2Tail512 // We have 448 bytes to hash, but main loop hashes 512 bytes at a time - perform some rounds, before the main loop VMOVDQA ·chacha20Constants<>+0(SB), Y0 VMOVDQA Y0, Y5 VMOVDQA Y0, Y6 VMOVDQA Y0, Y7 VMOVDQA 32(BP), Y14 VMOVDQA Y14, Y9 VMOVDQA Y14, Y10 VMOVDQA Y14, Y11 VMOVDQA 64(BP), Y12 VMOVDQA Y12, Y13 VMOVDQA Y12, Y8 VMOVDQA Y12, Y15 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 VMOVDQA Y4, 96(BP) VMOVDQA Y1, 128(BP) VMOVDQA Y2, 160(BP) VMOVDQA Y3, 192(BP) VMOVDQA Y15, 224(BP) VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VMOVDQA 224(BP), Y15 VMOVDQA Y13, 224(BP) VPADDD Y11, Y7, Y7 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y3, Y15, Y15 VPXOR Y15, Y11, Y11 VPSLLD $0x0c, Y11, Y13 VPSRLD $0x14, Y11, Y11 VPXOR Y13, Y11, Y11 VPADDD Y11, Y7, Y7 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y3, Y15, Y15 VPXOR Y15, Y11, Y11 VPSLLD $0x07, Y11, Y13 VPSRLD $0x19, Y11, Y11 VPXOR Y13, Y11, Y11 VMOVDQA 224(BP), Y13 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x0c, Y2, Y2, Y2 VPALIGNR $0x04, Y11, Y11, Y11 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x0c, Y3, Y3, Y3 VMOVDQA Y15, 224(BP) VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VMOVDQA 224(BP), Y15 VMOVDQA Y13, 224(BP) VPADDD Y11, Y7, Y7 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y3, Y15, Y15 VPXOR Y15, Y11, Y11 VPSLLD $0x0c, Y11, Y13 VPSRLD $0x14, Y11, Y11 VPXOR Y13, Y11, Y11 VPADDD Y11, Y7, Y7 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y3, Y15, Y15 VPXOR Y15, Y11, Y11 VPSLLD $0x07, Y11, Y13 VPSRLD $0x19, Y11, Y11 VPXOR Y13, Y11, Y11 VMOVDQA 224(BP), Y13 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x04, Y2, Y2, Y2 VPALIGNR $0x0c, Y11, Y11, Y11 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x04, Y3, Y3, Y3 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x0c, Y11, Y15 VPSRLD $0x14, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 SUBQ $0x10, DI MOVQ $0x00000009, CX JMP sealAVX2InternalLoopStart sealAVX2MainLoop: VMOVDQU ·chacha20Constants<>+0(SB), Y0 VMOVDQA Y0, Y5 VMOVDQA Y0, Y6 VMOVDQA Y0, Y7 VMOVDQA 32(BP), Y14 VMOVDQA Y14, Y9 VMOVDQA Y14, Y10 VMOVDQA Y14, Y11 VMOVDQA 64(BP), Y12 VMOVDQA Y12, Y13 VMOVDQA Y12, Y8 VMOVDQA Y12, Y15 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 VMOVDQA Y4, 96(BP) VMOVDQA Y1, 128(BP) VMOVDQA Y2, 160(BP) VMOVDQA Y3, 192(BP) MOVQ $0x0000000a, CX sealAVX2InternalLoop: ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y3, Y3 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 VMOVDQA Y15, 224(BP) VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x0c, Y11, Y15 VPSRLD $0x14, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 sealAVX2InternalLoopStart: VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y3, Y3 ADDQ 16(DI), R10 ADCQ 24(DI), R11 ADCQ $0x01, R12 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x07, Y11, Y15 VPSRLD $0x19, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x04, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x0c, Y2, Y2, Y2 VPALIGNR $0x0c, Y3, Y3, Y3 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y3, Y3 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 ADDQ 32(DI), R10 ADCQ 40(DI), R11 ADCQ $0x01, R12 LEAQ 48(DI), DI VMOVDQA Y15, 224(BP) VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x0c, Y11, Y15 VPSRLD $0x14, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x07, Y11, Y15 VPSRLD $0x19, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x0c, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x04, Y2, Y2, Y2 VPALIGNR $0x04, Y3, Y3, Y3 DECQ CX JNE sealAVX2InternalLoop VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 VPADDD 32(BP), Y10, Y10 VPADDD 32(BP), Y11, Y11 VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 VPADDD 64(BP), Y8, Y8 VPADDD 64(BP), Y15, Y15 VPADDD 96(BP), Y4, Y4 VPADDD 128(BP), Y1, Y1 VPADDD 160(BP), Y2, Y2 VPADDD 192(BP), Y3, Y3 VMOVDQA Y15, 224(BP) // We only hashed 480 of the 512 bytes available - hash the remaining 32 here ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 32(DI), DI VPERM2I128 $0x02, Y0, Y14, Y15 VPERM2I128 $0x13, Y0, Y14, Y14 VPERM2I128 $0x02, Y12, Y4, Y0 VPERM2I128 $0x13, Y12, Y4, Y12 VPXOR (SI), Y15, Y15 VPXOR 32(SI), Y0, Y0 VPXOR 64(SI), Y14, Y14 VPXOR 96(SI), Y12, Y12 VMOVDQU Y15, (DI) VMOVDQU Y0, 32(DI) VMOVDQU Y14, 64(DI) VMOVDQU Y12, 96(DI) VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 VPXOR 128(SI), Y0, Y0 VPXOR 160(SI), Y14, Y14 VPXOR 192(SI), Y12, Y12 VPXOR 224(SI), Y4, Y4 VMOVDQU Y0, 128(DI) VMOVDQU Y14, 160(DI) VMOVDQU Y12, 192(DI) VMOVDQU Y4, 224(DI) // and here ADDQ -16(DI), R10 ADCQ -8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPERM2I128 $0x02, Y6, Y10, Y0 VPERM2I128 $0x02, Y8, Y2, Y14 VPERM2I128 $0x13, Y6, Y10, Y12 VPERM2I128 $0x13, Y8, Y2, Y4 VPXOR 256(SI), Y0, Y0 VPXOR 288(SI), Y14, Y14 VPXOR 320(SI), Y12, Y12 VPXOR 352(SI), Y4, Y4 VMOVDQU Y0, 256(DI) VMOVDQU Y14, 288(DI) VMOVDQU Y12, 320(DI) VMOVDQU Y4, 352(DI) VPERM2I128 $0x02, Y7, Y11, Y0 VPERM2I128 $0x02, 224(BP), Y3, Y14 VPERM2I128 $0x13, Y7, Y11, Y12 VPERM2I128 $0x13, 224(BP), Y3, Y4 VPXOR 384(SI), Y0, Y0 VPXOR 416(SI), Y14, Y14 VPXOR 448(SI), Y12, Y12 VPXOR 480(SI), Y4, Y4 VMOVDQU Y0, 384(DI) VMOVDQU Y14, 416(DI) VMOVDQU Y12, 448(DI) VMOVDQU Y4, 480(DI) LEAQ 512(SI), SI SUBQ $0x00000200, BX CMPQ BX, $0x00000200 JG sealAVX2MainLoop // Tail can only hash 480 bytes ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 ADDQ 16(DI), R10 ADCQ 24(DI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 32(DI), DI MOVQ $0x0000000a, CX MOVQ $0x00000000, R9 CMPQ BX, $0x80 JBE sealAVX2Tail128 CMPQ BX, $0x00000100 JBE sealAVX2Tail256 CMPQ BX, $0x00000180 JBE sealAVX2Tail384 JMP sealAVX2Tail512 sealSSETail: TESTQ BX, BX JE sealSSEFinalize // We can only load the PT one byte at a time to avoid read after end of buffer MOVQ BX, R9 SHLQ $0x04, R9 LEAQ ·andMask<>+0(SB), R13 MOVQ BX, CX LEAQ -1(SI)(BX*1), SI XORQ R15, R15 XORQ R8, R8 XORQ AX, AX sealSSETailLoadLoop: SHLQ $0x08, R15, R8 SHLQ $0x08, R15 MOVB (SI), AX XORQ AX, R15 LEAQ -1(SI), SI DECQ CX JNE sealSSETailLoadLoop MOVQ R15, 64(BP) MOVQ R8, 72(BP) PXOR 64(BP), X1 MOVOU X1, (DI) MOVOU -16(R13)(R9*1), X12 PAND X12, X1 MOVQ X1, R13 PSRLDQ $0x08, X1 MOVQ X1, R14 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 ADDQ BX, DI sealSSEFinalize: // Hash in the buffer lengths ADDQ ad_len+80(FP), R10 ADCQ src_len+56(FP), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 // Final reduce MOVQ R10, R13 MOVQ R11, R14 MOVQ R12, R15 SUBQ $-5, R10 SBBQ $-1, R11 SBBQ $0x03, R12 CMOVQCS R13, R10 CMOVQCS R14, R11 CMOVQCS R15, R12 // Add in the "s" part of the key ADDQ 16(BP), R10 ADCQ 24(BP), R11 // Finally store the tag at the end of the message MOVQ R10, (DI) MOVQ R11, 8(DI) RET seal192AVX2: VMOVDQA Y0, Y5 VMOVDQA Y14, Y9 VMOVDQA Y12, Y13 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VMOVDQA Y0, Y6 VMOVDQA Y14, Y10 VMOVDQA Y12, Y8 VMOVDQA Y4, Y2 VMOVDQA Y1, Y15 MOVQ $0x0000000a, R9 sealAVX2192InnerCipherLoop: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 DECQ R9 JNE sealAVX2192InnerCipherLoop VPADDD Y6, Y0, Y0 VPADDD Y6, Y5, Y5 VPADDD Y10, Y14, Y14 VPADDD Y10, Y9, Y9 VPADDD Y8, Y12, Y12 VPADDD Y8, Y13, Y13 VPADDD Y2, Y4, Y4 VPADDD Y15, Y1, Y1 VPERM2I128 $0x02, Y0, Y14, Y3 // Clamp and store poly key VPAND ·polyClampMask<>+0(SB), Y3, Y3 VMOVDQA Y3, (BP) // Stream for up to 192 bytes VPERM2I128 $0x13, Y0, Y14, Y0 VPERM2I128 $0x13, Y12, Y4, Y14 VPERM2I128 $0x02, Y5, Y9, Y12 VPERM2I128 $0x02, Y13, Y1, Y4 VPERM2I128 $0x13, Y5, Y9, Y5 VPERM2I128 $0x13, Y13, Y1, Y9 sealAVX2ShortSeal: // Hash aad MOVQ ad_len+80(FP), R9 CALL polyHashADInternal<>(SB) XORQ CX, CX sealAVX2SealHash: // itr1 holds the number of bytes encrypted but not yet hashed CMPQ CX, $0x10 JB sealAVX2ShortSealLoop ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 SUBQ $0x10, CX ADDQ $0x10, DI JMP sealAVX2SealHash sealAVX2ShortSealLoop: CMPQ BX, $0x20 JB sealAVX2ShortTail32 SUBQ $0x20, BX // Load for encryption VPXOR (SI), Y0, Y0 VMOVDQU Y0, (DI) LEAQ 32(SI), SI // Now can hash ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 ADDQ 16(DI), R10 ADCQ 24(DI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 32(DI), DI // Shift stream left VMOVDQA Y14, Y0 VMOVDQA Y12, Y14 VMOVDQA Y4, Y12 VMOVDQA Y5, Y4 VMOVDQA Y9, Y5 VMOVDQA Y13, Y9 VMOVDQA Y1, Y13 VMOVDQA Y6, Y1 VMOVDQA Y10, Y6 JMP sealAVX2ShortSealLoop sealAVX2ShortTail32: CMPQ BX, $0x10 VMOVDQA X0, X1 JB sealAVX2ShortDone SUBQ $0x10, BX // Load for encryption VPXOR (SI), X0, X12 VMOVDQU X12, (DI) LEAQ 16(SI), SI // Hash ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(DI), DI VPERM2I128 $0x11, Y0, Y0, Y0 VMOVDQA X0, X1 sealAVX2ShortDone: VZEROUPPER JMP sealSSETail seal320AVX2: VMOVDQA Y0, Y5 VMOVDQA Y14, Y9 VMOVDQA Y12, Y13 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VMOVDQA Y0, Y6 VMOVDQA Y14, Y10 VMOVDQA Y12, Y8 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VMOVDQA Y14, Y7 VMOVDQA Y12, Y11 VMOVDQA Y4, Y15 MOVQ $0x0000000a, R9 sealAVX2320InnerCipherLoop: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y3 VPSRLD $0x14, Y10, Y10 VPXOR Y3, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 VPXOR Y3, Y10, Y10 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x0c, Y2, Y2, Y2 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y3 VPSRLD $0x14, Y10, Y10 VPXOR Y3, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 VPXOR Y3, Y10, Y10 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x04, Y2, Y2, Y2 DECQ R9 JNE sealAVX2320InnerCipherLoop VMOVDQA ·chacha20Constants<>+0(SB), Y3 VPADDD Y3, Y0, Y0 VPADDD Y3, Y5, Y5 VPADDD Y3, Y6, Y6 VPADDD Y7, Y14, Y14 VPADDD Y7, Y9, Y9 VPADDD Y7, Y10, Y10 VPADDD Y11, Y12, Y12 VPADDD Y11, Y13, Y13 VPADDD Y11, Y8, Y8 VMOVDQA ·avx2IncMask<>+0(SB), Y3 VPADDD Y15, Y4, Y4 VPADDD Y3, Y15, Y15 VPADDD Y15, Y1, Y1 VPADDD Y3, Y15, Y15 VPADDD Y15, Y2, Y2 // Clamp and store poly key VPERM2I128 $0x02, Y0, Y14, Y3 VPAND ·polyClampMask<>+0(SB), Y3, Y3 VMOVDQA Y3, (BP) // Stream for up to 320 bytes VPERM2I128 $0x13, Y0, Y14, Y0 VPERM2I128 $0x13, Y12, Y4, Y14 VPERM2I128 $0x02, Y5, Y9, Y12 VPERM2I128 $0x02, Y13, Y1, Y4 VPERM2I128 $0x13, Y5, Y9, Y5 VPERM2I128 $0x13, Y13, Y1, Y9 VPERM2I128 $0x02, Y6, Y10, Y13 VPERM2I128 $0x02, Y8, Y2, Y1 VPERM2I128 $0x13, Y6, Y10, Y6 VPERM2I128 $0x13, Y8, Y2, Y10 JMP sealAVX2ShortSeal sealAVX2Tail128: VMOVDQA ·chacha20Constants<>+0(SB), Y0 VMOVDQA 32(BP), Y14 VMOVDQA 64(BP), Y12 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VMOVDQA Y4, Y1 sealAVX2Tail128LoopA: ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(DI), DI sealAVX2Tail128LoopB: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x0c, Y4, Y4, Y4 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 ADDQ 16(DI), R10 ADCQ 24(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 32(DI), DI VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x04, Y4, Y4, Y4 DECQ CX JG sealAVX2Tail128LoopA DECQ R9 JGE sealAVX2Tail128LoopB VPADDD ·chacha20Constants<>+0(SB), Y0, Y5 VPADDD 32(BP), Y14, Y9 VPADDD 64(BP), Y12, Y13 VPADDD Y1, Y4, Y1 VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 JMP sealAVX2ShortSealLoop sealAVX2Tail256: VMOVDQA ·chacha20Constants<>+0(SB), Y0 VMOVDQA ·chacha20Constants<>+0(SB), Y5 VMOVDQA 32(BP), Y14 VMOVDQA 32(BP), Y9 VMOVDQA 64(BP), Y12 VMOVDQA 64(BP), Y13 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VMOVDQA Y4, Y7 VMOVDQA Y1, Y11 sealAVX2Tail256LoopA: ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(DI), DI sealAVX2Tail256LoopB: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 ADDQ 16(DI), R10 ADCQ 24(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 32(DI), DI VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 DECQ CX JG sealAVX2Tail256LoopA DECQ R9 JGE sealAVX2Tail256LoopB VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 VPADDD Y7, Y4, Y4 VPADDD Y11, Y1, Y1 VPERM2I128 $0x02, Y0, Y14, Y3 VPERM2I128 $0x02, Y12, Y4, Y7 VPERM2I128 $0x13, Y0, Y14, Y11 VPERM2I128 $0x13, Y12, Y4, Y15 VPXOR (SI), Y3, Y3 VPXOR 32(SI), Y7, Y7 VPXOR 64(SI), Y11, Y11 VPXOR 96(SI), Y15, Y15 VMOVDQU Y3, (DI) VMOVDQU Y7, 32(DI) VMOVDQU Y11, 64(DI) VMOVDQU Y15, 96(DI) MOVQ $0x00000080, CX LEAQ 128(SI), SI SUBQ $0x80, BX VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 JMP sealAVX2SealHash sealAVX2Tail384: VMOVDQA ·chacha20Constants<>+0(SB), Y0 VMOVDQA Y0, Y5 VMOVDQA Y0, Y6 VMOVDQA 32(BP), Y14 VMOVDQA Y14, Y9 VMOVDQA Y14, Y10 VMOVDQA 64(BP), Y12 VMOVDQA Y12, Y13 VMOVDQA Y12, Y8 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VMOVDQA Y4, Y7 VMOVDQA Y1, Y11 VMOVDQA Y2, Y15 sealAVX2Tail384LoopA: ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(DI), DI sealAVX2Tail384LoopB: VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y3 VPSRLD $0x14, Y10, Y10 VPXOR Y3, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 VPXOR Y3, Y10, Y10 ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x0c, Y2, Y2, Y2 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x0c, Y14, Y3 VPSRLD $0x14, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y14, Y0, Y0 VPXOR Y0, Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPADDD Y4, Y12, Y12 VPXOR Y12, Y14, Y14 VPSLLD $0x07, Y14, Y3 VPSRLD $0x19, Y14, Y14 VPXOR Y3, Y14, Y14 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x0c, Y9, Y3 VPSRLD $0x14, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y9, Y5, Y5 VPXOR Y5, Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPADDD Y1, Y13, Y13 VPXOR Y13, Y9, Y9 VPSLLD $0x07, Y9, Y3 VPSRLD $0x19, Y9, Y9 VPXOR Y3, Y9, Y9 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x0c, Y10, Y3 VPSRLD $0x14, Y10, Y10 VPXOR Y3, Y10, Y10 VPADDD Y10, Y6, Y6 VPXOR Y6, Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPADDD Y2, Y8, Y8 VPXOR Y8, Y10, Y10 VPSLLD $0x07, Y10, Y3 VPSRLD $0x19, Y10, Y10 VPXOR Y3, Y10, Y10 ADDQ 16(DI), R10 ADCQ 24(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 32(DI), DI VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x04, Y2, Y2, Y2 DECQ CX JG sealAVX2Tail384LoopA DECQ R9 JGE sealAVX2Tail384LoopB VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 VPADDD 32(BP), Y10, Y10 VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 VPADDD 64(BP), Y8, Y8 VPADDD Y7, Y4, Y4 VPADDD Y11, Y1, Y1 VPADDD Y15, Y2, Y2 VPERM2I128 $0x02, Y0, Y14, Y3 VPERM2I128 $0x02, Y12, Y4, Y7 VPERM2I128 $0x13, Y0, Y14, Y11 VPERM2I128 $0x13, Y12, Y4, Y15 VPXOR (SI), Y3, Y3 VPXOR 32(SI), Y7, Y7 VPXOR 64(SI), Y11, Y11 VPXOR 96(SI), Y15, Y15 VMOVDQU Y3, (DI) VMOVDQU Y7, 32(DI) VMOVDQU Y11, 64(DI) VMOVDQU Y15, 96(DI) VPERM2I128 $0x02, Y5, Y9, Y3 VPERM2I128 $0x02, Y13, Y1, Y7 VPERM2I128 $0x13, Y5, Y9, Y11 VPERM2I128 $0x13, Y13, Y1, Y15 VPXOR 128(SI), Y3, Y3 VPXOR 160(SI), Y7, Y7 VPXOR 192(SI), Y11, Y11 VPXOR 224(SI), Y15, Y15 VMOVDQU Y3, 128(DI) VMOVDQU Y7, 160(DI) VMOVDQU Y11, 192(DI) VMOVDQU Y15, 224(DI) MOVQ $0x00000100, CX LEAQ 256(SI), SI SUBQ $0x00000100, BX VPERM2I128 $0x02, Y6, Y10, Y0 VPERM2I128 $0x02, Y8, Y2, Y14 VPERM2I128 $0x13, Y6, Y10, Y12 VPERM2I128 $0x13, Y8, Y2, Y4 JMP sealAVX2SealHash sealAVX2Tail512: VMOVDQA ·chacha20Constants<>+0(SB), Y0 VMOVDQA Y0, Y5 VMOVDQA Y0, Y6 VMOVDQA Y0, Y7 VMOVDQA 32(BP), Y14 VMOVDQA Y14, Y9 VMOVDQA Y14, Y10 VMOVDQA Y14, Y11 VMOVDQA 64(BP), Y12 VMOVDQA Y12, Y13 VMOVDQA Y12, Y8 VMOVDQA Y12, Y15 VMOVDQA 192(BP), Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y4 VPADDD ·avx2IncMask<>+0(SB), Y4, Y1 VPADDD ·avx2IncMask<>+0(SB), Y1, Y2 VPADDD ·avx2IncMask<>+0(SB), Y2, Y3 VMOVDQA Y4, 96(BP) VMOVDQA Y1, 128(BP) VMOVDQA Y2, 160(BP) VMOVDQA Y3, 192(BP) sealAVX2Tail512LoopA: ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), AX MOVQ AX, R15 MULQ R10 MOVQ AX, R13 MOVQ DX, R14 MOVQ (BP), AX MULQ R11 IMULQ R12, R15 ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), AX MOVQ AX, R8 MULQ R10 ADDQ AX, R14 ADCQ $0x00, DX MOVQ DX, R10 MOVQ 8(BP), AX MULQ R11 ADDQ AX, R15 ADCQ $0x00, DX IMULQ R12, R8 ADDQ R10, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 16(DI), DI sealAVX2Tail512LoopB: VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x0c, Y11, Y15 VPSRLD $0x14, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 ADDQ (DI), R10 ADCQ 8(DI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x07, Y11, Y15 VPSRLD $0x19, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 VPALIGNR $0x04, Y14, Y14, Y14 VPALIGNR $0x04, Y9, Y9, Y9 VPALIGNR $0x04, Y10, Y10, Y10 VPALIGNR $0x04, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x0c, Y4, Y4, Y4 VPALIGNR $0x0c, Y1, Y1, Y1 VPALIGNR $0x0c, Y2, Y2, Y2 VPALIGNR $0x0c, Y3, Y3, Y3 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol16<>+0(SB), Y4, Y4 VPSHUFB ·rol16<>+0(SB), Y1, Y1 VPSHUFB ·rol16<>+0(SB), Y2, Y2 VPSHUFB ·rol16<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 ADDQ 16(DI), R10 ADCQ 24(DI), R11 ADCQ $0x01, R12 MOVQ (BP), DX MOVQ DX, R15 MULXQ R10, R13, R14 IMULQ R12, R15 MULXQ R11, AX, DX ADDQ AX, R14 ADCQ DX, R15 MOVQ 8(BP), DX MULXQ R10, R10, AX ADDQ R10, R14 MULXQ R11, R11, R8 ADCQ R11, R15 ADCQ $0x00, R8 IMULQ R12, DX ADDQ AX, R15 ADCQ DX, R8 MOVQ R13, R10 MOVQ R14, R11 MOVQ R15, R12 ANDQ $0x03, R12 MOVQ R15, R13 ANDQ $-4, R13 MOVQ R8, R14 SHRQ $0x02, R8, R15 SHRQ $0x02, R8 ADDQ R13, R10 ADCQ R14, R11 ADCQ $0x00, R12 ADDQ R15, R10 ADCQ R8, R11 ADCQ $0x00, R12 LEAQ 32(DI), DI VMOVDQA Y15, 224(BP) VPSLLD $0x0c, Y14, Y15 VPSRLD $0x14, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x0c, Y9, Y15 VPSRLD $0x14, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x0c, Y10, Y15 VPSRLD $0x14, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x0c, Y11, Y15 VPSRLD $0x14, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 VPADDD Y14, Y0, Y0 VPADDD Y9, Y5, Y5 VPADDD Y10, Y6, Y6 VPADDD Y11, Y7, Y7 VPXOR Y0, Y4, Y4 VPXOR Y5, Y1, Y1 VPXOR Y6, Y2, Y2 VPXOR Y7, Y3, Y3 VPSHUFB ·rol8<>+0(SB), Y4, Y4 VPSHUFB ·rol8<>+0(SB), Y1, Y1 VPSHUFB ·rol8<>+0(SB), Y2, Y2 VPSHUFB ·rol8<>+0(SB), Y3, Y3 VPADDD Y4, Y12, Y12 VPADDD Y1, Y13, Y13 VPADDD Y2, Y8, Y8 VPADDD Y3, Y15, Y15 VPXOR Y12, Y14, Y14 VPXOR Y13, Y9, Y9 VPXOR Y8, Y10, Y10 VPXOR Y15, Y11, Y11 VMOVDQA Y15, 224(BP) VPSLLD $0x07, Y14, Y15 VPSRLD $0x19, Y14, Y14 VPXOR Y15, Y14, Y14 VPSLLD $0x07, Y9, Y15 VPSRLD $0x19, Y9, Y9 VPXOR Y15, Y9, Y9 VPSLLD $0x07, Y10, Y15 VPSRLD $0x19, Y10, Y10 VPXOR Y15, Y10, Y10 VPSLLD $0x07, Y11, Y15 VPSRLD $0x19, Y11, Y11 VPXOR Y15, Y11, Y11 VMOVDQA 224(BP), Y15 VPALIGNR $0x0c, Y14, Y14, Y14 VPALIGNR $0x0c, Y9, Y9, Y9 VPALIGNR $0x0c, Y10, Y10, Y10 VPALIGNR $0x0c, Y11, Y11, Y11 VPALIGNR $0x08, Y12, Y12, Y12 VPALIGNR $0x08, Y13, Y13, Y13 VPALIGNR $0x08, Y8, Y8, Y8 VPALIGNR $0x08, Y15, Y15, Y15 VPALIGNR $0x04, Y4, Y4, Y4 VPALIGNR $0x04, Y1, Y1, Y1 VPALIGNR $0x04, Y2, Y2, Y2 VPALIGNR $0x04, Y3, Y3, Y3 DECQ CX JG sealAVX2Tail512LoopA DECQ R9 JGE sealAVX2Tail512LoopB VPADDD ·chacha20Constants<>+0(SB), Y0, Y0 VPADDD ·chacha20Constants<>+0(SB), Y5, Y5 VPADDD ·chacha20Constants<>+0(SB), Y6, Y6 VPADDD ·chacha20Constants<>+0(SB), Y7, Y7 VPADDD 32(BP), Y14, Y14 VPADDD 32(BP), Y9, Y9 VPADDD 32(BP), Y10, Y10 VPADDD 32(BP), Y11, Y11 VPADDD 64(BP), Y12, Y12 VPADDD 64(BP), Y13, Y13 VPADDD 64(BP), Y8, Y8 VPADDD 64(BP), Y15, Y15 VPADDD 96(BP), Y4, Y4 VPADDD 128(BP), Y1, Y1 VPADDD 160(BP), Y2, Y2 VPADDD 192(BP), Y3, Y3 VMOVDQA Y15, 224(BP) VPERM2I128 $0x02, Y0, Y14, Y15 VPXOR (SI), Y15, Y15 VMOVDQU Y15, (DI) VPERM2I128 $0x02, Y12, Y4, Y15 VPXOR 32(SI), Y15, Y15 VMOVDQU Y15, 32(DI) VPERM2I128 $0x13, Y0, Y14, Y15 VPXOR 64(SI), Y15, Y15 VMOVDQU Y15, 64(DI) VPERM2I128 $0x13, Y12, Y4, Y15 VPXOR 96(SI), Y15, Y15 VMOVDQU Y15, 96(DI) VPERM2I128 $0x02, Y5, Y9, Y0 VPERM2I128 $0x02, Y13, Y1, Y14 VPERM2I128 $0x13, Y5, Y9, Y12 VPERM2I128 $0x13, Y13, Y1, Y4 VPXOR 128(SI), Y0, Y0 VPXOR 160(SI), Y14, Y14 VPXOR 192(SI), Y12, Y12 VPXOR 224(SI), Y4, Y4 VMOVDQU Y0, 128(DI) VMOVDQU Y14, 160(DI) VMOVDQU Y12, 192(DI) VMOVDQU Y4, 224(DI) VPERM2I128 $0x02, Y6, Y10, Y0 VPERM2I128 $0x02, Y8, Y2, Y14 VPERM2I128 $0x13, Y6, Y10, Y12 VPERM2I128 $0x13, Y8, Y2, Y4 VPXOR 256(SI), Y0, Y0 VPXOR 288(SI), Y14, Y14 VPXOR 320(SI), Y12, Y12 VPXOR 352(SI), Y4, Y4 VMOVDQU Y0, 256(DI) VMOVDQU Y14, 288(DI) VMOVDQU Y12, 320(DI) VMOVDQU Y4, 352(DI) MOVQ $0x00000180, CX LEAQ 384(SI), SI SUBQ $0x00000180, BX VPERM2I128 $0x02, Y7, Y11, Y0 VPERM2I128 $0x02, 224(BP), Y3, Y14 VPERM2I128 $0x13, Y7, Y11, Y12 VPERM2I128 $0x13, 224(BP), Y3, Y4 JMP sealAVX2SealHash