; Copyright (c) 2022, The rav1e contributors. All rights reserved
;
; This source code is subject to the terms of the BSD 2 Clause License and
; the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
; was not distributed with this source code in the LICENSE file, you can
; obtain it at www.aomedia.org/license/software. If the Alliance for Open
; Media Patent License 1.0 was not distributed with this source code in the
; PATENTS file, you can obtain it at www.aomedia.org/license/patent.

%include "config.asm"
%include "src/asm/x86/x86inc.asm"

%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX)

%if ARCH_X86_64

SECTION_RODATA 32

align 32
pw_1x16:   times 16 dw 1

SECTION .text

%macro NORMALIZE4PT 0
    add eax, 2
    shr eax, 2
%endmacro

%macro NORMALIZE8PT 0
    add eax, 4
    shr eax, 3
%endmacro

; Add and subtract registers
;
; Takes m0 and m1 as both input and output.
; Requires m2 as a free register.
;
; If we start with this permutation:
;
; m0    0 1  2  3    4  5  6  7
; m1    8 9 10 11   12 13 14 15
;
; Then the output will be as such:
;
; m0    [0+8][1+9][2+10][3+11] [4+12][5+13][6+14][7+15]
; m1    [0-8][1-9][2-10][3-11] [4-12][5-13][6-14][7-15]
%macro BUTTERFLY 3
    %define BIT_PRECISION %1
    %define VEC_SIZE %2
    ; use alternate registers 3,4,5
    %define USE_ALT %3

    %if USE_ALT == 1
        SWAP 3, 0
        SWAP 4, 1
        SWAP 5, 2
    %endif

    %if VEC_SIZE == 32
        %define V ym
    %elif VEC_SIZE == 16
        %define V xm
    %endif

    ; Use m2 as a temporary register, then swap
    ; so that m0 and m1 contain the output.
    %if BIT_PRECISION == 16
        paddw       V%+ 2, V%+ 0, V%+ 1
        psubw       V%+ 0, V%+ 1
    %elif BIT_PRECISION == 32
        paddd       ym2, ym0, ym1
        psubd       ym0, ym1
    %else
        %error Incorrect precision specified (16 or 32 expected)
    %endif

    SWAP 2, 1, 0

    %if USE_ALT == 1
        SWAP 3, 0
        SWAP 4, 1
        SWAP 5, 2
    %endif
%endmacro

; Interleave packed rows together (in m0 and m1).
; m2 should contain a free register.
;
; Macro argument takes size in bits of each element (where one
; element is the difference between two original source pixels).
;
; If we start with this permutation:
;
; m0    0 1  2  3    4  5  6  7
; m1    8 9 10 11   12 13 14 15
;
; Then, after INTERLEAVE, this will be the permutation:
;
; m0    0  8  1  9   2 10  3 11
; m1    4 12  5 13   6 14  7 15
%macro INTERLEAVE 3
    %define BIT_PRECISION %1
    %define VEC_SIZE %2
    %define USE_ALT %3

    %if USE_ALT == 1
        SWAP 3, 0
        SWAP 4, 1
        SWAP 5, 2
    %endif

    %if VEC_SIZE == 16
        %define V xm
    %elif VEC_SIZE == 32
        %define V ym
    %else
        %error Invalid vector size (expected 16 or 32)
    %endif

    %if BIT_PRECISION == 16
        punpcklwd   V%+ 2, V%+ 0, V%+ 1
        punpckhwd   V%+ 0, V%+ 1
        SWAP 2, 1, 0
    %elif BIT_PRECISION == 32
        punpckldq   ym2, ym0, ym1
        punpckhdq   ym0, ym1
        ; AVX2 shuffles operate over 128-bit halves of the full ymm register
        ; in parallel, so these shuffles are required to fix up the permutation.
        vperm2i128  ym1, ym2, ym0, 0x20
        vperm2i128  ym0, ym2, ym0, 0x31
        SWAP 0, 1
    %else
        %error Incorrect precision specified (16 or 32 expected)
    %endif

    %if USE_ALT == 1
        SWAP 3, 0
        SWAP 4, 1
        SWAP 5, 2
    %endif
%endmacro

; Interleave pairs of 2 elements (in m0 and m1)
; m2 should contain a free register.
%macro INTERLEAVE_PAIRS 3
    %define BIT_PRECISION %1
    %define VEC_SIZE %2
    %define USE_ALT %3

    %if USE_ALT == 1
        SWAP 3, 0
        SWAP 4, 1
        SWAP 5, 2
    %endif

    %if VEC_SIZE == 16
        %define V xm
    %elif VEC_SIZE == 32
        %define V ym
    %else
        %error Invalid vector size (expected 16 or 32)
    %endif

    %if BIT_PRECISION == 16
        punpckldq   V%+ 2, V%+ 0, V%+ 1
        punpckhdq   V%+ 0, V%+ 1
    %elif BIT_PRECISION == 32
        punpcklqdq  ym2, ym0, ym1
        punpckhqdq  ym0, ym1
    %else
        %error Incorrect precision specified (16 or 32 expected)
    %endif
    SWAP 2, 1, 0

    %if USE_ALT == 1
        SWAP 3, 0
        SWAP 4, 1
        SWAP 5, 2
    %endif
%endmacro

%macro HADAMARD_4X4_PACKED 2
    %define BIT_PRECISION %1
    ; Register size to use (in bytes)
    %define VEC_SIZE %2

    %if VEC_SIZE == 16
        %define V xm
    %elif VEC_SIZE == 32
        %define V ym
    %else
        %error Invalid vector size (expected 16 or 32)
    %endif

    ; Starting registers:

    ; m0    0    1   2   3
    ; m1    4    5   6   7
    ; m2    8    9  10  11
    ; m3    12  13  14  15

    ; Where each number represents an index of the
    ; original block of differences.

    ; Pack rows 0,2 and 1,3 into m0 and m1
    %if BIT_PRECISION == 16
        %if VEC_SIZE == 16
            ; In this case, each row only has 64 bits, so we use
            ; punpcklqdq only. The high 64 bits are always 0.
            punpcklqdq  xm0, xm2
            punpcklqdq  xm1, xm3
        %elif VEC_SIZE == 32
            ; The upper 128 bits of all input registers are zeroed
            punpcklqdq      m4, m0, m2
            punpcklqdq      m5, m1, m3
            punpckhqdq      m0, m0, m2
            punpckhqdq      m1, m1, m3
            vinserti128     m0, m4, xm0, 1
            vinserti128     m1, m5, xm1, 1
        %endif
    %elif BIT_PRECISION == 32
        vinserti128 ym0, ym0, xm2, 1
        vinserti128 ym1, ym1, xm3, 1
    %else
        %error Invalid bit precision (expected 16 or 32)
    %endif

    ; Now that we've packed rows 0-2 and 1-3 together,
    ; this is our permutation:

    ; m0    0 1 2 3   8  9 10 11
    ; m1    4 5 6 7  12 13 14 15

    ; For a 8x4 transform (with 16-bit coefficients), this pattern is
    ; extended for each 128-bit half but for the second block, and thus
    ; all comments also apply to the upper 128-bits for the 8x4 transform.

    BUTTERFLY %1, %2, 0

    ; m0    [0+4][1+5][2+6][3+7] [8+12][9+13][10+14][11+15]
    ; m1    [0-4][1-5][2-6][3-7] [8-12][9-13][10-14][11-15]

    INTERLEAVE %1, %2, 0

    ; m0    [ 0+4][ 0-4][ 1+5][ 1-5] [2 + 6][2 - 6][3 + 7][3 - 7]
    ; m1    [8+12][8-12][9+13][9-13] [10+14][10-14][11+15][11-15]

    BUTTERFLY %1, %2, 0

    ; m0    [0+4+8+12][0-4+8-12][1+5+9+13][1-5+9-13] [2+6+10+14][2-6+10-14][3+7+11+15][3-7+11-15]
    ; m1    [0+4-8-12][0-4-8+12][1+5-9-13][1-5-9+13] [2+6-10-14][2-6-10+14][3+7-11-15][3-7-11+15]

    ; for one row:
    ; [0+1+2+3][0-1+2-3][0+1-2-3][0-1-2+3]
    ; For the vertical transform, these are packed into a new column.

    INTERLEAVE_PAIRS %1, %2, 0

    ;               p0         p1         p2         p3
    ; m0    [0+4+ 8+12][0-4+ 8-12][0+4- 8-12][0-4- 8+12] [1+5+ 9+13][1-5+ 9-13][1+5- 9-13][1-5- 9+13]
    ; m1    [2+6+10+14][2-6+10-14][2+6-10-14][2-6-10+14] [3+7+11+15][3-7+11-15][3+7-11-15][3-7-11+15]

    ; According to this grid:

    ; p0  q0  r0  s0
    ; p1  q1  r1  s1
    ; p2  q2  r2  s2
    ; p3  q3  r3  s3

    ; Horizontal transform; since the output is transposed from the original order,
    ; we can do the same steps as the vertical transform and the result will be the same.
    BUTTERFLY   %1, %2, 0
    INTERLEAVE  %1, %2, 0
    BUTTERFLY   %1, %2, 0

    ; Finished horizontal transform except for the last step (interleaving pairs),
    ; which we skip, because after this we add up the absolute value of the
    ; coefficients, which is a commutative operation (order does not matter).
%endmacro

; Horizontal sum of mm register
;
; Inputs:
; %1 = Element size in bits (16 or 32)
; %2 = Size of input register in bytes (16 or 32)
;      You can e.g. pass 16 for this argument if you
;      only want to sum up the bottom 128-bits of a
;      ymm register.
; %3 = Input register number
; %4 = Temporary register number
; %5 = Output register (e.g., eax)
%macro HSUM 5
    %define E_SIZE %1
    %define REG_SIZE %2
    %define INPUT %3
    %define TMP %4
    %define OUTPUT %5

    %if REG_SIZE == 16
    %define V xm
    %elif REG_SIZE == 32
    %define V ym
    %else
        %error Invalid register size (expected 16 or 32)
    %endif

    %if E_SIZE == 16
        ; Add adjacent pairs of 16-bit elements to produce 32-bit results,
        ; then proceed with 32-bit sum
        pmaddwd     V%+INPUT, [pw_1x16]
    %endif

    %if mmsize == 32 && REG_SIZE == 32
        ; Add upper half of ymm to xmm
        vextracti128    xm%+TMP,   ym%+INPUT, 1
        paddd           xm%+INPUT, xm%+TMP
    %endif

    ; Reduce 32-bit results
    pshufd      xm%+TMP,     xm%+INPUT, q2323
    paddd       xm%+INPUT,   xm%+TMP
    pshufd      xm%+TMP,     xm%+INPUT, q1111
    paddd       xm%+INPUT,   xm%+TMP
    movd        OUTPUT,      xm%+INPUT
%endmacro

; given m0-7, do butterfly as follows:
; (m0, m1) = butterfly(m0, m1)
; (m2, m3) = butterfly(m2, m3)
; (m4, m5) = butterfly(m4, m5)
; (m6, m7) = butterfly(m6, m7)
%macro BUTTERFLY_8X8 0
    ; m8 is free
    paddd   m8, m0, m1
    psubd   m0, m1
    SWAP    8, 1, 0

    ; m8 is free
    paddd   m8, m2, m3
    psubd   m2, m3
    SWAP    8, 3, 2

    paddd   m8, m4, m5
    psubd   m4, m5
    SWAP    8, 5, 4

    paddd   m8, m6, m7
    psubd   m6, m7
    SWAP    8, 7, 6
%endmacro

%macro HADAMARD_8X8_VERTICAL 0
    BUTTERFLY_8X8
    ; m0-7 contain a0-7

    SWAP 2, 1
    SWAP 6, 5

    BUTTERFLY_8X8

    SWAP 1, 4
    SWAP 3, 6

    BUTTERFLY_8X8

    SWAP 2, 1
    SWAP 2, 4
    SWAP 3, 6
    SWAP 5, 6
%endmacro

; Transpose rows m0-7.
; Output is also contained in m0-7.
;
; Uses m8, m10-15 as temporary registers (i.e. m9 is left unchanged.)
%macro TRANSPOSE8X8D 0
    SWAP   9, 0
    SWAP  10, 1
    SWAP  11, 2
    SWAP  12, 3
    SWAP  13, 4
    SWAP  14, 5
    SWAP  15, 6
    SWAP   2, 7

    punpckldq    m6,  m9,  m10
    punpckldq    m1,  m11, m12
    punpckhdq    m8,  m9,  m10
    punpckldq    m4,  m13, m14
    punpckldq    m9,  m15, m2
    vshufps      m3,  m6,  m1,  0x4e
    vpblendd     m10, m6,  m3,  0xcc
    vshufps      m6,  m4,  m9,  0x4e
    punpckhdq    m7,  m11, m12
    vpblendd     m11, m4,  m6,  0xcc
    vpblendd     m12, m3,  m1,  0xcc
    vperm2i128   m3,  m10, m11, 0x20
    punpckhdq    m5,  m13, m14
    vpblendd     m13, m6,  m9,  0xcc
    punpckhdq    m4,  m15, m2
    vperm2i128   m2,  m12, m13, 0x20
    vshufps      m14, m8,  m7,  0x4e
    vpblendd     m15, m14, m7,  0xcc
    vshufps      m7,  m5,  m4,  0x4e
    vpblendd     m8,  m8,  m14, 0xcc
    vpblendd     m5,  m5,  m7,  0xcc
    vperm2i128   m6,  m8,  m5,  0x20
    vpblendd     m4,  m7,  m4,  0xcc
    vperm2i128   m7,  m15, m4,  0x20
    vperm2i128   m1,  m10, m11, 0x31
    vperm2i128   m9,  m12, m13, 0x31
    vperm2i128   m5,  m8,  m5,  0x31
    vperm2i128   m4,  m15, m4,  0x31

    SWAP 0, 9

    ; Output order is as follows:
    ; 3 2 6 7 1 0 5 4

    ; sort rows
    SWAP 3, 0
    ; 0 2 6 7 1 3 5 4
    SWAP 1, 2
    ; 0 1 6 7 2 3 5 4
    SWAP 6, 2
    ; 0 1 2 7 6 3 5 4
    SWAP 7, 3
    ; 0 1 2 3 6 7 5 4
    SWAP 6, 4
    ; 0 1 2 3 4 7 5 6
    SWAP 7, 5
    ; 0 1 2 3 4 5 7 6
    SWAP 6, 7
    ; 0 1 2 3 4 5 6 7
%endmacro

; m0-7 as input; add coefficients to ymm9.
INIT_YMM avx2
cglobal satd_8x8_hbd_internal, 0, 0, 0, src, src_stride, dst, dst_stride, bdmax, \
                                        src_stride3, dst_stride3
    HADAMARD_8X8_VERTICAL

    TRANSPOSE8X8D

    HADAMARD_8X8_VERTICAL

    REPX    {pabsd x, x}, m0, m1, m2, m3, m4, m5, m6, m7

    ; Add m0-7
    paddd   m0, m4
    paddd   m1, m5
    paddd   m2, m6
    paddd   m3, m7

    paddd   m0, m2
    paddd   m1, m3

    paddd   m0, m1
    paddd ymm9, m0
    ret

%macro LOAD_DIFF_8X8 0
    movu        xm0, [srcq + 0*src_strideq]
    movu        xm1, [srcq + 1*src_strideq]
    movu        xm2, [srcq + 2*src_strideq]
    movu        xm3, [srcq + src_stride3q ]
    lea        srcq, [srcq + 4*src_strideq]
    movu        xm4, [srcq + 0*src_strideq]
    movu        xm5, [srcq + 1*src_strideq]
    movu        xm6, [srcq + 2*src_strideq]
    movu        xm7, [srcq + src_stride3q ]

    psubw       xm0, [dstq + 0*dst_strideq]
    psubw       xm1, [dstq + 1*dst_strideq]
    psubw       xm2, [dstq + 2*dst_strideq]
    psubw       xm3, [dstq + dst_stride3q ]
    lea        dstq, [dstq + 4*dst_strideq]
    psubw       xm4, [dstq + 0*dst_strideq]
    psubw       xm5, [dstq + 1*dst_strideq]
    psubw       xm6, [dstq + 2*dst_strideq]
    psubw       xm7, [dstq + dst_stride3q ]

    pmovsxwd    m0, xm0
    pmovsxwd    m1, xm1
    pmovsxwd    m2, xm2
    pmovsxwd    m3, xm3
    pmovsxwd    m4, xm4
    pmovsxwd    m5, xm5
    pmovsxwd    m6, xm6
    pmovsxwd    m7, xm7
%endmacro

INIT_YMM avx2
cglobal satd_8x8_hbd, 5, 7, 16, src, src_stride, dst, dst_stride, bdmax, \
                                src_stride3, dst_stride3
    lea         src_stride3q, [3*src_strideq]
    lea         dst_stride3q, [3*dst_strideq]

    LOAD_DIFF_8X8

    ; m0-7 contain rows of 8x8 block to transform
    ; with 32-bit coefficients

    HADAMARD_8X8_VERTICAL

    TRANSPOSE8X8D

    HADAMARD_8X8_VERTICAL

    REPX    {pabsd x, x}, m0, m1, m2, m3, m4, m5, m6, m7

    ; Add m0-7
    paddd   m0, m4
    paddd   m1, m5
    paddd   m2, m6
    paddd   m3, m7

    paddd   m0, m2
    paddd   m1, m3

    paddd   m0, m1

    HSUM 32, 32, 0, 1, eax
    NORMALIZE8PT
    RET

INIT_YMM avx2
cglobal satd_4x4_hbd, 5, 7, 8, src, src_stride, dst, dst_stride, bdmax, \
                               src_stride3, dst_stride3
    lea         src_stride3q, [3*src_strideq]
    lea         dst_stride3q, [3*dst_strideq]

    cmp         bdmaxd, (1 << 10) - 1
    jne         .12bpc

    ; Load src rows
    movq        xm0, [srcq + 0*src_strideq]
    movq        xm1, [srcq + 1*src_strideq]
    movq        xm2, [srcq + 2*src_strideq]
    movq        xm3, [srcq + src_stride3q ]

    ; src -= dst
    psubw       xm0, [dstq + 0*dst_strideq]
    psubw       xm1, [dstq + 1*dst_strideq]
    psubw       xm2, [dstq + 2*dst_strideq]
    psubw       xm3, [dstq + dst_stride3q ]

    HADAMARD_4X4_PACKED 16, 16

    ; Sum up absolute value of transform coefficients
    pabsw       xm0, xm0
    pabsw       xm1, xm1
    paddw       xm0, xm1
    HSUM 16, 16, 0, 1, eax
    NORMALIZE4PT
    RET
.12bpc:
    ; this gives a nicer disassembly
    RESET_MM_PERMUTATION

    ; Load src rows
    pmovzxwd    xm0, [srcq + 0*src_strideq]
    pmovzxwd    xm1, [srcq + 1*src_strideq]
    pmovzxwd    xm2, [srcq + 2*src_strideq]
    pmovzxwd    xm3, [srcq + src_stride3q ]

    ; Load dst rows
    pmovzxwd    xm4, [dstq + 0*dst_strideq]
    pmovzxwd    xm5, [dstq + 1*dst_strideq]
    pmovzxwd    xm6, [dstq + 2*dst_strideq]
    pmovzxwd    xm7, [dstq + dst_stride3q ]

    ; src -= dst
    psubd       xm0, xm4
    psubd       xm1, xm5
    psubd       xm2, xm6
    psubd       xm3, xm7

    HADAMARD_4X4_PACKED 32, 32

    pabsd       m0, m0
    pabsd       m1, m1
    paddd       m0, m1
    HSUM 32, 32, 0, 1, eax
    NORMALIZE4PT
    RET

; 32-bit input rows are in m0-3; result is in m0.
; Uses m0-5 as temporary registers.
%macro HADAMARD_8X4_12BPC 0
    vperm2i128      m4, m0, m2, 0x31
    vperm2i128      m5, m1, m3, 0x31
    vinserti128     m0, m0, xm2, 1
    vinserti128     m1, m1, xm3, 1

    ; Swap so m3,m4 are used as inputs.
    SWAP 3, 4, 5

    ; instead of using HADAMARD_4X4_PACKED twice, we interleave
    ; 2 transforms operating over different registers for more
    ; opportunity for instruction level parallelism.

    BUTTERFLY           32, 32, 0
    BUTTERFLY           32, 32, 1
    INTERLEAVE          32, 32, 0
    INTERLEAVE          32, 32, 1
    BUTTERFLY           32, 32, 0
    BUTTERFLY           32, 32, 1
    INTERLEAVE_PAIRS    32, 32, 0
    INTERLEAVE_PAIRS    32, 32, 1
    BUTTERFLY           32, 32, 0
    BUTTERFLY           32, 32, 1
    INTERLEAVE          32, 32, 0
    INTERLEAVE          32, 32, 1
    BUTTERFLY           32, 32, 0
    BUTTERFLY           32, 32, 1

    pabsd       m0, m0
    pabsd       m1, m1
    pabsd       m3, m3
    pabsd       m4, m4

    paddd       m0, m1
    paddd       m3, m4
    paddd       m0, m3
%endmacro

INIT_YMM avx2
cglobal satd_16x4_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \
                               src_stride3, dst_stride3
    lea         src_stride3q, [3*src_strideq]
    lea         dst_stride3q, [3*dst_strideq]

    cmp         bdmaxd, (1 << 10) - 1
    jne         .12bpc

    ; Load src rows
    movu         m0, [srcq + 0*src_strideq]
    movu         m1, [srcq + 1*src_strideq]
    movu         m2, [srcq + 2*src_strideq]
    movu         m3, [srcq + src_stride3q ]

    ; src -= dst
    psubw        m0, [dstq + 0*dst_strideq]
    psubw        m1, [dstq + 1*dst_strideq]
    psubw        m2, [dstq + 2*dst_strideq]
    psubw        m3, [dstq + dst_stride3q ]

.10bpc_main:

    ; Original permutation
    ; m0    0   1   2   3      4   5   6   7      8   9  10  11     12  13  14  15
    ; m1   16  17  18  19     20  21  22  23     24  25  26  27     28  29  30  31
    ; m2   32  33  34  35     36  37  38  39     40  41  42  43     44  45  46  47
    ; m3   48  49  50  51     52  53  54  55     56  57  58  59     60  61  62  63

    ; Two registers perform 2 4x4 transforms in parallel

    punpcklqdq  m4, m0, m2
    punpcklqdq  m5, m1, m3

    punpckhqdq  m0, m0, m2
    punpckhqdq  m1, m1, m3

    SWAP 4, 3
    SWAP 5, 4

    ; New permutation
    ; m0    0   1   2   3     32  33  34  35      8   9  10  11     40  41  42  43
    ; m1   16  17  18  19     48  49  50  51     24  25  26  27     56  57  58  59
    ; m3    4   5   6   7     36  37  38  39     12  13  14  15     44  45  46  47
    ; m4   20  21  22  23     52  53  54  55     28  29  30  31     60  61  62  63

    BUTTERFLY           16, 32, 0
    BUTTERFLY           16, 32, 1
    INTERLEAVE          16, 32, 0
    INTERLEAVE          16, 32, 1
    BUTTERFLY           16, 32, 0
    BUTTERFLY           16, 32, 1
    INTERLEAVE_PAIRS    16, 32, 0
    INTERLEAVE_PAIRS    16, 32, 1
    BUTTERFLY           16, 32, 0
    BUTTERFLY           16, 32, 1
    INTERLEAVE          16, 32, 0
    INTERLEAVE          16, 32, 1
    BUTTERFLY           16, 32, 0
    BUTTERFLY           16, 32, 1

    pabsw       m0, m0
    pabsw       m1, m1
    pabsw       m3, m3
    pabsw       m4, m4

    paddw       m0, m1
    paddw       m3, m4
    paddw       m0, m3

    HSUM 16, 32, 0, 1, eax
    NORMALIZE4PT
    RET
.12bpc:
    RESET_MM_PERMUTATION

    mov        bdmaxd, 2
    pxor       m6, m6
.12bpc_loop:
    movu       xm0, [srcq + 0*src_strideq]
    movu       xm1, [srcq + 1*src_strideq]
    movu       xm2, [srcq + 2*src_strideq]
    movu       xm3, [srcq + src_stride3q ]

    psubw      xm0, [dstq + 0*dst_strideq]
    psubw      xm1, [dstq + 1*dst_strideq]
    psubw      xm2, [dstq + 2*dst_strideq]
    psubw      xm3, [dstq + dst_stride3q ]

    pmovsxwd    m0, xm0
    pmovsxwd    m1, xm1
    pmovsxwd    m2, xm2
    pmovsxwd    m3, xm3

    add     srcq, 16
    add     dstq, 16

    HADAMARD_8X4_12BPC
    paddd   m6, m0
    dec     bdmaxd
    jnz     .12bpc_loop

    HSUM 32, 32, 6, 1, eax
    NORMALIZE4PT
    RET

INIT_YMM avx2
cglobal satd_4x16_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \
                               src_stride3, dst_stride3
    lea         src_stride3q, [3*src_strideq]
    lea         dst_stride3q, [3*dst_strideq]

    cmp         bdmaxd, (1 << 10) - 1
    jne         .12bpc

    ; BLOCK 1
    movq        xm0, [srcq + 0*src_strideq]
    movq        xm1, [srcq + 1*src_strideq]
    movq        xm2, [srcq + 2*src_strideq]
    movq        xm3, [srcq + src_stride3q ]
    lea        srcq, [srcq + 4*src_strideq]

    psubw       xm0, [dstq + 0*dst_strideq]
    psubw       xm1, [dstq + 1*dst_strideq]
    psubw       xm2, [dstq + 2*dst_strideq]
    psubw       xm3, [dstq + dst_stride3q ]
    lea        dstq, [dstq + 4*dst_strideq]

    ; BLOCK 2
    movq        xm4, [srcq + 0*src_strideq]
    movq        xm5, [srcq + 1*src_strideq]
    movq        xm6, [srcq + 2*src_strideq]
    movq        xm7, [srcq + src_stride3q ]
    lea        srcq, [srcq + 4*src_strideq]

    psubw       xm4, [dstq + 0*dst_strideq]
    psubw       xm5, [dstq + 1*dst_strideq]
    psubw       xm6, [dstq + 2*dst_strideq]
    psubw       xm7, [dstq + dst_stride3q ]
    lea        dstq, [dstq + 4*dst_strideq]

    vinserti128 m0, m0, xm4, 1
    vinserti128 m1, m1, xm5, 1
    vinserti128 m2, m2, xm6, 1
    vinserti128 m3, m3, xm7, 1

    ; BLOCK 3
    movq        xm4, [srcq + 0*src_strideq]
    movq        xm5, [srcq + 1*src_strideq]
    movq        xm6, [srcq + 2*src_strideq]
    movq        xm7, [srcq + src_stride3q ]
    lea        srcq, [srcq + 4*src_strideq]

    psubw       xm4, [dstq + 0*dst_strideq]
    psubw       xm5, [dstq + 1*dst_strideq]
    psubw       xm6, [dstq + 2*dst_strideq]
    psubw       xm7, [dstq + dst_stride3q ]
    lea        dstq, [dstq + 4*dst_strideq]

    ; BLOCK 4
    movq        xm8, [srcq + 0*src_strideq]
    movq        xm9, [srcq + 1*src_strideq]
    movq       xm10, [srcq + 2*src_strideq]
    movq       xm11, [srcq + src_stride3q ]

    psubw       xm8, [dstq + 0*dst_strideq]
    psubw       xm9, [dstq + 1*dst_strideq]
    psubw      xm10, [dstq + 2*dst_strideq]
    psubw      xm11, [dstq + dst_stride3q ]

    vinserti128  m4, m4, xm8,  1
    vinserti128  m5, m5, xm9,  1
    vinserti128  m6, m6, xm10, 1
    vinserti128  m7, m7, xm11, 1

    punpcklqdq   m0, m0, m4
    punpcklqdq   m1, m1, m5
    punpcklqdq   m2, m2, m6
    punpcklqdq   m3, m3, m7

    jmp     m(satd_16x4_hbd).10bpc_main

.12bpc:
    mov     bdmaxd, 2
    pxor    m8, m8

.12bpc_loop:
    ; BLOCK 1
    movq       xm0, [srcq + 0*src_strideq]
    movq       xm1, [srcq + 1*src_strideq]
    movq       xm2, [srcq + 2*src_strideq]
    movq       xm3, [srcq + src_stride3q ]
    lea       srcq, [srcq + 4*src_strideq]

    psubw      xm0, [dstq + 0*dst_strideq]
    psubw      xm1, [dstq + 1*dst_strideq]
    psubw      xm2, [dstq + 2*dst_strideq]
    psubw      xm3, [dstq + dst_stride3q ]
    lea       dstq, [dstq + 4*dst_strideq]

    pmovsxwd    xm0, xm0
    pmovsxwd    xm1, xm1
    pmovsxwd    xm2, xm2
    pmovsxwd    xm3, xm3

    ; BLOCK 2
    movq       xm4, [srcq + 0*src_strideq]
    movq       xm5, [srcq + 1*src_strideq]
    movq       xm6, [srcq + 2*src_strideq]
    movq       xm7, [srcq + src_stride3q ]
    lea       srcq, [srcq + 4*src_strideq]

    psubw      xm4, [dstq + 0*dst_strideq]
    psubw      xm5, [dstq + 1*dst_strideq]
    psubw      xm6, [dstq + 2*dst_strideq]
    psubw      xm7, [dstq + dst_stride3q ]
    lea       dstq, [dstq + 4*dst_strideq]

    pmovsxwd    xm4, xm4
    pmovsxwd    xm5, xm5
    pmovsxwd    xm6, xm6
    pmovsxwd    xm7, xm7

    vinserti128 m0, m0, xm4, 1
    vinserti128 m1, m1, xm5, 1
    vinserti128 m2, m2, xm6, 1
    vinserti128 m3, m3, xm7, 1

    HADAMARD_8X4_12BPC
    paddd   m8, m0
    dec     bdmaxd
    jnz     .12bpc_loop

    HSUM 32, 32, 8, 0, eax
    NORMALIZE4PT
    RET

INIT_YMM avx2
cglobal satd_8x4_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \
                               src_stride3, dst_stride3
    lea         src_stride3q, [3*src_strideq]
    lea         dst_stride3q, [3*dst_strideq]

    cmp         bdmaxd, (1 << 10) - 1
    jne         .12bpc

    ; Load src rows
    movu        xm0, [srcq + 0*src_strideq]
    movu        xm1, [srcq + 1*src_strideq]
    movu        xm2, [srcq + 2*src_strideq]
    movu        xm3, [srcq + src_stride3q ]

    ; src -= dst
    psubw       xm0, [dstq + 0*dst_strideq]
    psubw       xm1, [dstq + 1*dst_strideq]
    psubw       xm2, [dstq + 2*dst_strideq]
    psubw       xm3, [dstq + dst_stride3q ]

.10bpc_main:
    HADAMARD_4X4_PACKED 16, 32

    pabsw   m0, m0
    pabsw   m1, m1
    paddw   m0, m1
    HSUM    16, 32, 0, 1, eax
    NORMALIZE4PT
    RET
.12bpc:
    RESET_MM_PERMUTATION

    pmovzxwd    m0, [srcq + 0*src_strideq]
    pmovzxwd    m1, [srcq + 1*src_strideq]
    pmovzxwd    m2, [srcq + 2*src_strideq]
    pmovzxwd    m3, [srcq + src_stride3q ]

    pmovzxwd    m4, [dstq + 0*dst_strideq]
    pmovzxwd    m5, [dstq + 1*dst_strideq]
    pmovzxwd    m6, [dstq + 2*dst_strideq]
    pmovzxwd    m7, [dstq + dst_stride3q ]

    ; src -= dst
    psubd       m0, m4
    psubd       m1, m5
    psubd       m2, m6
    psubd       m3, m7

.12bpc_main:
    HADAMARD_8X4_12BPC
    HSUM 32, 32, 0, 1, eax
    NORMALIZE4PT
    RET

INIT_YMM avx2
cglobal satd_4x8_hbd, 5, 7, 12, src, src_stride, dst, dst_stride, bdmax, \
                               src_stride3, dst_stride3
    lea         src_stride3q, [3*src_strideq]
    lea         dst_stride3q, [3*dst_strideq]

    cmp         bdmaxd, (1 << 10) - 1
    jne         .12bpc

    movq        xm0, [srcq + 0*src_strideq]
    movq        xm1, [srcq + 1*src_strideq]
    movq        xm2, [srcq + 2*src_strideq]
    movq        xm3, [srcq + src_stride3q ]
    lea        srcq, [srcq + 4*src_strideq]
    movq        xm4, [srcq + 0*src_strideq]
    movq        xm5, [srcq + 1*src_strideq]
    movq        xm6, [srcq + 2*src_strideq]
    movq        xm7, [srcq + src_stride3q ]

    ; This loads past the number of elements we are technically supposed
    ; to read, however, this should still be safe, since at least one
    ; valid element is in the memory address.
    psubw       xm0, [dstq + 0*dst_strideq]
    psubw       xm1, [dstq + 1*dst_strideq]
    psubw       xm2, [dstq + 2*dst_strideq]
    psubw       xm3, [dstq + dst_stride3q ]
    lea        dstq, [dstq + 4*dst_strideq]
    psubw       xm4, [dstq + 0*dst_strideq]
    psubw       xm5, [dstq + 1*dst_strideq]
    psubw       xm6, [dstq + 2*dst_strideq]
    psubw       xm7, [dstq + dst_stride3q ]

    punpcklqdq  xm0, xm0, xm4
    punpcklqdq  xm1, xm1, xm5
    punpcklqdq  xm2, xm2, xm6
    punpcklqdq  xm3, xm3, xm7

    ; Jump to HADAMARD_4X4_PACKED in 8x4 satd, this saves us some binary size
    ; by deduplicating the shared code.
    jmp m(satd_8x4_hbd).10bpc_main
    ; No return; we return in the other function.

.12bpc:
    RESET_MM_PERMUTATION

    pmovzxwd    xm0, [srcq + 0*src_strideq]
    pmovzxwd    xm1, [srcq + 1*src_strideq]
    pmovzxwd    xm2, [srcq + 2*src_strideq]
    pmovzxwd    xm3, [srcq + src_stride3q ]
    lea        srcq, [srcq + 4*src_strideq]

    pmovzxwd    xm4, [dstq + 0*dst_strideq]
    pmovzxwd    xm5, [dstq + 1*dst_strideq]
    pmovzxwd    xm6, [dstq + 2*dst_strideq]
    pmovzxwd    xm7, [dstq + dst_stride3q ]
    lea        dstq, [dstq + 4*dst_strideq]

    ; src -= dst
    psubd       xm0, xm4
    psubd       xm1, xm5
    psubd       xm2, xm6
    psubd       xm3, xm7

    pmovzxwd    xm4, [srcq + 0*src_strideq]
    pmovzxwd    xm5, [srcq + 1*src_strideq]
    pmovzxwd    xm6, [srcq + 2*src_strideq]
    pmovzxwd    xm7, [srcq + src_stride3q ]

    pmovzxwd    xm8, [dstq + 0*dst_strideq]
    pmovzxwd    xm9, [dstq + 1*dst_strideq]
    pmovzxwd   xm10, [dstq + 2*dst_strideq]
    pmovzxwd   xm11, [dstq + dst_stride3q ]

    ; src -= dst (second block)
    psubd       xm4, xm8
    psubd       xm5, xm9
    psubd       xm6, xm10
    psubd       xm7, xm11

    vinserti128 m0, m0, xm4, 1
    vinserti128 m1, m1, xm5, 1
    vinserti128 m2, m2, xm6, 1
    vinserti128 m3, m3, xm7, 1

    ; Jump to HADAMARD_4X4_PACKED in 8x4 satd, this saves us some binary size
    ; by deduplicating the shared code.
    jmp m(satd_8x4_hbd).12bpc_main
    ; No return; we return in the other function.


; <width>, <height>
%macro SATD_NXM 2

INIT_YMM avx2
cglobal satd_%1x%2_hbd, 5, 10, 16, src, src_stride, dst, dst_stride, bdmax, \
                                 src_stride3, dst_stride3, nsrc_stride4, ndst_stride4, rows
    lea     nsrc_stride4q, [4*src_strideq]
    lea     ndst_stride4q, [4*dst_strideq]

    lea     src_stride3q, [3*src_strideq]
    lea     dst_stride3q, [3*dst_strideq]

    neg     nsrc_stride4q
    neg     ndst_stride4q

    pxor    m9, m9

    ; Height contains the number of rows.
    mov     rowsd, %2/8
.outer:
    mov     bdmaxd, %1/8

; Loop over blocks in same row.
.loop:
    LOAD_DIFF_8X8

    ; Fix up pointers and go to next block in same row.
    lea     srcq, [srcq + nsrc_stride4q + 16]
    lea     dstq, [dstq + ndst_stride4q + 16]

    call    m(satd_8x8_hbd_internal)
    dec     bdmaxd
    jnz     .loop

    lea     srcq, [srcq + 8*src_strideq - (%1*16)/8]
    lea     dstq, [dstq + 8*dst_strideq - (%1*16)/8]

    dec     rowsd
    jnz     .outer

    HSUM 32, 32, 9, 0, eax
    NORMALIZE8PT
    RET
%endmacro

%macro SATD_NX8 1
INIT_YMM avx2
cglobal satd_%1x8_hbd, 5, 9, 16, src, src_stride, dst, dst_stride, bdmax, \
                                 src_stride3, dst_stride3, nsrc_stride4, ndst_stride4
    lea     nsrc_stride4q, [4*src_strideq]
    lea     ndst_stride4q, [4*dst_strideq]

    lea     src_stride3q, [3*src_strideq]
    lea     dst_stride3q, [3*dst_strideq]

    neg     nsrc_stride4q
    neg     ndst_stride4q

    pxor    m9, m9
    mov     bdmaxd, %1/8

.loop:
    LOAD_DIFF_8X8

    lea     srcq, [srcq + nsrc_stride4q + 16]
    lea     dstq, [dstq + ndst_stride4q + 16]

    call    m(satd_8x8_hbd_internal)
    dec     bdmaxd
    jnz     .loop

    HSUM 32, 32, 9, 0, eax
    NORMALIZE8PT
    RET
%endmacro

%macro SATD_8XM 1
INIT_YMM avx2
cglobal satd_8x%1_hbd, 5, 7, 16, src, src_stride, dst, dst_stride, bdmax, \
                                 src_stride3, dst_stride3
    lea     src_stride3q, [3*src_strideq]
    lea     dst_stride3q, [3*dst_strideq]

    pxor    m9, m9
    mov     bdmaxd, %1/8

.loop:
    LOAD_DIFF_8X8

    lea     srcq, [srcq + 4*src_strideq]
    lea     dstq, [dstq + 4*dst_strideq]

    call    m(satd_8x8_hbd_internal)
    dec     bdmaxd
    jnz     .loop

    HSUM 32, 32, 9, 0, eax
    NORMALIZE8PT
    RET
%endmacro

SATD_NXM 16, 16
SATD_NXM 32, 32
SATD_NXM 64, 64
SATD_NXM 128, 128
SATD_NXM 16, 32
SATD_NXM 16, 64
SATD_NXM 32, 16
SATD_NXM 32, 64
SATD_NXM 64, 16
SATD_NXM 64, 32
SATD_NXM 64, 128
SATD_NXM 128, 64
SATD_NX8 16
SATD_NX8 32
SATD_8XM 16
SATD_8XM 32

%endif ; ARCH_X86_64
