// Code generated by command: go run gen.go -out ../encodeblock.s -arch arm64 -arm64gen -pkg=s2. DO NOT EDIT. // EXPERIMENTAL arm64 output lowered from an amd64 avo program. //go:build arm64 && ((amd64 || arm64) && !appengine && !noasm && gc && !noasm) #include "textflag.h" // func encodeBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int // Requires: BMI, SSE2 TEXT ·encodeBlockAsm(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000200, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBlockAsm: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBlockAsm MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBlockAsm: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x06, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBlockAsm MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 LSL $0x10, R10, R10 MUL R8, R10, R10 LSR $0x32, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeBlockAsm ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R7 MOVWU R6, R5 MOVWU 24(RSP), R16 SUBSW R16, R5, R5 BEQ repeat_extend_back_end_encodeBlockAsm repeat_extend_back_loop_encodeBlockAsm: CMPW R7, R6 BLS repeat_extend_back_end_encodeBlockAsm ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeBlockAsm SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R5, R5 BNE repeat_extend_back_loop_encodeBlockAsm repeat_extend_back_end_encodeBlockAsm: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $5, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeBlockAsm: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeBlockAsm MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeBlockAsm CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeBlockAsm CMPW $0x00010000, R5 BLO three_bytes_repeat_emit_encodeBlockAsm CMPW $0x01000000, R5 BLO four_bytes_repeat_emit_encodeBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R5, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_repeat_emit_encodeBlockAsm four_bytes_repeat_emit_encodeBlockAsm: MOVWU R5, R10 LSRW $0x10, R10, R10 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R5, 1(R1) MOVB R10, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_repeat_emit_encodeBlockAsm three_bytes_repeat_emit_encodeBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeBlockAsm two_bytes_repeat_emit_encodeBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeBlockAsm JMP memmove_long_repeat_emit_encodeBlockAsm one_byte_repeat_emit_encodeBlockAsm: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeBlockAsm: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_repeat_emit_encodeBlockAsm emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm emit_lit_memmove_repeat_emit_encodeBlockAsm_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeBlockAsm: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeBlockAsm memmove_long_repeat_emit_encodeBlockAsm: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R11 MOVD R8, R12 emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R11) FMOVQ F1, 16(R11) ADD $0x20, R10, R10 ADD $0x20, R11, R11 SUB $0x20, R12, R12 CMP $0x20, R12 BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsmlarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeBlockAsm: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R8 SUBW R2, R8, R8 ADD R2, R3, R9 ADD R5, R3, R5 // matchLen MOVD $0, R11 matchlen_loopback_16_repeat_extend_encodeBlockAsm: CMPW $0x10, R8 BLO matchlen_match8_repeat_extend_encodeBlockAsm MOVD (R9)(R11), R10 ADD R11, R9, R15 MOVD 8(R15), R12 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm ADD R11, R5, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16repeat_extend_encodeBlockAsm SUB $16, R8, R8 MOVWU R8, R8 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm matchlen_bsf_16repeat_extend_encodeBlockAsm: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm matchlen_match8_repeat_extend_encodeBlockAsm: CMPW $0x08, R8 BLO matchlen_match4_repeat_extend_encodeBlockAsm MOVD (R9)(R11), R10 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm SUB $8, R8, R8 MOVWU R8, R8 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_repeat_extend_encodeBlockAsm matchlen_bsf_8_repeat_extend_encodeBlockAsm: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm matchlen_match4_repeat_extend_encodeBlockAsm: CMPW $0x04, R8 BLO matchlen_match2_repeat_extend_encodeBlockAsm MOVWU (R9)(R11), R10 MOVWU (R5)(R11), R16 CMPW R10, R16 BNE matchlen_match2_repeat_extend_encodeBlockAsm SUB $4, R8, R8 MOVWU R8, R8 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_repeat_extend_encodeBlockAsm: CMPW $0x01, R8 BEQ matchlen_match1_repeat_extend_encodeBlockAsm BLO repeat_extend_forward_end_encodeBlockAsm MOVHU (R9)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R5, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeBlockAsm ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R8, R8 BEQ repeat_extend_forward_end_encodeBlockAsm matchlen_match1_repeat_extend_encodeBlockAsm: MOVBU (R9)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R5, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeBlockAsm ADD $1, R11, R11 MOVWU R11, R11 repeat_extend_forward_end_encodeBlockAsm: ADDW R11, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 TSTW R7, R7 BEQ repeat_as_copy_encodeBlockAsm // emitRepeat emit_repeat_again_match_repeat_encodeBlockAsm: MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_match_repeat_encodeBlockAsm CMPW $0x0c, R7 BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm CMPW $0x00000800, R6 BLO repeat_two_offset_match_repeat_encodeBlockAsm cant_repeat_two_offset_match_repeat_encodeBlockAsm: CMPW $0x00000104, R5 BLO repeat_three_match_repeat_encodeBlockAsm CMPW $0x00010100, R5 BLO repeat_four_match_repeat_encodeBlockAsm CMPW $0x0100ffff, R5 BLO repeat_five_match_repeat_encodeBlockAsm SUB $16842747, R5, R5 MOVWU R5, R5 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_match_repeat_encodeBlockAsm repeat_five_match_repeat_encodeBlockAsm: SUB $65536, R5, R5 MOVWU R5, R5 MOVWU R5, R6 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R5, 2(R1) ASRW $0x10, R6, R6 MOVB R6, 4(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_four_match_repeat_encodeBlockAsm: SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_three_match_repeat_encodeBlockAsm: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_two_match_repeat_encodeBlockAsm: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_two_offset_match_repeat_encodeBlockAsm: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_as_copy_encodeBlockAsm: // emitCopy CMPW $0x00010000, R6 BLO two_byte_offset_repeat_as_copy_encodeBlockAsm CMPW $0x40, R5 BLS four_bytes_remain_repeat_as_copy_encodeBlockAsm MOVD $0xff, R16 MOVB R16, (R1) MOVW R6, 1(R1) SUB $64, R5, R5 MOVWU R5, R5 ADD $0x05, R1, R1 CMPW $0x04, R5 BLO four_bytes_remain_repeat_as_copy_encodeBlockAsm // emitRepeat emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy: MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy CMPW $0x00010100, R5 BLO repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy CMPW $0x0100ffff, R5 BLO repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy SUB $16842747, R5, R5 MOVWU R5, R5 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy: SUB $65536, R5, R5 MOVWU R5, R5 MOVWU R5, R6 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R5, 2(R1) ASRW $0x10, R6, R6 MOVB R6, 4(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy: SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm four_bytes_remain_repeat_as_copy_encodeBlockAsm: TSTW R5, R5 BEQ repeat_end_emit_encodeBlockAsm MOVD $0, R7 ADD R5<<2, R7, R5 SUB $1, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVW R6, 1(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm two_byte_offset_repeat_as_copy_encodeBlockAsm: CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm CMPW $0x00000800, R6 BHS long_offset_short_repeat_as_copy_encodeBlockAsm MOVD $0x00000001, R7 ADD $16, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) MOVWU R6, R8 LSRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 SUBW $0x08, R5, R5 // emitRepeat SUB $4, R5, R5 MOVWU R5, R5 JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b CMPW $0x00010100, R5 BLO repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b CMPW $0x0100ffff, R5 BLO repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b SUB $16842747, R5, R5 MOVWU R5, R5 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: SUB $65536, R5, R5 MOVWU R5, R5 MOVWU R5, R6 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R5, 2(R1) ASRW $0x10, R6, R6 MOVB R6, 4(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short_2b: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm long_offset_short_repeat_as_copy_encodeBlockAsm: MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 // emitRepeat emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short: MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short CMPW $0x00010100, R5 BLO repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short CMPW $0x0100ffff, R5 BLO repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short SUB $16842747, R5, R5 MOVWU R5, R5 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_repeat_as_copy_encodeBlockAsm_emit_copy_short repeat_five_repeat_as_copy_encodeBlockAsm_emit_copy_short: SUB $65536, R5, R5 MOVWU R5, R5 MOVWU R5, R6 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R5, 2(R1) ASRW $0x10, R6, R6 MOVB R6, 4(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_four_repeat_as_copy_encodeBlockAsm_emit_copy_short: SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_three_repeat_as_copy_encodeBlockAsm_emit_copy_short: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_two_repeat_as_copy_encodeBlockAsm_emit_copy_short: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm repeat_two_offset_repeat_as_copy_encodeBlockAsm_emit_copy_short: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm two_byte_offset_short_repeat_as_copy_encodeBlockAsm: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeBlockAsm CMPW $0x00000800, R6 BHS emit_copy_three_repeat_as_copy_encodeBlockAsm SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm emit_copy_three_repeat_as_copy_encodeBlockAsm: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeBlockAsm: MOVW R2, 20(RSP) JMP search_loop_encodeBlockAsm no_repeat_found_encodeBlockAsm: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBlockAsm LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeBlockAsm MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeBlockAsm MOVWU 28(RSP), R2 JMP search_loop_encodeBlockAsm candidate3_match_encodeBlockAsm: ADDW $0x02, R2, R2 JMP candidate_match_encodeBlockAsm candidate2_match_encodeBlockAsm: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBlockAsm: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBlockAsm match_extend_back_loop_encodeBlockAsm: CMPW R6, R2 BLS match_extend_back_end_encodeBlockAsm ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBlockAsm SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBlockAsm JMP match_extend_back_loop_encodeBlockAsm match_extend_back_end_encodeBlockAsm: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $5, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBlockAsm: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeBlockAsm MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeBlockAsm CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeBlockAsm CMPW $0x00010000, R7 BLO three_bytes_match_emit_encodeBlockAsm CMPW $0x01000000, R7 BLO four_bytes_match_emit_encodeBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R7, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_match_emit_encodeBlockAsm four_bytes_match_emit_encodeBlockAsm: MOVWU R7, R9 LSRW $0x10, R9, R9 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R7, 1(R1) MOVB R9, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_match_emit_encodeBlockAsm three_bytes_match_emit_encodeBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBlockAsm two_bytes_match_emit_encodeBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeBlockAsm JMP memmove_long_match_emit_encodeBlockAsm one_byte_match_emit_encodeBlockAsm: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBlockAsm: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeBlockAsm emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm emit_lit_memmove_match_emit_encodeBlockAsm_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBlockAsm: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeBlockAsm memmove_long_match_emit_encodeBlockAsm: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeBlockAsmlarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeBlockAsm: match_nolit_loop_encodeBlockAsm: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeBlockAsm: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeBlockAsm MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeBlockAsm SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeBlockAsm matchlen_bsf_16match_nolit_encodeBlockAsm: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm matchlen_match8_match_nolit_encodeBlockAsm: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeBlockAsm MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeBlockAsm matchlen_bsf_8_match_nolit_encodeBlockAsm: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm matchlen_match4_match_nolit_encodeBlockAsm: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeBlockAsm MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeBlockAsm SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeBlockAsm: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeBlockAsm BLO match_nolit_end_encodeBlockAsm MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBlockAsm ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeBlockAsm matchlen_match1_match_nolit_encodeBlockAsm: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeBlockAsm ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeBlockAsm: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy CMPW $0x00010000, R5 BLO two_byte_offset_match_nolit_encodeBlockAsm CMPW $0x40, R9 BLS four_bytes_remain_match_nolit_encodeBlockAsm MOVD $0xff, R16 MOVB R16, (R1) MOVW R5, 1(R1) SUB $64, R9, R9 MOVWU R9, R9 ADD $0x05, R1, R1 CMPW $0x04, R9 BLO four_bytes_remain_match_nolit_encodeBlockAsm // emitRepeat emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy: MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm_emit_copy CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm_emit_copy CMPW $0x00010100, R9 BLO repeat_four_match_nolit_encodeBlockAsm_emit_copy CMPW $0x0100ffff, R9 BLO repeat_five_match_nolit_encodeBlockAsm_emit_copy SUB $16842747, R9, R9 MOVWU R9, R9 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy repeat_five_match_nolit_encodeBlockAsm_emit_copy: SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R5 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R9, 2(R1) ASRW $0x10, R5, R5 MOVB R5, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_four_match_nolit_encodeBlockAsm_emit_copy: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_three_match_nolit_encodeBlockAsm_emit_copy: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_two_match_nolit_encodeBlockAsm_emit_copy: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm four_bytes_remain_match_nolit_encodeBlockAsm: TSTW R9, R9 BEQ match_nolit_emitcopy_end_encodeBlockAsm MOVD $0, R6 ADD R9<<2, R6, R9 SUB $1, R9, R9 MOVWU R9, R9 MOVB R9, (R1) MOVW R5, 1(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm two_byte_offset_match_nolit_encodeBlockAsm: CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeBlockAsm CMPW $0x00000800, R5 BHS long_offset_short_match_nolit_encodeBlockAsm MOVD $0x00000001, R6 ADD $16, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) MOVWU R5, R7 LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 SUBW $0x08, R9, R9 // emitRepeat SUB $4, R9, R9 MOVWU R9, R9 JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b: MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b CMPW $0x00010100, R9 BLO repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b CMPW $0x0100ffff, R9 BLO repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b SUB $16842747, R9, R9 MOVWU R9, R9 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short_2b repeat_five_match_nolit_encodeBlockAsm_emit_copy_short_2b: SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R5 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R9, 2(R1) ASRW $0x10, R5, R5 MOVB R5, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_four_match_nolit_encodeBlockAsm_emit_copy_short_2b: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_three_match_nolit_encodeBlockAsm_emit_copy_short_2b: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_two_match_nolit_encodeBlockAsm_emit_copy_short_2b: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short_2b: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm long_offset_short_match_nolit_encodeBlockAsm: MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 // emitRepeat emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short: MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm_emit_copy_short CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm_emit_copy_short CMPW $0x00010100, R9 BLO repeat_four_match_nolit_encodeBlockAsm_emit_copy_short CMPW $0x0100ffff, R9 BLO repeat_five_match_nolit_encodeBlockAsm_emit_copy_short SUB $16842747, R9, R9 MOVWU R9, R9 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_match_nolit_encodeBlockAsm_emit_copy_short repeat_five_match_nolit_encodeBlockAsm_emit_copy_short: SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R5 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R9, 2(R1) ASRW $0x10, R5, R5 MOVB R5, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_four_match_nolit_encodeBlockAsm_emit_copy_short: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_three_match_nolit_encodeBlockAsm_emit_copy_short: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_two_match_nolit_encodeBlockAsm_emit_copy_short: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm repeat_two_offset_match_nolit_encodeBlockAsm_emit_copy_short: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm two_byte_offset_short_match_nolit_encodeBlockAsm: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeBlockAsm CMPW $0x00000800, R5 BHS emit_copy_three_match_nolit_encodeBlockAsm SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm emit_copy_three_match_nolit_encodeBlockAsm: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeBlockAsm: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBlockAsm ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBlockAsm: MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x10, R7, R7 MUL R8, R7, R7 LSR $0x32, R7, R7 LSL $0x10, R5, R5 MUL R8, R5, R5 LSR $0x32, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeBlockAsm ADDW $1, R2, R2 JMP search_loop_encodeBlockAsm emit_remainder_encodeBlockAsm: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $5, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBlockAsm: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBlockAsm MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBlockAsm CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBlockAsm CMPW $0x00010000, R2 BLO three_bytes_emit_remainder_encodeBlockAsm CMPW $0x01000000, R2 BLO four_bytes_emit_remainder_encodeBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R2, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_emit_remainder_encodeBlockAsm four_bytes_emit_remainder_encodeBlockAsm: MOVWU R2, R3 LSRW $0x10, R3, R3 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R2, 1(R1) MOVB R3, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_emit_remainder_encodeBlockAsm three_bytes_emit_remainder_encodeBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBlockAsm two_bytes_emit_remainder_encodeBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBlockAsm JMP memmove_long_emit_remainder_encodeBlockAsm one_byte_emit_remainder_encodeBlockAsm: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBlockAsm: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBlockAsm emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm emit_lit_memmove_emit_remainder_encodeBlockAsm_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBlockAsm: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBlockAsm memmove_long_emit_remainder_encodeBlockAsm: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsmlarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBlockAsm: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeBlockAsm4MB(dst []byte, src []byte, tmp *[65536]byte) int // Requires: BMI, SSE2 TEXT ·encodeBlockAsm4MB(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000200, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBlockAsm4MB: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBlockAsm4MB MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBlockAsm4MB: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x06, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBlockAsm4MB MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 LSL $0x10, R10, R10 MUL R8, R10, R10 LSR $0x32, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeBlockAsm4MB ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R7 MOVWU R6, R5 MOVWU 24(RSP), R16 SUBSW R16, R5, R5 BEQ repeat_extend_back_end_encodeBlockAsm4MB repeat_extend_back_loop_encodeBlockAsm4MB: CMPW R7, R6 BLS repeat_extend_back_end_encodeBlockAsm4MB ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeBlockAsm4MB SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R5, R5 BNE repeat_extend_back_loop_encodeBlockAsm4MB repeat_extend_back_end_encodeBlockAsm4MB: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $4, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeBlockAsm4MB MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeBlockAsm4MB: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeBlockAsm4MB MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeBlockAsm4MB CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeBlockAsm4MB CMPW $0x00010000, R5 BLO three_bytes_repeat_emit_encodeBlockAsm4MB MOVWU R5, R10 LSRW $0x10, R10, R10 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R5, 1(R1) MOVB R10, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_repeat_emit_encodeBlockAsm4MB three_bytes_repeat_emit_encodeBlockAsm4MB: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeBlockAsm4MB two_bytes_repeat_emit_encodeBlockAsm4MB: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeBlockAsm4MB JMP memmove_long_repeat_emit_encodeBlockAsm4MB one_byte_repeat_emit_encodeBlockAsm4MB: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeBlockAsm4MB: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm4MB emit_lit_memmove_repeat_emit_encodeBlockAsm4MB_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeBlockAsm4MB: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeBlockAsm4MB memmove_long_repeat_emit_encodeBlockAsm4MB: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R11 MOVD R8, R12 emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R11) FMOVQ F1, 16(R11) ADD $0x20, R10, R10 ADD $0x20, R11, R11 SUB $0x20, R12, R12 CMP $0x20, R12 BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm4MBlarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeBlockAsm4MB: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R8 SUBW R2, R8, R8 ADD R2, R3, R9 ADD R5, R3, R5 // matchLen MOVD $0, R11 matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB: CMPW $0x10, R8 BLO matchlen_match8_repeat_extend_encodeBlockAsm4MB MOVD (R9)(R11), R10 ADD R11, R9, R15 MOVD 8(R15), R12 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB ADD R11, R5, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16repeat_extend_encodeBlockAsm4MB SUB $16, R8, R8 MOVWU R8, R8 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm4MB matchlen_bsf_16repeat_extend_encodeBlockAsm4MB: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm4MB matchlen_match8_repeat_extend_encodeBlockAsm4MB: CMPW $0x08, R8 BLO matchlen_match4_repeat_extend_encodeBlockAsm4MB MOVD (R9)(R11), R10 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB SUB $8, R8, R8 MOVWU R8, R8 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_repeat_extend_encodeBlockAsm4MB matchlen_bsf_8_repeat_extend_encodeBlockAsm4MB: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm4MB matchlen_match4_repeat_extend_encodeBlockAsm4MB: CMPW $0x04, R8 BLO matchlen_match2_repeat_extend_encodeBlockAsm4MB MOVWU (R9)(R11), R10 MOVWU (R5)(R11), R16 CMPW R10, R16 BNE matchlen_match2_repeat_extend_encodeBlockAsm4MB SUB $4, R8, R8 MOVWU R8, R8 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_repeat_extend_encodeBlockAsm4MB: CMPW $0x01, R8 BEQ matchlen_match1_repeat_extend_encodeBlockAsm4MB BLO repeat_extend_forward_end_encodeBlockAsm4MB MOVHU (R9)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R5, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeBlockAsm4MB ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R8, R8 BEQ repeat_extend_forward_end_encodeBlockAsm4MB matchlen_match1_repeat_extend_encodeBlockAsm4MB: MOVBU (R9)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R5, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeBlockAsm4MB ADD $1, R11, R11 MOVWU R11, R11 repeat_extend_forward_end_encodeBlockAsm4MB: ADDW R11, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 TSTW R7, R7 BEQ repeat_as_copy_encodeBlockAsm4MB // emitRepeat MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_match_repeat_encodeBlockAsm4MB CMPW $0x0c, R7 BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB CMPW $0x00000800, R6 BLO repeat_two_offset_match_repeat_encodeBlockAsm4MB cant_repeat_two_offset_match_repeat_encodeBlockAsm4MB: CMPW $0x00000104, R5 BLO repeat_three_match_repeat_encodeBlockAsm4MB CMPW $0x00010100, R5 BLO repeat_four_match_repeat_encodeBlockAsm4MB SUB $65536, R5, R5 MOVWU R5, R5 MOVWU R5, R6 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R5, 2(R1) ASRW $0x10, R6, R6 MOVB R6, 4(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_four_match_repeat_encodeBlockAsm4MB: SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_three_match_repeat_encodeBlockAsm4MB: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_two_match_repeat_encodeBlockAsm4MB: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_two_offset_match_repeat_encodeBlockAsm4MB: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_as_copy_encodeBlockAsm4MB: // emitCopy CMPW $0x00010000, R6 BLO two_byte_offset_repeat_as_copy_encodeBlockAsm4MB CMPW $0x40, R5 BLS four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB MOVD $0xff, R16 MOVB R16, (R1) MOVW R6, 1(R1) SUB $64, R5, R5 MOVWU R5, R5 ADD $0x05, R1, R1 CMPW $0x04, R5 BLO four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB // emitRepeat MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy CMPW $0x00010100, R5 BLO repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy SUB $65536, R5, R5 MOVWU R5, R5 MOVWU R5, R6 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R5, 2(R1) ASRW $0x10, R6, R6 MOVB R6, 4(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy: SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB four_bytes_remain_repeat_as_copy_encodeBlockAsm4MB: TSTW R5, R5 BEQ repeat_end_emit_encodeBlockAsm4MB MOVD $0, R7 ADD R5<<2, R7, R5 SUB $1, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVW R6, 1(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB two_byte_offset_repeat_as_copy_encodeBlockAsm4MB: CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB CMPW $0x00000800, R6 BHS long_offset_short_repeat_as_copy_encodeBlockAsm4MB MOVD $0x00000001, R7 ADD $16, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 SUBW $0x08, R5, R5 // emitRepeat SUB $4, R5, R5 MOVWU R5, R5 JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b CMPW $0x00010100, R5 BLO repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b SUB $65536, R5, R5 MOVWU R5, R5 MOVWU R5, R6 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R5, 2(R1) ASRW $0x10, R6, R6 MOVB R6, 4(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short_2b: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB long_offset_short_repeat_as_copy_encodeBlockAsm4MB: MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 // emitRepeat MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short CMPW $0x00010100, R5 BLO repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short SUB $65536, R5, R5 MOVWU R5, R5 MOVWU R5, R6 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R5, 2(R1) ASRW $0x10, R6, R6 MOVB R6, 4(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_four_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_three_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_two_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB repeat_two_offset_repeat_as_copy_encodeBlockAsm4MB_emit_copy_short: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB two_byte_offset_short_repeat_as_copy_encodeBlockAsm4MB: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeBlockAsm4MB CMPW $0x00000800, R6 BHS emit_copy_three_repeat_as_copy_encodeBlockAsm4MB SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm4MB emit_copy_three_repeat_as_copy_encodeBlockAsm4MB: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeBlockAsm4MB: MOVW R2, 20(RSP) JMP search_loop_encodeBlockAsm4MB no_repeat_found_encodeBlockAsm4MB: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBlockAsm4MB LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeBlockAsm4MB MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeBlockAsm4MB MOVWU 28(RSP), R2 JMP search_loop_encodeBlockAsm4MB candidate3_match_encodeBlockAsm4MB: ADDW $0x02, R2, R2 JMP candidate_match_encodeBlockAsm4MB candidate2_match_encodeBlockAsm4MB: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBlockAsm4MB: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBlockAsm4MB match_extend_back_loop_encodeBlockAsm4MB: CMPW R6, R2 BLS match_extend_back_end_encodeBlockAsm4MB ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBlockAsm4MB SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBlockAsm4MB JMP match_extend_back_loop_encodeBlockAsm4MB match_extend_back_end_encodeBlockAsm4MB: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $4, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBlockAsm4MB MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBlockAsm4MB: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeBlockAsm4MB MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeBlockAsm4MB CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeBlockAsm4MB CMPW $0x00010000, R7 BLO three_bytes_match_emit_encodeBlockAsm4MB MOVWU R7, R9 LSRW $0x10, R9, R9 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R7, 1(R1) MOVB R9, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_match_emit_encodeBlockAsm4MB three_bytes_match_emit_encodeBlockAsm4MB: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBlockAsm4MB two_bytes_match_emit_encodeBlockAsm4MB: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeBlockAsm4MB JMP memmove_long_match_emit_encodeBlockAsm4MB one_byte_match_emit_encodeBlockAsm4MB: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBlockAsm4MB: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeBlockAsm4MB emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm4MB emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm4MB emit_lit_memmove_match_emit_encodeBlockAsm4MB_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBlockAsm4MB: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeBlockAsm4MB memmove_long_match_emit_encodeBlockAsm4MB: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeBlockAsm4MBlarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeBlockAsm4MB: match_nolit_loop_encodeBlockAsm4MB: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeBlockAsm4MB: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeBlockAsm4MB MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm4MB ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeBlockAsm4MB SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeBlockAsm4MB matchlen_bsf_16match_nolit_encodeBlockAsm4MB: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm4MB matchlen_match8_match_nolit_encodeBlockAsm4MB: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeBlockAsm4MB MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm4MB SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeBlockAsm4MB matchlen_bsf_8_match_nolit_encodeBlockAsm4MB: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm4MB matchlen_match4_match_nolit_encodeBlockAsm4MB: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeBlockAsm4MB MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeBlockAsm4MB SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeBlockAsm4MB: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeBlockAsm4MB BLO match_nolit_end_encodeBlockAsm4MB MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBlockAsm4MB ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeBlockAsm4MB matchlen_match1_match_nolit_encodeBlockAsm4MB: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeBlockAsm4MB ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeBlockAsm4MB: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy CMPW $0x00010000, R5 BLO two_byte_offset_match_nolit_encodeBlockAsm4MB CMPW $0x40, R9 BLS four_bytes_remain_match_nolit_encodeBlockAsm4MB MOVD $0xff, R16 MOVB R16, (R1) MOVW R5, 1(R1) SUB $64, R9, R9 MOVWU R9, R9 ADD $0x05, R1, R1 CMPW $0x04, R9 BLO four_bytes_remain_match_nolit_encodeBlockAsm4MB // emitRepeat MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy CMPW $0x00010100, R9 BLO repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R5 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R9, 2(R1) ASRW $0x10, R5, R5 MOVB R5, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB four_bytes_remain_match_nolit_encodeBlockAsm4MB: TSTW R9, R9 BEQ match_nolit_emitcopy_end_encodeBlockAsm4MB MOVD $0, R6 ADD R9<<2, R6, R9 SUB $1, R9, R9 MOVWU R9, R9 MOVB R9, (R1) MOVW R5, 1(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB two_byte_offset_match_nolit_encodeBlockAsm4MB: CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeBlockAsm4MB CMPW $0x00000800, R5 BHS long_offset_short_match_nolit_encodeBlockAsm4MB MOVD $0x00000001, R6 ADD $16, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 SUBW $0x08, R9, R9 // emitRepeat SUB $4, R9, R9 MOVWU R9, R9 JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b CMPW $0x00010100, R9 BLO repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R5 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R9, 2(R1) ASRW $0x10, R5, R5 MOVB R5, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short_2b: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB long_offset_short_match_nolit_encodeBlockAsm4MB: MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 // emitRepeat MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short CMPW $0x00010100, R9 BLO repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R5 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R9, 2(R1) ASRW $0x10, R5, R5 MOVB R5, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_four_match_nolit_encodeBlockAsm4MB_emit_copy_short: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_three_match_nolit_encodeBlockAsm4MB_emit_copy_short: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_two_match_nolit_encodeBlockAsm4MB_emit_copy_short: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB repeat_two_offset_match_nolit_encodeBlockAsm4MB_emit_copy_short: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB two_byte_offset_short_match_nolit_encodeBlockAsm4MB: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeBlockAsm4MB CMPW $0x00000800, R5 BHS emit_copy_three_match_nolit_encodeBlockAsm4MB SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm4MB emit_copy_three_match_nolit_encodeBlockAsm4MB: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeBlockAsm4MB: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBlockAsm4MB ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBlockAsm4MB MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBlockAsm4MB: MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x10, R7, R7 MUL R8, R7, R7 LSR $0x32, R7, R7 LSL $0x10, R5, R5 MUL R8, R5, R5 LSR $0x32, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeBlockAsm4MB ADDW $1, R2, R2 JMP search_loop_encodeBlockAsm4MB emit_remainder_encodeBlockAsm4MB: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $4, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBlockAsm4MB MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBlockAsm4MB: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBlockAsm4MB MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBlockAsm4MB CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBlockAsm4MB CMPW $0x00010000, R2 BLO three_bytes_emit_remainder_encodeBlockAsm4MB MOVWU R2, R3 LSRW $0x10, R3, R3 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R2, 1(R1) MOVB R3, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_emit_remainder_encodeBlockAsm4MB three_bytes_emit_remainder_encodeBlockAsm4MB: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBlockAsm4MB two_bytes_emit_remainder_encodeBlockAsm4MB: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBlockAsm4MB JMP memmove_long_emit_remainder_encodeBlockAsm4MB one_byte_emit_remainder_encodeBlockAsm4MB: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBlockAsm4MB: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBlockAsm4MB_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBlockAsm4MB: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBlockAsm4MB memmove_long_emit_remainder_encodeBlockAsm4MB: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm4MBlarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBlockAsm4MB: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int // Requires: BMI, SSE2 TEXT ·encodeBlockAsm12B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000080, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBlockAsm12B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBlockAsm12B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBlockAsm12B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x05, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBlockAsm12B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x000000cf1bbcdcbb, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x18, R9, R9 MUL R8, R9, R9 LSR $0x34, R9, R9 LSL $0x18, R10, R10 MUL R8, R10, R10 LSR $0x34, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x18, R9, R9 MUL R8, R9, R9 LSR $0x34, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeBlockAsm12B ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R7 MOVWU R6, R5 MOVWU 24(RSP), R16 SUBSW R16, R5, R5 BEQ repeat_extend_back_end_encodeBlockAsm12B repeat_extend_back_loop_encodeBlockAsm12B: CMPW R7, R6 BLS repeat_extend_back_end_encodeBlockAsm12B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeBlockAsm12B SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R5, R5 BNE repeat_extend_back_loop_encodeBlockAsm12B repeat_extend_back_end_encodeBlockAsm12B: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $3, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeBlockAsm12B: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeBlockAsm12B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeBlockAsm12B CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeBlockAsm12B BLO three_bytes_repeat_emit_encodeBlockAsm12B three_bytes_repeat_emit_encodeBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeBlockAsm12B two_bytes_repeat_emit_encodeBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeBlockAsm12B JMP memmove_long_repeat_emit_encodeBlockAsm12B one_byte_repeat_emit_encodeBlockAsm12B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeBlockAsm12B: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm12B emit_lit_memmove_repeat_emit_encodeBlockAsm12B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeBlockAsm12B: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeBlockAsm12B memmove_long_repeat_emit_encodeBlockAsm12B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R11 MOVD R8, R12 emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R11) FMOVQ F1, 16(R11) ADD $0x20, R10, R10 ADD $0x20, R11, R11 SUB $0x20, R12, R12 CMP $0x20, R12 BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm12Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeBlockAsm12B: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R8 SUBW R2, R8, R8 ADD R2, R3, R9 ADD R5, R3, R5 // matchLen MOVD $0, R11 matchlen_loopback_16_repeat_extend_encodeBlockAsm12B: CMPW $0x10, R8 BLO matchlen_match8_repeat_extend_encodeBlockAsm12B MOVD (R9)(R11), R10 ADD R11, R9, R15 MOVD 8(R15), R12 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm12B ADD R11, R5, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16repeat_extend_encodeBlockAsm12B SUB $16, R8, R8 MOVWU R8, R8 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm12B matchlen_bsf_16repeat_extend_encodeBlockAsm12B: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm12B matchlen_match8_repeat_extend_encodeBlockAsm12B: CMPW $0x08, R8 BLO matchlen_match4_repeat_extend_encodeBlockAsm12B MOVD (R9)(R11), R10 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm12B SUB $8, R8, R8 MOVWU R8, R8 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_repeat_extend_encodeBlockAsm12B matchlen_bsf_8_repeat_extend_encodeBlockAsm12B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm12B matchlen_match4_repeat_extend_encodeBlockAsm12B: CMPW $0x04, R8 BLO matchlen_match2_repeat_extend_encodeBlockAsm12B MOVWU (R9)(R11), R10 MOVWU (R5)(R11), R16 CMPW R10, R16 BNE matchlen_match2_repeat_extend_encodeBlockAsm12B SUB $4, R8, R8 MOVWU R8, R8 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_repeat_extend_encodeBlockAsm12B: CMPW $0x01, R8 BEQ matchlen_match1_repeat_extend_encodeBlockAsm12B BLO repeat_extend_forward_end_encodeBlockAsm12B MOVHU (R9)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R5, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeBlockAsm12B ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R8, R8 BEQ repeat_extend_forward_end_encodeBlockAsm12B matchlen_match1_repeat_extend_encodeBlockAsm12B: MOVBU (R9)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R5, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeBlockAsm12B ADD $1, R11, R11 MOVWU R11, R11 repeat_extend_forward_end_encodeBlockAsm12B: ADDW R11, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 TSTW R7, R7 BEQ repeat_as_copy_encodeBlockAsm12B // emitRepeat MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_match_repeat_encodeBlockAsm12B CMPW $0x0c, R7 BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm12B CMPW $0x00000800, R6 BLO repeat_two_offset_match_repeat_encodeBlockAsm12B cant_repeat_two_offset_match_repeat_encodeBlockAsm12B: CMPW $0x00000104, R5 BLO repeat_three_match_repeat_encodeBlockAsm12B SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_three_match_repeat_encodeBlockAsm12B: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_two_match_repeat_encodeBlockAsm12B: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_two_offset_match_repeat_encodeBlockAsm12B: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_as_copy_encodeBlockAsm12B: // emitCopy CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B CMPW $0x00000800, R6 BHS long_offset_short_repeat_as_copy_encodeBlockAsm12B MOVD $0x00000001, R7 ADD $16, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 SUBW $0x08, R5, R5 // emitRepeat SUB $4, R5, R5 MOVWU R5, R5 JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short_2b: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B long_offset_short_repeat_as_copy_encodeBlockAsm12B: MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 // emitRepeat MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_three_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_two_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B repeat_two_offset_repeat_as_copy_encodeBlockAsm12B_emit_copy_short: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B two_byte_offset_short_repeat_as_copy_encodeBlockAsm12B: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeBlockAsm12B CMPW $0x00000800, R6 BHS emit_copy_three_repeat_as_copy_encodeBlockAsm12B SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm12B emit_copy_three_repeat_as_copy_encodeBlockAsm12B: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeBlockAsm12B: MOVW R2, 20(RSP) JMP search_loop_encodeBlockAsm12B no_repeat_found_encodeBlockAsm12B: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBlockAsm12B LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeBlockAsm12B MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeBlockAsm12B MOVWU 28(RSP), R2 JMP search_loop_encodeBlockAsm12B candidate3_match_encodeBlockAsm12B: ADDW $0x02, R2, R2 JMP candidate_match_encodeBlockAsm12B candidate2_match_encodeBlockAsm12B: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBlockAsm12B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBlockAsm12B match_extend_back_loop_encodeBlockAsm12B: CMPW R6, R2 BLS match_extend_back_end_encodeBlockAsm12B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBlockAsm12B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBlockAsm12B JMP match_extend_back_loop_encodeBlockAsm12B match_extend_back_end_encodeBlockAsm12B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBlockAsm12B: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeBlockAsm12B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeBlockAsm12B CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeBlockAsm12B BLO three_bytes_match_emit_encodeBlockAsm12B three_bytes_match_emit_encodeBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBlockAsm12B two_bytes_match_emit_encodeBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeBlockAsm12B JMP memmove_long_match_emit_encodeBlockAsm12B one_byte_match_emit_encodeBlockAsm12B: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBlockAsm12B: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeBlockAsm12B emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm12B emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm12B emit_lit_memmove_match_emit_encodeBlockAsm12B_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBlockAsm12B: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeBlockAsm12B memmove_long_match_emit_encodeBlockAsm12B: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeBlockAsm12Blarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeBlockAsm12B: match_nolit_loop_encodeBlockAsm12B: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeBlockAsm12B: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeBlockAsm12B MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm12B ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeBlockAsm12B SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeBlockAsm12B matchlen_bsf_16match_nolit_encodeBlockAsm12B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm12B matchlen_match8_match_nolit_encodeBlockAsm12B: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeBlockAsm12B MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm12B SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeBlockAsm12B matchlen_bsf_8_match_nolit_encodeBlockAsm12B: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm12B matchlen_match4_match_nolit_encodeBlockAsm12B: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeBlockAsm12B MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeBlockAsm12B SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeBlockAsm12B: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeBlockAsm12B BLO match_nolit_end_encodeBlockAsm12B MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBlockAsm12B ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeBlockAsm12B matchlen_match1_match_nolit_encodeBlockAsm12B: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeBlockAsm12B ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeBlockAsm12B: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeBlockAsm12B CMPW $0x00000800, R5 BHS long_offset_short_match_nolit_encodeBlockAsm12B MOVD $0x00000001, R6 ADD $16, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 SUBW $0x08, R9, R9 // emitRepeat SUB $4, R9, R9 MOVWU R9, R9 JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm12B repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm12B repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm12B repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short_2b: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm12B long_offset_short_match_nolit_encodeBlockAsm12B: MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 // emitRepeat MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm12B repeat_three_match_nolit_encodeBlockAsm12B_emit_copy_short: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm12B repeat_two_match_nolit_encodeBlockAsm12B_emit_copy_short: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm12B repeat_two_offset_match_nolit_encodeBlockAsm12B_emit_copy_short: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm12B two_byte_offset_short_match_nolit_encodeBlockAsm12B: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeBlockAsm12B CMPW $0x00000800, R5 BHS emit_copy_three_match_nolit_encodeBlockAsm12B SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm12B emit_copy_three_match_nolit_encodeBlockAsm12B: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeBlockAsm12B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBlockAsm12B ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBlockAsm12B: MOVD $0x000000cf1bbcdcbb, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x18, R7, R7 MUL R8, R7, R7 LSR $0x34, R7, R7 LSL $0x18, R5, R5 MUL R8, R5, R5 LSR $0x34, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeBlockAsm12B ADDW $1, R2, R2 JMP search_loop_encodeBlockAsm12B emit_remainder_encodeBlockAsm12B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBlockAsm12B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBlockAsm12B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBlockAsm12B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBlockAsm12B BLO three_bytes_emit_remainder_encodeBlockAsm12B three_bytes_emit_remainder_encodeBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBlockAsm12B two_bytes_emit_remainder_encodeBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBlockAsm12B JMP memmove_long_emit_remainder_encodeBlockAsm12B one_byte_emit_remainder_encodeBlockAsm12B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBlockAsm12B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm12B emit_lit_memmove_emit_remainder_encodeBlockAsm12B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBlockAsm12B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBlockAsm12B memmove_long_emit_remainder_encodeBlockAsm12B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm12Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBlockAsm12B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int // Requires: BMI, SSE2 TEXT ·encodeBlockAsm10B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000020, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBlockAsm10B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBlockAsm10B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBlockAsm10B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x05, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBlockAsm10B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x9e3779b1, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x36, R9, R9 LSL $0x20, R10, R10 MUL R8, R10, R10 LSR $0x36, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x36, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeBlockAsm10B ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R7 MOVWU R6, R5 MOVWU 24(RSP), R16 SUBSW R16, R5, R5 BEQ repeat_extend_back_end_encodeBlockAsm10B repeat_extend_back_loop_encodeBlockAsm10B: CMPW R7, R6 BLS repeat_extend_back_end_encodeBlockAsm10B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeBlockAsm10B SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R5, R5 BNE repeat_extend_back_loop_encodeBlockAsm10B repeat_extend_back_end_encodeBlockAsm10B: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $3, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeBlockAsm10B: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeBlockAsm10B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeBlockAsm10B CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeBlockAsm10B BLO three_bytes_repeat_emit_encodeBlockAsm10B three_bytes_repeat_emit_encodeBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeBlockAsm10B two_bytes_repeat_emit_encodeBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeBlockAsm10B JMP memmove_long_repeat_emit_encodeBlockAsm10B one_byte_repeat_emit_encodeBlockAsm10B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeBlockAsm10B: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm10B emit_lit_memmove_repeat_emit_encodeBlockAsm10B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeBlockAsm10B: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeBlockAsm10B memmove_long_repeat_emit_encodeBlockAsm10B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R11 MOVD R8, R12 emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R11) FMOVQ F1, 16(R11) ADD $0x20, R10, R10 ADD $0x20, R11, R11 SUB $0x20, R12, R12 CMP $0x20, R12 BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm10Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeBlockAsm10B: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R8 SUBW R2, R8, R8 ADD R2, R3, R9 ADD R5, R3, R5 // matchLen MOVD $0, R11 matchlen_loopback_16_repeat_extend_encodeBlockAsm10B: CMPW $0x10, R8 BLO matchlen_match8_repeat_extend_encodeBlockAsm10B MOVD (R9)(R11), R10 ADD R11, R9, R15 MOVD 8(R15), R12 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm10B ADD R11, R5, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16repeat_extend_encodeBlockAsm10B SUB $16, R8, R8 MOVWU R8, R8 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm10B matchlen_bsf_16repeat_extend_encodeBlockAsm10B: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm10B matchlen_match8_repeat_extend_encodeBlockAsm10B: CMPW $0x08, R8 BLO matchlen_match4_repeat_extend_encodeBlockAsm10B MOVD (R9)(R11), R10 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm10B SUB $8, R8, R8 MOVWU R8, R8 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_repeat_extend_encodeBlockAsm10B matchlen_bsf_8_repeat_extend_encodeBlockAsm10B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm10B matchlen_match4_repeat_extend_encodeBlockAsm10B: CMPW $0x04, R8 BLO matchlen_match2_repeat_extend_encodeBlockAsm10B MOVWU (R9)(R11), R10 MOVWU (R5)(R11), R16 CMPW R10, R16 BNE matchlen_match2_repeat_extend_encodeBlockAsm10B SUB $4, R8, R8 MOVWU R8, R8 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_repeat_extend_encodeBlockAsm10B: CMPW $0x01, R8 BEQ matchlen_match1_repeat_extend_encodeBlockAsm10B BLO repeat_extend_forward_end_encodeBlockAsm10B MOVHU (R9)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R5, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeBlockAsm10B ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R8, R8 BEQ repeat_extend_forward_end_encodeBlockAsm10B matchlen_match1_repeat_extend_encodeBlockAsm10B: MOVBU (R9)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R5, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeBlockAsm10B ADD $1, R11, R11 MOVWU R11, R11 repeat_extend_forward_end_encodeBlockAsm10B: ADDW R11, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 TSTW R7, R7 BEQ repeat_as_copy_encodeBlockAsm10B // emitRepeat MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_match_repeat_encodeBlockAsm10B CMPW $0x0c, R7 BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm10B CMPW $0x00000800, R6 BLO repeat_two_offset_match_repeat_encodeBlockAsm10B cant_repeat_two_offset_match_repeat_encodeBlockAsm10B: CMPW $0x00000104, R5 BLO repeat_three_match_repeat_encodeBlockAsm10B SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_three_match_repeat_encodeBlockAsm10B: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_two_match_repeat_encodeBlockAsm10B: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_two_offset_match_repeat_encodeBlockAsm10B: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_as_copy_encodeBlockAsm10B: // emitCopy CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B CMPW $0x00000800, R6 BHS long_offset_short_repeat_as_copy_encodeBlockAsm10B MOVD $0x00000001, R7 ADD $16, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 SUBW $0x08, R5, R5 // emitRepeat SUB $4, R5, R5 MOVWU R5, R5 JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short_2b: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B long_offset_short_repeat_as_copy_encodeBlockAsm10B: MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 // emitRepeat MOVWU R5, R7 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R7 BLS repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short CMPW $0x0c, R7 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short CMPW $0x00000800, R6 BLO repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_three_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_two_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B repeat_two_offset_repeat_as_copy_encodeBlockAsm10B_emit_copy_short: MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B two_byte_offset_short_repeat_as_copy_encodeBlockAsm10B: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeBlockAsm10B CMPW $0x00000800, R6 BHS emit_copy_three_repeat_as_copy_encodeBlockAsm10B SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm10B emit_copy_three_repeat_as_copy_encodeBlockAsm10B: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeBlockAsm10B: MOVW R2, 20(RSP) JMP search_loop_encodeBlockAsm10B no_repeat_found_encodeBlockAsm10B: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBlockAsm10B LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeBlockAsm10B MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeBlockAsm10B MOVWU 28(RSP), R2 JMP search_loop_encodeBlockAsm10B candidate3_match_encodeBlockAsm10B: ADDW $0x02, R2, R2 JMP candidate_match_encodeBlockAsm10B candidate2_match_encodeBlockAsm10B: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBlockAsm10B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBlockAsm10B match_extend_back_loop_encodeBlockAsm10B: CMPW R6, R2 BLS match_extend_back_end_encodeBlockAsm10B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBlockAsm10B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBlockAsm10B JMP match_extend_back_loop_encodeBlockAsm10B match_extend_back_end_encodeBlockAsm10B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBlockAsm10B: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeBlockAsm10B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeBlockAsm10B CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeBlockAsm10B BLO three_bytes_match_emit_encodeBlockAsm10B three_bytes_match_emit_encodeBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBlockAsm10B two_bytes_match_emit_encodeBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeBlockAsm10B JMP memmove_long_match_emit_encodeBlockAsm10B one_byte_match_emit_encodeBlockAsm10B: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBlockAsm10B: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeBlockAsm10B emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm10B emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm10B emit_lit_memmove_match_emit_encodeBlockAsm10B_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBlockAsm10B: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeBlockAsm10B memmove_long_match_emit_encodeBlockAsm10B: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeBlockAsm10Blarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeBlockAsm10B: match_nolit_loop_encodeBlockAsm10B: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeBlockAsm10B: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeBlockAsm10B MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm10B ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeBlockAsm10B SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeBlockAsm10B matchlen_bsf_16match_nolit_encodeBlockAsm10B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm10B matchlen_match8_match_nolit_encodeBlockAsm10B: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeBlockAsm10B MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm10B SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeBlockAsm10B matchlen_bsf_8_match_nolit_encodeBlockAsm10B: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm10B matchlen_match4_match_nolit_encodeBlockAsm10B: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeBlockAsm10B MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeBlockAsm10B SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeBlockAsm10B: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeBlockAsm10B BLO match_nolit_end_encodeBlockAsm10B MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBlockAsm10B ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeBlockAsm10B matchlen_match1_match_nolit_encodeBlockAsm10B: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeBlockAsm10B ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeBlockAsm10B: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeBlockAsm10B CMPW $0x00000800, R5 BHS long_offset_short_match_nolit_encodeBlockAsm10B MOVD $0x00000001, R6 ADD $16, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 SUBW $0x08, R9, R9 // emitRepeat SUB $4, R9, R9 MOVWU R9, R9 JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm10B repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm10B repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm10B repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short_2b: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm10B long_offset_short_match_nolit_encodeBlockAsm10B: MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 // emitRepeat MOVWU R9, R6 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R6 BLS repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short CMPW $0x00000800, R5 BLO repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm10B repeat_three_match_nolit_encodeBlockAsm10B_emit_copy_short: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm10B repeat_two_match_nolit_encodeBlockAsm10B_emit_copy_short: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm10B repeat_two_offset_match_nolit_encodeBlockAsm10B_emit_copy_short: MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm10B two_byte_offset_short_match_nolit_encodeBlockAsm10B: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeBlockAsm10B CMPW $0x00000800, R5 BHS emit_copy_three_match_nolit_encodeBlockAsm10B SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm10B emit_copy_three_match_nolit_encodeBlockAsm10B: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeBlockAsm10B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBlockAsm10B ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBlockAsm10B: MOVD $0x9e3779b1, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x20, R7, R7 MUL R8, R7, R7 LSR $0x36, R7, R7 LSL $0x20, R5, R5 MUL R8, R5, R5 LSR $0x36, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeBlockAsm10B ADDW $1, R2, R2 JMP search_loop_encodeBlockAsm10B emit_remainder_encodeBlockAsm10B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBlockAsm10B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBlockAsm10B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBlockAsm10B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBlockAsm10B BLO three_bytes_emit_remainder_encodeBlockAsm10B three_bytes_emit_remainder_encodeBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBlockAsm10B two_bytes_emit_remainder_encodeBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBlockAsm10B JMP memmove_long_emit_remainder_encodeBlockAsm10B one_byte_emit_remainder_encodeBlockAsm10B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBlockAsm10B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm10B emit_lit_memmove_emit_remainder_encodeBlockAsm10B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBlockAsm10B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBlockAsm10B memmove_long_emit_remainder_encodeBlockAsm10B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm10Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBlockAsm10B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int // Requires: BMI, SSE2 TEXT ·encodeBlockAsm8B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000008, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBlockAsm8B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBlockAsm8B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBlockAsm8B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x04, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBlockAsm8B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x9e3779b1, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x38, R9, R9 LSL $0x20, R10, R10 MUL R8, R10, R10 LSR $0x38, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x38, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeBlockAsm8B ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R7 MOVWU R6, R5 MOVWU 24(RSP), R16 SUBSW R16, R5, R5 BEQ repeat_extend_back_end_encodeBlockAsm8B repeat_extend_back_loop_encodeBlockAsm8B: CMPW R7, R6 BLS repeat_extend_back_end_encodeBlockAsm8B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeBlockAsm8B SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R5, R5 BNE repeat_extend_back_loop_encodeBlockAsm8B repeat_extend_back_end_encodeBlockAsm8B: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $3, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeBlockAsm8B: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeBlockAsm8B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeBlockAsm8B CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeBlockAsm8B BLO three_bytes_repeat_emit_encodeBlockAsm8B three_bytes_repeat_emit_encodeBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeBlockAsm8B two_bytes_repeat_emit_encodeBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeBlockAsm8B JMP memmove_long_repeat_emit_encodeBlockAsm8B one_byte_repeat_emit_encodeBlockAsm8B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeBlockAsm8B: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeBlockAsm8B emit_lit_memmove_repeat_emit_encodeBlockAsm8B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeBlockAsm8B: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeBlockAsm8B memmove_long_repeat_emit_encodeBlockAsm8B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R11 MOVD R8, R12 emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R11) FMOVQ F1, 16(R11) ADD $0x20, R10, R10 ADD $0x20, R11, R11 SUB $0x20, R12, R12 CMP $0x20, R12 BHS emit_lit_memmove_long_repeat_emit_encodeBlockAsm8Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeBlockAsm8B: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R8 SUBW R2, R8, R8 ADD R2, R3, R9 ADD R5, R3, R5 // matchLen MOVD $0, R11 matchlen_loopback_16_repeat_extend_encodeBlockAsm8B: CMPW $0x10, R8 BLO matchlen_match8_repeat_extend_encodeBlockAsm8B MOVD (R9)(R11), R10 ADD R11, R9, R15 MOVD 8(R15), R12 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm8B ADD R11, R5, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16repeat_extend_encodeBlockAsm8B SUB $16, R8, R8 MOVWU R8, R8 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_repeat_extend_encodeBlockAsm8B matchlen_bsf_16repeat_extend_encodeBlockAsm8B: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm8B matchlen_match8_repeat_extend_encodeBlockAsm8B: CMPW $0x08, R8 BLO matchlen_match4_repeat_extend_encodeBlockAsm8B MOVD (R9)(R11), R10 MOVD (R5)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_repeat_extend_encodeBlockAsm8B SUB $8, R8, R8 MOVWU R8, R8 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_repeat_extend_encodeBlockAsm8B matchlen_bsf_8_repeat_extend_encodeBlockAsm8B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP repeat_extend_forward_end_encodeBlockAsm8B matchlen_match4_repeat_extend_encodeBlockAsm8B: CMPW $0x04, R8 BLO matchlen_match2_repeat_extend_encodeBlockAsm8B MOVWU (R9)(R11), R10 MOVWU (R5)(R11), R16 CMPW R10, R16 BNE matchlen_match2_repeat_extend_encodeBlockAsm8B SUB $4, R8, R8 MOVWU R8, R8 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_repeat_extend_encodeBlockAsm8B: CMPW $0x01, R8 BEQ matchlen_match1_repeat_extend_encodeBlockAsm8B BLO repeat_extend_forward_end_encodeBlockAsm8B MOVHU (R9)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R5, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeBlockAsm8B ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R8, R8 BEQ repeat_extend_forward_end_encodeBlockAsm8B matchlen_match1_repeat_extend_encodeBlockAsm8B: MOVBU (R9)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R5, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeBlockAsm8B ADD $1, R11, R11 MOVWU R11, R11 repeat_extend_forward_end_encodeBlockAsm8B: ADDW R11, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 TSTW R7, R7 BEQ repeat_as_copy_encodeBlockAsm8B // emitRepeat MOVWU R5, R6 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R6 BLS repeat_two_match_repeat_encodeBlockAsm8B CMPW $0x0c, R6 BHS cant_repeat_two_offset_match_repeat_encodeBlockAsm8B cant_repeat_two_offset_match_repeat_encodeBlockAsm8B: CMPW $0x00000104, R5 BLO repeat_three_match_repeat_encodeBlockAsm8B SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B repeat_three_match_repeat_encodeBlockAsm8B: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B repeat_two_match_repeat_encodeBlockAsm8B: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B repeat_as_copy_encodeBlockAsm8B: // emitCopy CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B CMPW $0x00000800, R6 BHS long_offset_short_repeat_as_copy_encodeBlockAsm8B MOVD $0x00000001, R7 ADD $16, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 SUBW $0x08, R5, R5 // emitRepeat SUB $4, R5, R5 MOVWU R5, R5 JMP cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b MOVWU R5, R6 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R6 BLS repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b CMPW $0x0c, R6 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short_2b: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B long_offset_short_repeat_as_copy_encodeBlockAsm8B: MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 // emitRepeat MOVWU R5, R6 SUB $4, R5, R5 MOVWU R5, R5 CMPW $0x08, R6 BLS repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short CMPW $0x0c, R6 BHS cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short cant_repeat_two_offset_repeat_as_copy_encodeBlockAsm8B_emit_copy_short: CMPW $0x00000104, R5 BLO repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short SUB $256, R5, R5 MOVWU R5, R5 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R5, 2(R1) ADD $0x04, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B repeat_three_repeat_as_copy_encodeBlockAsm8B_emit_copy_short: SUB $4, R5, R5 MOVWU R5, R5 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R5, 2(R1) ADD $0x03, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B repeat_two_repeat_as_copy_encodeBlockAsm8B_emit_copy_short: LSLW $0x02, R5, R5 ORRW $0x01, R5, R5 MOVH R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B MOVD $0, R7 ADD R5<<2, R7, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVB R6, 1(R1) ASRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B two_byte_offset_short_repeat_as_copy_encodeBlockAsm8B: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeBlockAsm8B SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeBlockAsm8B emit_copy_three_repeat_as_copy_encodeBlockAsm8B: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeBlockAsm8B: MOVW R2, 20(RSP) JMP search_loop_encodeBlockAsm8B no_repeat_found_encodeBlockAsm8B: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBlockAsm8B LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeBlockAsm8B MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeBlockAsm8B MOVWU 28(RSP), R2 JMP search_loop_encodeBlockAsm8B candidate3_match_encodeBlockAsm8B: ADDW $0x02, R2, R2 JMP candidate_match_encodeBlockAsm8B candidate2_match_encodeBlockAsm8B: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBlockAsm8B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBlockAsm8B match_extend_back_loop_encodeBlockAsm8B: CMPW R6, R2 BLS match_extend_back_end_encodeBlockAsm8B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBlockAsm8B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBlockAsm8B JMP match_extend_back_loop_encodeBlockAsm8B match_extend_back_end_encodeBlockAsm8B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBlockAsm8B: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeBlockAsm8B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeBlockAsm8B CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeBlockAsm8B BLO three_bytes_match_emit_encodeBlockAsm8B three_bytes_match_emit_encodeBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBlockAsm8B two_bytes_match_emit_encodeBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeBlockAsm8B JMP memmove_long_match_emit_encodeBlockAsm8B one_byte_match_emit_encodeBlockAsm8B: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBlockAsm8B: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeBlockAsm8B emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm8B emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBlockAsm8B emit_lit_memmove_match_emit_encodeBlockAsm8B_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBlockAsm8B: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeBlockAsm8B memmove_long_match_emit_encodeBlockAsm8B: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeBlockAsm8Blarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeBlockAsm8B: match_nolit_loop_encodeBlockAsm8B: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeBlockAsm8B: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeBlockAsm8B MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm8B ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeBlockAsm8B SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeBlockAsm8B matchlen_bsf_16match_nolit_encodeBlockAsm8B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm8B matchlen_match8_match_nolit_encodeBlockAsm8B: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeBlockAsm8B MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeBlockAsm8B SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeBlockAsm8B matchlen_bsf_8_match_nolit_encodeBlockAsm8B: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeBlockAsm8B matchlen_match4_match_nolit_encodeBlockAsm8B: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeBlockAsm8B MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeBlockAsm8B SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeBlockAsm8B: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeBlockAsm8B BLO match_nolit_end_encodeBlockAsm8B MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBlockAsm8B ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeBlockAsm8B matchlen_match1_match_nolit_encodeBlockAsm8B: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeBlockAsm8B ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeBlockAsm8B: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeBlockAsm8B CMPW $0x00000800, R5 BHS long_offset_short_match_nolit_encodeBlockAsm8B MOVD $0x00000001, R6 ADD $16, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 SUBW $0x08, R9, R9 // emitRepeat SUB $4, R9, R9 MOVWU R9, R9 JMP cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b MOVWU R9, R5 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short_2b: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm8B repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short_2b: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm8B repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short_2b: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm8B MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm8B long_offset_short_match_nolit_encodeBlockAsm8B: MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 // emitRepeat MOVWU R9, R5 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBlockAsm8B_emit_copy_short: CMPW $0x00000104, R9 BLO repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R9, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm8B repeat_three_match_nolit_encodeBlockAsm8B_emit_copy_short: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R9, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm8B repeat_two_match_nolit_encodeBlockAsm8B_emit_copy_short: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm8B MOVD $0, R6 ADD R9<<2, R6, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R5, 1(R1) ASRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R9, R9 MOVB R9, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm8B two_byte_offset_short_match_nolit_encodeBlockAsm8B: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeBlockAsm8B SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBlockAsm8B emit_copy_three_match_nolit_encodeBlockAsm8B: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeBlockAsm8B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBlockAsm8B ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBlockAsm8B: MOVD $0x9e3779b1, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x20, R7, R7 MUL R8, R7, R7 LSR $0x38, R7, R7 LSL $0x20, R5, R5 MUL R8, R5, R5 LSR $0x38, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeBlockAsm8B ADDW $1, R2, R2 JMP search_loop_encodeBlockAsm8B emit_remainder_encodeBlockAsm8B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBlockAsm8B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBlockAsm8B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBlockAsm8B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBlockAsm8B BLO three_bytes_emit_remainder_encodeBlockAsm8B three_bytes_emit_remainder_encodeBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBlockAsm8B two_bytes_emit_remainder_encodeBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBlockAsm8B JMP memmove_long_emit_remainder_encodeBlockAsm8B one_byte_emit_remainder_encodeBlockAsm8B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBlockAsm8B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBlockAsm8B emit_lit_memmove_emit_remainder_encodeBlockAsm8B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBlockAsm8B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBlockAsm8B memmove_long_emit_remainder_encodeBlockAsm8B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBlockAsm8Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBlockAsm8B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int // Requires: BMI, SSE2 TEXT ·encodeBetterBlockAsm(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00001200, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBetterBlockAsm: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBetterBlockAsm MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $6, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBetterBlockAsm: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x07, R5, R5 CMPW $0x63, R5 BLS check_maxskip_ok_encodeBetterBlockAsm ADD $100, R2, R5 MOVWU R5, R5 JMP check_maxskip_cont_encodeBetterBlockAsm check_maxskip_ok_encodeBetterBlockAsm: ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 check_maxskip_cont_encodeBetterBlockAsm: MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBetterBlockAsm MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x00cf1bbcdcbfa563, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x08, R9, R9 MUL R8, R9, R9 LSR $0x2f, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x32, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 524288(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 524288(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeBetterBlockAsm CMP R6, R10 BNE no_short_found_encodeBetterBlockAsm MOVWU R7, R5 JMP candidate_match_encodeBetterBlockAsm no_short_found_encodeBetterBlockAsm: CMPW R6, R9 BEQ candidate_match_encodeBetterBlockAsm CMPW R6, R10 BEQ candidateS_match_encodeBetterBlockAsm MOVWU 28(RSP), R2 JMP search_loop_encodeBetterBlockAsm candidateS_match_encodeBetterBlockAsm: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x08, R9, R9 MUL R8, R9, R9 LSR $0x2f, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBetterBlockAsm SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBetterBlockAsm: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm match_extend_back_loop_encodeBetterBlockAsm: CMPW R6, R2 BLS match_extend_back_end_encodeBetterBlockAsm ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBetterBlockAsm SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm JMP match_extend_back_loop_encodeBetterBlockAsm match_extend_back_end_encodeBetterBlockAsm: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $5, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBetterBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBetterBlockAsm: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeBetterBlockAsm: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeBetterBlockAsm MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm matchlen_bsf_16match_nolit_encodeBetterBlockAsm: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm matchlen_match8_match_nolit_encodeBetterBlockAsm: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeBetterBlockAsm MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeBetterBlockAsm matchlen_bsf_8_match_nolit_encodeBetterBlockAsm: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm matchlen_match4_match_nolit_encodeBetterBlockAsm: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeBetterBlockAsm MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeBetterBlockAsm SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeBetterBlockAsm: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm BLO match_nolit_end_encodeBetterBlockAsm MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBetterBlockAsm ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeBetterBlockAsm matchlen_match1_match_nolit_encodeBetterBlockAsm: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeBetterBlockAsm ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeBetterBlockAsm: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat MOVWU 24(RSP), R16 CMPW R7, R16 BEQ match_is_repeat_encodeBetterBlockAsm CMPW $0x01, R11 BHI match_length_ok_encodeBetterBlockAsm CMPW $0x0000ffff, R7 BLS match_length_ok_encodeBetterBlockAsm MOVWU 28(RSP), R2 ADDW $1, R2, R2 JMP search_loop_encodeBetterBlockAsm match_length_ok_encodeBetterBlockAsm: MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeBetterBlockAsm MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeBetterBlockAsm CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeBetterBlockAsm CMPW $0x00010000, R5 BLO three_bytes_match_emit_encodeBetterBlockAsm CMPW $0x01000000, R5 BLO four_bytes_match_emit_encodeBetterBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R5, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_match_emit_encodeBetterBlockAsm four_bytes_match_emit_encodeBetterBlockAsm: MOVWU R5, R10 LSRW $0x10, R10, R10 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R5, 1(R1) MOVB R10, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_match_emit_encodeBetterBlockAsm three_bytes_match_emit_encodeBetterBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBetterBlockAsm two_bytes_match_emit_encodeBetterBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeBetterBlockAsm JMP memmove_long_match_emit_encodeBetterBlockAsm one_byte_match_emit_encodeBetterBlockAsm: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBetterBlockAsm: ADD R8, R1, R5 // genMemMoveShort CMP $0x04, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4 CMP $0x08, R8 BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4: MOVWU (R9), R10 MOVW R10, (R1) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_4through7: MOVWU (R9), R10 ADD R8, R9, R15 MOVWU -4(R15), R9 MOVW R10, (R1) ADD R8, R1, R15 MOVW R9, -4(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm emit_lit_memmove_match_emit_encodeBetterBlockAsm_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBetterBlockAsm: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeBetterBlockAsm memmove_long_match_emit_encodeBetterBlockAsm: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsmlarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeBetterBlockAsm: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy CMPW $0x00010000, R7 BLO two_byte_offset_match_nolit_encodeBetterBlockAsm CMPW $0x40, R11 BLS four_bytes_remain_match_nolit_encodeBetterBlockAsm MOVD $0xff, R16 MOVB R16, (R1) MOVW R7, 1(R1) SUB $64, R11, R11 MOVWU R11, R11 ADD $0x05, R1, R1 CMPW $0x04, R11 BLO four_bytes_remain_match_nolit_encodeBetterBlockAsm // emitRepeat emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy: MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy CMPW $0x00010100, R11 BLO repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy CMPW $0x0100ffff, R11 BLO repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy SUB $16842747, R11, R11 MOVWU R11, R11 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy: SUB $65536, R11, R11 MOVWU R11, R11 MOVWU R11, R7 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R11, 2(R1) ASRW $0x10, R7, R7 MOVB R7, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy: SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm four_bytes_remain_match_nolit_encodeBetterBlockAsm: TSTW R11, R11 BEQ match_nolit_emitcopy_end_encodeBetterBlockAsm MOVD $0, R5 ADD R11<<2, R5, R11 SUB $1, R11, R11 MOVWU R11, R11 MOVB R11, (R1) MOVW R7, 1(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm two_byte_offset_match_nolit_encodeBetterBlockAsm: CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm CMPW $0x00000800, R7 BHS long_offset_short_match_nolit_encodeBetterBlockAsm MOVD $0x00000001, R5 ADD $16, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) MOVWU R7, R8 LSRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 SUBW $0x08, R11, R11 // emitRepeat SUB $4, R11, R11 MOVWU R11, R11 JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b CMPW $0x00010100, R11 BLO repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b CMPW $0x0100ffff, R11 BLO repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b SUB $16842747, R11, R11 MOVWU R11, R11 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: SUB $65536, R11, R11 MOVWU R11, R11 MOVWU R11, R7 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R11, 2(R1) ASRW $0x10, R7, R7 MOVB R7, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short_2b: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm long_offset_short_match_nolit_encodeBetterBlockAsm: MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 // emitRepeat emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short: MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short CMPW $0x00010100, R11 BLO repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short CMPW $0x0100ffff, R11 BLO repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short SUB $16842747, R11, R11 MOVWU R11, R11 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_match_nolit_encodeBetterBlockAsm_emit_copy_short repeat_five_match_nolit_encodeBetterBlockAsm_emit_copy_short: SUB $65536, R11, R11 MOVWU R11, R11 MOVWU R11, R7 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R11, 2(R1) ASRW $0x10, R7, R7 MOVB R7, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_four_match_nolit_encodeBetterBlockAsm_emit_copy_short: SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_three_match_nolit_encodeBetterBlockAsm_emit_copy_short: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_two_match_nolit_encodeBetterBlockAsm_emit_copy_short: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_two_offset_match_nolit_encodeBetterBlockAsm_emit_copy_short: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm two_byte_offset_short_match_nolit_encodeBetterBlockAsm: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeBetterBlockAsm CMPW $0x00000800, R7 BHS emit_copy_three_match_nolit_encodeBetterBlockAsm SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm emit_copy_three_match_nolit_encodeBetterBlockAsm: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm match_is_repeat_encodeBetterBlockAsm: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_repeat_encodeBetterBlockAsm CMPW $0x00000100, R5 BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm CMPW $0x00010000, R5 BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm CMPW $0x01000000, R5 BLO four_bytes_match_emit_repeat_encodeBetterBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R5, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm four_bytes_match_emit_repeat_encodeBetterBlockAsm: MOVWU R5, R10 LSRW $0x10, R10, R10 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R5, 1(R1) MOVB R10, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm three_bytes_match_emit_repeat_encodeBetterBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm two_bytes_match_emit_repeat_encodeBetterBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_repeat_encodeBetterBlockAsm JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm one_byte_match_emit_repeat_encodeBetterBlockAsm: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_repeat_encodeBetterBlockAsm: ADD R8, R1, R5 // genMemMoveShort CMP $0x04, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4 CMP $0x08, R8 BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64 emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4: MOVWU (R9), R10 MOVW R10, (R1) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_4through7: MOVWU (R9), R10 ADD R8, R9, R15 MOVWU -4(R15), R9 MOVW R10, (R1) ADD R8, R1, R15 MOVW R9, -4(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm: MOVD R5, R1 JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm memmove_long_match_emit_repeat_encodeBetterBlockAsm: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsmlarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_repeat_encodeBetterBlockAsm: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitRepeat emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm: MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm CMPW $0x00010100, R11 BLO repeat_four_match_nolit_repeat_encodeBetterBlockAsm CMPW $0x0100ffff, R11 BLO repeat_five_match_nolit_repeat_encodeBetterBlockAsm SUB $16842747, R11, R11 MOVWU R11, R11 MOVD $0xfffb001d, R16 MOVW R16, (R1) MOVD $0xff, R16 MOVB R16, 4(R1) ADD $0x05, R1, R1 JMP emit_repeat_again_match_nolit_repeat_encodeBetterBlockAsm repeat_five_match_nolit_repeat_encodeBetterBlockAsm: SUB $65536, R11, R11 MOVWU R11, R11 MOVWU R11, R7 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R11, 2(R1) ASRW $0x10, R7, R7 MOVB R7, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_four_match_nolit_repeat_encodeBetterBlockAsm: SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_three_match_nolit_repeat_encodeBetterBlockAsm: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_two_match_nolit_repeat_encodeBetterBlockAsm: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 match_nolit_emitcopy_end_encodeBetterBlockAsm: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBetterBlockAsm MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBetterBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBetterBlockAsm: MOVD $0x00cf1bbcdcbfa563, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x08, R9, R9 MUL R5, R9, R9 LSR $0x2f, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x32, R10, R10 LSL $0x08, R11, R11 MUL R5, R11, R11 LSR $0x2f, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x32, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 524288(R15) ADD R12<<2, R0, R15 MOVW R13, 524288(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeBetterBlockAsm: CMP R8, R7 BHS search_loop_encodeBetterBlockAsm MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x08, R9, R9 MUL R5, R9, R9 LSR $0x2f, R9, R9 LSL $0x08, R10, R10 MUL R5, R10, R10 LSR $0x2f, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeBetterBlockAsm emit_remainder_encodeBetterBlockAsm: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $5, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBetterBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBetterBlockAsm: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBetterBlockAsm CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBetterBlockAsm CMPW $0x00010000, R2 BLO three_bytes_emit_remainder_encodeBetterBlockAsm CMPW $0x01000000, R2 BLO four_bytes_emit_remainder_encodeBetterBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R2, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_emit_remainder_encodeBetterBlockAsm four_bytes_emit_remainder_encodeBetterBlockAsm: MOVWU R2, R3 LSRW $0x10, R3, R3 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R2, 1(R1) MOVB R3, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_emit_remainder_encodeBetterBlockAsm three_bytes_emit_remainder_encodeBetterBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBetterBlockAsm two_bytes_emit_remainder_encodeBetterBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBetterBlockAsm JMP memmove_long_emit_remainder_encodeBetterBlockAsm one_byte_emit_remainder_encodeBetterBlockAsm: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBetterBlockAsm: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm emit_lit_memmove_emit_remainder_encodeBetterBlockAsm_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBetterBlockAsm: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm memmove_long_emit_remainder_encodeBetterBlockAsm: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsmlarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBetterBlockAsm: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeBetterBlockAsm4MB(dst []byte, src []byte, tmp *[589824]byte) int // Requires: BMI, SSE2 TEXT ·encodeBetterBlockAsm4MB(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00001200, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBetterBlockAsm4MB: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBetterBlockAsm4MB MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $6, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBetterBlockAsm4MB: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x07, R5, R5 CMPW $0x63, R5 BLS check_maxskip_ok_encodeBetterBlockAsm4MB ADD $100, R2, R5 MOVWU R5, R5 JMP check_maxskip_cont_encodeBetterBlockAsm4MB check_maxskip_ok_encodeBetterBlockAsm4MB: ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 check_maxskip_cont_encodeBetterBlockAsm4MB: MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBetterBlockAsm4MB MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x00cf1bbcdcbfa563, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x08, R9, R9 MUL R8, R9, R9 LSR $0x2f, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x32, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 524288(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 524288(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeBetterBlockAsm4MB CMP R6, R10 BNE no_short_found_encodeBetterBlockAsm4MB MOVWU R7, R5 JMP candidate_match_encodeBetterBlockAsm4MB no_short_found_encodeBetterBlockAsm4MB: CMPW R6, R9 BEQ candidate_match_encodeBetterBlockAsm4MB CMPW R6, R10 BEQ candidateS_match_encodeBetterBlockAsm4MB MOVWU 28(RSP), R2 JMP search_loop_encodeBetterBlockAsm4MB candidateS_match_encodeBetterBlockAsm4MB: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x08, R9, R9 MUL R8, R9, R9 LSR $0x2f, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBetterBlockAsm4MB SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBetterBlockAsm4MB: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm4MB match_extend_back_loop_encodeBetterBlockAsm4MB: CMPW R6, R2 BLS match_extend_back_end_encodeBetterBlockAsm4MB ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBetterBlockAsm4MB SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm4MB JMP match_extend_back_loop_encodeBetterBlockAsm4MB match_extend_back_end_encodeBetterBlockAsm4MB: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $4, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBetterBlockAsm4MB MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBetterBlockAsm4MB: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeBetterBlockAsm4MB MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm4MB matchlen_bsf_16match_nolit_encodeBetterBlockAsm4MB: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm4MB matchlen_match8_match_nolit_encodeBetterBlockAsm4MB: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeBetterBlockAsm4MB MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeBetterBlockAsm4MB matchlen_bsf_8_match_nolit_encodeBetterBlockAsm4MB: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm4MB matchlen_match4_match_nolit_encodeBetterBlockAsm4MB: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeBetterBlockAsm4MB MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeBetterBlockAsm4MB SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeBetterBlockAsm4MB: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm4MB BLO match_nolit_end_encodeBetterBlockAsm4MB MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBetterBlockAsm4MB ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeBetterBlockAsm4MB matchlen_match1_match_nolit_encodeBetterBlockAsm4MB: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeBetterBlockAsm4MB ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeBetterBlockAsm4MB: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat MOVWU 24(RSP), R16 CMPW R7, R16 BEQ match_is_repeat_encodeBetterBlockAsm4MB CMPW $0x01, R11 BHI match_length_ok_encodeBetterBlockAsm4MB CMPW $0x0000ffff, R7 BLS match_length_ok_encodeBetterBlockAsm4MB MOVWU 28(RSP), R2 ADDW $1, R2, R2 JMP search_loop_encodeBetterBlockAsm4MB match_length_ok_encodeBetterBlockAsm4MB: MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeBetterBlockAsm4MB MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeBetterBlockAsm4MB CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeBetterBlockAsm4MB CMPW $0x00010000, R5 BLO three_bytes_match_emit_encodeBetterBlockAsm4MB MOVWU R5, R10 LSRW $0x10, R10, R10 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R5, 1(R1) MOVB R10, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_match_emit_encodeBetterBlockAsm4MB three_bytes_match_emit_encodeBetterBlockAsm4MB: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBetterBlockAsm4MB two_bytes_match_emit_encodeBetterBlockAsm4MB: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeBetterBlockAsm4MB JMP memmove_long_match_emit_encodeBetterBlockAsm4MB one_byte_match_emit_encodeBetterBlockAsm4MB: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBetterBlockAsm4MB: ADD R8, R1, R5 // genMemMoveShort CMP $0x04, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4 CMP $0x08, R8 BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4: MOVWU (R9), R10 MOVW R10, (R1) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_4through7: MOVWU (R9), R10 ADD R8, R9, R15 MOVWU -4(R15), R9 MOVW R10, (R1) ADD R8, R1, R15 MOVW R9, -4(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm4MB emit_lit_memmove_match_emit_encodeBetterBlockAsm4MB_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBetterBlockAsm4MB: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeBetterBlockAsm4MB memmove_long_match_emit_encodeBetterBlockAsm4MB: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm4MBlarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeBetterBlockAsm4MB: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy CMPW $0x00010000, R7 BLO two_byte_offset_match_nolit_encodeBetterBlockAsm4MB CMPW $0x40, R11 BLS four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB MOVD $0xff, R16 MOVB R16, (R1) MOVW R7, 1(R1) SUB $64, R11, R11 MOVWU R11, R11 ADD $0x05, R1, R1 CMPW $0x04, R11 BLO four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB // emitRepeat MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy CMPW $0x00010100, R11 BLO repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy SUB $65536, R11, R11 MOVWU R11, R11 MOVWU R11, R7 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R11, 2(R1) ASRW $0x10, R7, R7 MOVB R7, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy: SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB four_bytes_remain_match_nolit_encodeBetterBlockAsm4MB: TSTW R11, R11 BEQ match_nolit_emitcopy_end_encodeBetterBlockAsm4MB MOVD $0, R5 ADD R11<<2, R5, R11 SUB $1, R11, R11 MOVWU R11, R11 MOVB R11, (R1) MOVW R7, 1(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB two_byte_offset_match_nolit_encodeBetterBlockAsm4MB: CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB CMPW $0x00000800, R7 BHS long_offset_short_match_nolit_encodeBetterBlockAsm4MB MOVD $0x00000001, R5 ADD $16, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 SUBW $0x08, R11, R11 // emitRepeat SUB $4, R11, R11 MOVWU R11, R11 JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b CMPW $0x00010100, R11 BLO repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b SUB $65536, R11, R11 MOVWU R11, R11 MOVWU R11, R7 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R11, 2(R1) ASRW $0x10, R7, R7 MOVB R7, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short_2b: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB long_offset_short_match_nolit_encodeBetterBlockAsm4MB: MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 // emitRepeat MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short CMPW $0x00010100, R11 BLO repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short SUB $65536, R11, R11 MOVWU R11, R11 MOVWU R11, R7 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R11, 2(R1) ASRW $0x10, R7, R7 MOVB R7, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_four_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_three_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_two_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_two_offset_match_nolit_encodeBetterBlockAsm4MB_emit_copy_short: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB two_byte_offset_short_match_nolit_encodeBetterBlockAsm4MB: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeBetterBlockAsm4MB CMPW $0x00000800, R7 BHS emit_copy_three_match_nolit_encodeBetterBlockAsm4MB SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB emit_copy_three_match_nolit_encodeBetterBlockAsm4MB: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB match_is_repeat_encodeBetterBlockAsm4MB: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_repeat_encodeBetterBlockAsm4MB CMPW $0x00000100, R5 BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB CMPW $0x00010000, R5 BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB MOVWU R5, R10 LSRW $0x10, R10, R10 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R5, 1(R1) MOVB R10, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB three_bytes_match_emit_repeat_encodeBetterBlockAsm4MB: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB two_bytes_match_emit_repeat_encodeBetterBlockAsm4MB: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_repeat_encodeBetterBlockAsm4MB JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB one_byte_match_emit_repeat_encodeBetterBlockAsm4MB: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_repeat_encodeBetterBlockAsm4MB: ADD R8, R1, R5 // genMemMoveShort CMP $0x04, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4 CMP $0x08, R8 BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32 JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64 emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4: MOVWU (R9), R10 MOVW R10, (R1) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_4through7: MOVWU (R9), R10 ADD R8, R9, R15 MOVWU -4(R15), R9 MOVW R10, (R1) ADD R8, R1, R15 MOVW R9, -4(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm4MB_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm4MB: MOVD R5, R1 JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB memmove_long_match_emit_repeat_encodeBetterBlockAsm4MB: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm4MBlarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_repeat_encodeBetterBlockAsm4MB: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitRepeat MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB CMPW $0x00010100, R11 BLO repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB SUB $65536, R11, R11 MOVWU R11, R11 MOVWU R11, R7 MOVD $0x001d, R16 MOVH R16, (R1) MOVH R11, 2(R1) ASRW $0x10, R7, R7 MOVB R7, 4(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_four_match_nolit_repeat_encodeBetterBlockAsm4MB: SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_three_match_nolit_repeat_encodeBetterBlockAsm4MB: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_two_match_nolit_repeat_encodeBetterBlockAsm4MB: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm4MB repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm4MB: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 match_nolit_emitcopy_end_encodeBetterBlockAsm4MB: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBetterBlockAsm4MB MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBetterBlockAsm4MB MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBetterBlockAsm4MB: MOVD $0x00cf1bbcdcbfa563, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x08, R9, R9 MUL R5, R9, R9 LSR $0x2f, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x32, R10, R10 LSL $0x08, R11, R11 MUL R5, R11, R11 LSR $0x2f, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x32, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 524288(R15) ADD R12<<2, R0, R15 MOVW R13, 524288(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeBetterBlockAsm4MB: CMP R8, R7 BHS search_loop_encodeBetterBlockAsm4MB MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x08, R9, R9 MUL R5, R9, R9 LSR $0x2f, R9, R9 LSL $0x08, R10, R10 MUL R5, R10, R10 LSR $0x2f, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeBetterBlockAsm4MB emit_remainder_encodeBetterBlockAsm4MB: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $4, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBetterBlockAsm4MB MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBetterBlockAsm4MB: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBetterBlockAsm4MB CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBetterBlockAsm4MB CMPW $0x00010000, R2 BLO three_bytes_emit_remainder_encodeBetterBlockAsm4MB MOVWU R2, R3 LSRW $0x10, R3, R3 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R2, 1(R1) MOVB R3, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB three_bytes_emit_remainder_encodeBetterBlockAsm4MB: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB two_bytes_emit_remainder_encodeBetterBlockAsm4MB: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBetterBlockAsm4MB JMP memmove_long_emit_remainder_encodeBetterBlockAsm4MB one_byte_emit_remainder_encodeBetterBlockAsm4MB: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBetterBlockAsm4MB: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB emit_lit_memmove_emit_remainder_encodeBetterBlockAsm4MB_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBetterBlockAsm4MB: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB memmove_long_emit_remainder_encodeBetterBlockAsm4MB: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm4MBlarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBetterBlockAsm4MB: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int // Requires: BMI, SSE2 TEXT ·encodeBetterBlockAsm12B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000280, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBetterBlockAsm12B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBetterBlockAsm12B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $6, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBetterBlockAsm12B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x06, R5, R5 ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBetterBlockAsm12B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x34, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 65536(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 65536(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeBetterBlockAsm12B CMP R6, R10 BNE no_short_found_encodeBetterBlockAsm12B MOVWU R7, R5 JMP candidate_match_encodeBetterBlockAsm12B no_short_found_encodeBetterBlockAsm12B: CMPW R6, R9 BEQ candidate_match_encodeBetterBlockAsm12B CMPW R6, R10 BEQ candidateS_match_encodeBetterBlockAsm12B MOVWU 28(RSP), R2 JMP search_loop_encodeBetterBlockAsm12B candidateS_match_encodeBetterBlockAsm12B: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBetterBlockAsm12B SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBetterBlockAsm12B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm12B match_extend_back_loop_encodeBetterBlockAsm12B: CMPW R6, R2 BLS match_extend_back_end_encodeBetterBlockAsm12B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBetterBlockAsm12B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm12B JMP match_extend_back_loop_encodeBetterBlockAsm12B match_extend_back_end_encodeBetterBlockAsm12B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBetterBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBetterBlockAsm12B: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeBetterBlockAsm12B MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm12B matchlen_bsf_16match_nolit_encodeBetterBlockAsm12B: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm12B matchlen_match8_match_nolit_encodeBetterBlockAsm12B: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeBetterBlockAsm12B MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeBetterBlockAsm12B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm12B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm12B matchlen_match4_match_nolit_encodeBetterBlockAsm12B: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeBetterBlockAsm12B MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeBetterBlockAsm12B SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeBetterBlockAsm12B: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm12B BLO match_nolit_end_encodeBetterBlockAsm12B MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBetterBlockAsm12B ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeBetterBlockAsm12B matchlen_match1_match_nolit_encodeBetterBlockAsm12B: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeBetterBlockAsm12B ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeBetterBlockAsm12B: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat MOVWU 24(RSP), R16 CMPW R7, R16 BEQ match_is_repeat_encodeBetterBlockAsm12B MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeBetterBlockAsm12B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeBetterBlockAsm12B CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeBetterBlockAsm12B BLO three_bytes_match_emit_encodeBetterBlockAsm12B three_bytes_match_emit_encodeBetterBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBetterBlockAsm12B two_bytes_match_emit_encodeBetterBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeBetterBlockAsm12B JMP memmove_long_match_emit_encodeBetterBlockAsm12B one_byte_match_emit_encodeBetterBlockAsm12B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBetterBlockAsm12B: ADD R8, R1, R5 // genMemMoveShort CMP $0x04, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4 CMP $0x08, R8 BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4: MOVWU (R9), R10 MOVW R10, (R1) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_4through7: MOVWU (R9), R10 ADD R8, R9, R15 MOVWU -4(R15), R9 MOVW R10, (R1) ADD R8, R1, R15 MOVW R9, -4(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm12B emit_lit_memmove_match_emit_encodeBetterBlockAsm12B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBetterBlockAsm12B: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeBetterBlockAsm12B memmove_long_match_emit_encodeBetterBlockAsm12B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm12Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeBetterBlockAsm12B: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B CMPW $0x00000800, R7 BHS long_offset_short_match_nolit_encodeBetterBlockAsm12B MOVD $0x00000001, R5 ADD $16, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 SUBW $0x08, R11, R11 // emitRepeat SUB $4, R11, R11 MOVWU R11, R11 JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short_2b: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B long_offset_short_match_nolit_encodeBetterBlockAsm12B: MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 // emitRepeat MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B repeat_three_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B repeat_two_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B repeat_two_offset_match_nolit_encodeBetterBlockAsm12B_emit_copy_short: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B two_byte_offset_short_match_nolit_encodeBetterBlockAsm12B: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeBetterBlockAsm12B CMPW $0x00000800, R7 BHS emit_copy_three_match_nolit_encodeBetterBlockAsm12B SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B emit_copy_three_match_nolit_encodeBetterBlockAsm12B: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B match_is_repeat_encodeBetterBlockAsm12B: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_repeat_encodeBetterBlockAsm12B CMPW $0x00000100, R5 BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm12B BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm12B three_bytes_match_emit_repeat_encodeBetterBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B two_bytes_match_emit_repeat_encodeBetterBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_repeat_encodeBetterBlockAsm12B JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm12B one_byte_match_emit_repeat_encodeBetterBlockAsm12B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_repeat_encodeBetterBlockAsm12B: ADD R8, R1, R5 // genMemMoveShort CMP $0x04, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4 CMP $0x08, R8 BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64 emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4: MOVWU (R9), R10 MOVW R10, (R1) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_4through7: MOVWU (R9), R10 ADD R8, R9, R15 MOVWU -4(R15), R9 MOVW R10, (R1) ADD R8, R1, R15 MOVW R9, -4(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm12B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm12B: MOVD R5, R1 JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B memmove_long_match_emit_repeat_encodeBetterBlockAsm12B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm12Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_repeat_encodeBetterBlockAsm12B: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitRepeat MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B repeat_three_match_nolit_repeat_encodeBetterBlockAsm12B: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B repeat_two_match_nolit_repeat_encodeBetterBlockAsm12B: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm12B repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm12B: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 match_nolit_emitcopy_end_encodeBetterBlockAsm12B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBetterBlockAsm12B MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBetterBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBetterBlockAsm12B: MOVD $0x0000cf1bbcdcbf9b, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x32, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x34, R10, R10 LSL $0x10, R11, R11 MUL R5, R11, R11 LSR $0x32, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x34, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 65536(R15) ADD R12<<2, R0, R15 MOVW R13, 65536(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeBetterBlockAsm12B: CMP R8, R7 BHS search_loop_encodeBetterBlockAsm12B MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x32, R9, R9 LSL $0x10, R10, R10 MUL R5, R10, R10 LSR $0x32, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeBetterBlockAsm12B emit_remainder_encodeBetterBlockAsm12B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBetterBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBetterBlockAsm12B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm12B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBetterBlockAsm12B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBetterBlockAsm12B BLO three_bytes_emit_remainder_encodeBetterBlockAsm12B three_bytes_emit_remainder_encodeBetterBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B two_bytes_emit_remainder_encodeBetterBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBetterBlockAsm12B JMP memmove_long_emit_remainder_encodeBetterBlockAsm12B one_byte_emit_remainder_encodeBetterBlockAsm12B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBetterBlockAsm12B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm12B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBetterBlockAsm12B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm12B memmove_long_emit_remainder_encodeBetterBlockAsm12B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm12Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBetterBlockAsm12B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int // Requires: BMI, SSE2 TEXT ·encodeBetterBlockAsm10B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x000000a0, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBetterBlockAsm10B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBetterBlockAsm10B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $6, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBetterBlockAsm10B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x05, R5, R5 ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBetterBlockAsm10B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x34, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x36, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 16384(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 16384(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeBetterBlockAsm10B CMP R6, R10 BNE no_short_found_encodeBetterBlockAsm10B MOVWU R7, R5 JMP candidate_match_encodeBetterBlockAsm10B no_short_found_encodeBetterBlockAsm10B: CMPW R6, R9 BEQ candidate_match_encodeBetterBlockAsm10B CMPW R6, R10 BEQ candidateS_match_encodeBetterBlockAsm10B MOVWU 28(RSP), R2 JMP search_loop_encodeBetterBlockAsm10B candidateS_match_encodeBetterBlockAsm10B: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x34, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBetterBlockAsm10B SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBetterBlockAsm10B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm10B match_extend_back_loop_encodeBetterBlockAsm10B: CMPW R6, R2 BLS match_extend_back_end_encodeBetterBlockAsm10B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBetterBlockAsm10B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm10B JMP match_extend_back_loop_encodeBetterBlockAsm10B match_extend_back_end_encodeBetterBlockAsm10B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBetterBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBetterBlockAsm10B: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeBetterBlockAsm10B MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm10B matchlen_bsf_16match_nolit_encodeBetterBlockAsm10B: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm10B matchlen_match8_match_nolit_encodeBetterBlockAsm10B: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeBetterBlockAsm10B MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeBetterBlockAsm10B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm10B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm10B matchlen_match4_match_nolit_encodeBetterBlockAsm10B: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeBetterBlockAsm10B MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeBetterBlockAsm10B SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeBetterBlockAsm10B: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm10B BLO match_nolit_end_encodeBetterBlockAsm10B MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBetterBlockAsm10B ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeBetterBlockAsm10B matchlen_match1_match_nolit_encodeBetterBlockAsm10B: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeBetterBlockAsm10B ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeBetterBlockAsm10B: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat MOVWU 24(RSP), R16 CMPW R7, R16 BEQ match_is_repeat_encodeBetterBlockAsm10B MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeBetterBlockAsm10B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeBetterBlockAsm10B CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeBetterBlockAsm10B BLO three_bytes_match_emit_encodeBetterBlockAsm10B three_bytes_match_emit_encodeBetterBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBetterBlockAsm10B two_bytes_match_emit_encodeBetterBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeBetterBlockAsm10B JMP memmove_long_match_emit_encodeBetterBlockAsm10B one_byte_match_emit_encodeBetterBlockAsm10B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBetterBlockAsm10B: ADD R8, R1, R5 // genMemMoveShort CMP $0x04, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4 CMP $0x08, R8 BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4: MOVWU (R9), R10 MOVW R10, (R1) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_4through7: MOVWU (R9), R10 ADD R8, R9, R15 MOVWU -4(R15), R9 MOVW R10, (R1) ADD R8, R1, R15 MOVW R9, -4(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm10B emit_lit_memmove_match_emit_encodeBetterBlockAsm10B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBetterBlockAsm10B: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeBetterBlockAsm10B memmove_long_match_emit_encodeBetterBlockAsm10B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm10Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeBetterBlockAsm10B: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B CMPW $0x00000800, R7 BHS long_offset_short_match_nolit_encodeBetterBlockAsm10B MOVD $0x00000001, R5 ADD $16, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 SUBW $0x08, R11, R11 // emitRepeat SUB $4, R11, R11 MOVWU R11, R11 JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short_2b: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B long_offset_short_match_nolit_encodeBetterBlockAsm10B: MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 // emitRepeat MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B repeat_three_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B repeat_two_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B repeat_two_offset_match_nolit_encodeBetterBlockAsm10B_emit_copy_short: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B two_byte_offset_short_match_nolit_encodeBetterBlockAsm10B: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeBetterBlockAsm10B CMPW $0x00000800, R7 BHS emit_copy_three_match_nolit_encodeBetterBlockAsm10B SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B emit_copy_three_match_nolit_encodeBetterBlockAsm10B: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B match_is_repeat_encodeBetterBlockAsm10B: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_repeat_encodeBetterBlockAsm10B CMPW $0x00000100, R5 BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm10B BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm10B three_bytes_match_emit_repeat_encodeBetterBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B two_bytes_match_emit_repeat_encodeBetterBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_repeat_encodeBetterBlockAsm10B JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm10B one_byte_match_emit_repeat_encodeBetterBlockAsm10B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_repeat_encodeBetterBlockAsm10B: ADD R8, R1, R5 // genMemMoveShort CMP $0x04, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4 CMP $0x08, R8 BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64 emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4: MOVWU (R9), R10 MOVW R10, (R1) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_4through7: MOVWU (R9), R10 ADD R8, R9, R15 MOVWU -4(R15), R9 MOVW R10, (R1) ADD R8, R1, R15 MOVW R9, -4(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm10B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm10B: MOVD R5, R1 JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B memmove_long_match_emit_repeat_encodeBetterBlockAsm10B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm10Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_repeat_encodeBetterBlockAsm10B: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitRepeat MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B CMPW $0x00000800, R7 BLO repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B repeat_three_match_nolit_repeat_encodeBetterBlockAsm10B: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B repeat_two_match_nolit_repeat_encodeBetterBlockAsm10B: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm10B repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm10B: MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 match_nolit_emitcopy_end_encodeBetterBlockAsm10B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBetterBlockAsm10B MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBetterBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBetterBlockAsm10B: MOVD $0x0000cf1bbcdcbf9b, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x34, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x36, R10, R10 LSL $0x10, R11, R11 MUL R5, R11, R11 LSR $0x34, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x36, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 16384(R15) ADD R12<<2, R0, R15 MOVW R13, 16384(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeBetterBlockAsm10B: CMP R8, R7 BHS search_loop_encodeBetterBlockAsm10B MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x34, R9, R9 LSL $0x10, R10, R10 MUL R5, R10, R10 LSR $0x34, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeBetterBlockAsm10B emit_remainder_encodeBetterBlockAsm10B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBetterBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBetterBlockAsm10B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm10B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBetterBlockAsm10B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBetterBlockAsm10B BLO three_bytes_emit_remainder_encodeBetterBlockAsm10B three_bytes_emit_remainder_encodeBetterBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B two_bytes_emit_remainder_encodeBetterBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBetterBlockAsm10B JMP memmove_long_emit_remainder_encodeBetterBlockAsm10B one_byte_emit_remainder_encodeBetterBlockAsm10B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBetterBlockAsm10B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm10B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBetterBlockAsm10B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm10B memmove_long_emit_remainder_encodeBetterBlockAsm10B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm10Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBetterBlockAsm10B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int // Requires: BMI, SSE2 TEXT ·encodeBetterBlockAsm8B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000028, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeBetterBlockAsm8B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeBetterBlockAsm8B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $6, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeBetterBlockAsm8B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x04, R5, R5 ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeBetterBlockAsm8B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x36, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x38, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 4096(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 4096(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeBetterBlockAsm8B CMP R6, R10 BNE no_short_found_encodeBetterBlockAsm8B MOVWU R7, R5 JMP candidate_match_encodeBetterBlockAsm8B no_short_found_encodeBetterBlockAsm8B: CMPW R6, R9 BEQ candidate_match_encodeBetterBlockAsm8B CMPW R6, R10 BEQ candidateS_match_encodeBetterBlockAsm8B MOVWU 28(RSP), R2 JMP search_loop_encodeBetterBlockAsm8B candidateS_match_encodeBetterBlockAsm8B: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x36, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeBetterBlockAsm8B SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeBetterBlockAsm8B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm8B match_extend_back_loop_encodeBetterBlockAsm8B: CMPW R6, R2 BLS match_extend_back_end_encodeBetterBlockAsm8B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeBetterBlockAsm8B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeBetterBlockAsm8B JMP match_extend_back_loop_encodeBetterBlockAsm8B match_extend_back_end_encodeBetterBlockAsm8B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeBetterBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeBetterBlockAsm8B: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeBetterBlockAsm8B MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeBetterBlockAsm8B matchlen_bsf_16match_nolit_encodeBetterBlockAsm8B: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm8B matchlen_match8_match_nolit_encodeBetterBlockAsm8B: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeBetterBlockAsm8B MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeBetterBlockAsm8B matchlen_bsf_8_match_nolit_encodeBetterBlockAsm8B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeBetterBlockAsm8B matchlen_match4_match_nolit_encodeBetterBlockAsm8B: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeBetterBlockAsm8B MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeBetterBlockAsm8B SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeBetterBlockAsm8B: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeBetterBlockAsm8B BLO match_nolit_end_encodeBetterBlockAsm8B MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeBetterBlockAsm8B ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeBetterBlockAsm8B matchlen_match1_match_nolit_encodeBetterBlockAsm8B: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeBetterBlockAsm8B ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeBetterBlockAsm8B: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat MOVWU 24(RSP), R16 CMPW R7, R16 BEQ match_is_repeat_encodeBetterBlockAsm8B MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeBetterBlockAsm8B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeBetterBlockAsm8B CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeBetterBlockAsm8B BLO three_bytes_match_emit_encodeBetterBlockAsm8B three_bytes_match_emit_encodeBetterBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeBetterBlockAsm8B two_bytes_match_emit_encodeBetterBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeBetterBlockAsm8B JMP memmove_long_match_emit_encodeBetterBlockAsm8B one_byte_match_emit_encodeBetterBlockAsm8B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeBetterBlockAsm8B: ADD R8, R1, R5 // genMemMoveShort CMP $0x04, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4 CMP $0x08, R8 BLO emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4: MOVWU (R9), R10 MOVW R10, (R1) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_4through7: MOVWU (R9), R10 ADD R8, R9, R15 MOVWU -4(R15), R9 MOVW R10, (R1) ADD R8, R1, R15 MOVW R9, -4(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeBetterBlockAsm8B emit_lit_memmove_match_emit_encodeBetterBlockAsm8B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeBetterBlockAsm8B: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeBetterBlockAsm8B memmove_long_match_emit_encodeBetterBlockAsm8B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeBetterBlockAsm8Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeBetterBlockAsm8B: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B CMPW $0x00000800, R7 BHS long_offset_short_match_nolit_encodeBetterBlockAsm8B MOVD $0x00000001, R5 ADD $16, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 SUBW $0x08, R11, R11 // emitRepeat SUB $4, R11, R11 MOVWU R11, R11 JMP cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short_2b: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B long_offset_short_match_nolit_encodeBetterBlockAsm8B: MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 // emitRepeat MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short cant_repeat_two_offset_match_nolit_encodeBetterBlockAsm8B_emit_copy_short: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B repeat_three_match_nolit_encodeBetterBlockAsm8B_emit_copy_short: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B repeat_two_match_nolit_encodeBetterBlockAsm8B_emit_copy_short: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B two_byte_offset_short_match_nolit_encodeBetterBlockAsm8B: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeBetterBlockAsm8B SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B emit_copy_three_match_nolit_encodeBetterBlockAsm8B: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B match_is_repeat_encodeBetterBlockAsm8B: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B MOVWU R6, R7 MOVW R6, 20(RSP) ADD R5, R3, R8 SUBW R5, R7, R7 SUB $1, R7, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_repeat_encodeBetterBlockAsm8B CMPW $0x00000100, R5 BLO two_bytes_match_emit_repeat_encodeBetterBlockAsm8B BLO three_bytes_match_emit_repeat_encodeBetterBlockAsm8B three_bytes_match_emit_repeat_encodeBetterBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B two_bytes_match_emit_repeat_encodeBetterBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_repeat_encodeBetterBlockAsm8B JMP memmove_long_match_emit_repeat_encodeBetterBlockAsm8B one_byte_match_emit_repeat_encodeBetterBlockAsm8B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_repeat_encodeBetterBlockAsm8B: ADD R7, R1, R5 // genMemMoveShort CMP $0x04, R7 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4 CMP $0x08, R7 BLO emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7 CMP $0x10, R7 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16 CMP $0x20, R7 BLS emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64 emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4: MOVWU (R8), R9 MOVW R9, (R1) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_4through7: MOVWU (R8), R9 ADD R7, R8, R15 MOVWU -4(R15), R8 MOVW R9, (R1) ADD R7, R1, R15 MOVW R8, -4(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_8through16: MOVD (R8), R9 ADD R7, R8, R15 MOVD -8(R15), R8 MOVD R9, (R1) ADD R7, R1, R15 MOVD R8, -8(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_17through32: FMOVQ (R8), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R7, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B emit_lit_memmove_match_emit_repeat_encodeBetterBlockAsm8B_memmove_move_33through64: FMOVQ (R8), F0 FMOVQ 16(R8), F1 ADD R7, R8, R15 FMOVQ -32(R15), F2 ADD R7, R8, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R7, R1, R15 FMOVQ F2, -32(R15) ADD R7, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_repeat_encodeBetterBlockAsm8B: MOVD R5, R1 JMP emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B memmove_long_match_emit_repeat_encodeBetterBlockAsm8B: ADD R7, R1, R5 // genMemMoveLong MOVD R8, R9 MOVD R1, R10 MOVD R7, R12 emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R12, R12 CMP $0x20, R12 BHS emit_lit_memmove_long_match_emit_repeat_encodeBetterBlockAsm8Blarge_big_loop_back ADD R7, R8, R15 FMOVQ -32(R15), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 ADD R7, R1, R15 FMOVQ F0, -32(R15) ADD R7, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_repeat_encodeBetterBlockAsm8B: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitRepeat MOVWU R11, R5 SUB $4, R11, R11 MOVWU R11, R11 CMPW $0x08, R5 BLS repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B CMPW $0x0c, R5 BHS cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B cant_repeat_two_offset_match_nolit_repeat_encodeBetterBlockAsm8B: CMPW $0x00000104, R11 BLO repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B SUB $256, R11, R11 MOVWU R11, R11 MOVD $0x0019, R16 MOVH R16, (R1) MOVH R11, 2(R1) ADD $0x04, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B repeat_three_match_nolit_repeat_encodeBetterBlockAsm8B: SUB $4, R11, R11 MOVWU R11, R11 MOVD $0x0015, R16 MOVH R16, (R1) MOVB R11, 2(R1) ADD $0x03, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B repeat_two_match_nolit_repeat_encodeBetterBlockAsm8B: LSLW $0x02, R11, R11 ORRW $0x01, R11, R11 MOVH R11, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeBetterBlockAsm8B MOVD $0, R5 ADD R11<<2, R5, R11 ADD $1, R11, R11 MOVWU R11, R11 MOVB R7, 1(R1) ASRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R11, R11 MOVB R11, (R1) ADD $0x02, R1, R1 match_nolit_emitcopy_end_encodeBetterBlockAsm8B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeBetterBlockAsm8B MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeBetterBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeBetterBlockAsm8B: MOVD $0x0000cf1bbcdcbf9b, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x36, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x38, R10, R10 LSL $0x10, R11, R11 MUL R5, R11, R11 LSR $0x36, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x38, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 4096(R15) ADD R12<<2, R0, R15 MOVW R13, 4096(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeBetterBlockAsm8B: CMP R8, R7 BHS search_loop_encodeBetterBlockAsm8B MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x36, R9, R9 LSL $0x10, R10, R10 MUL R5, R10, R10 LSR $0x36, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeBetterBlockAsm8B emit_remainder_encodeBetterBlockAsm8B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeBetterBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeBetterBlockAsm8B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeBetterBlockAsm8B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeBetterBlockAsm8B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeBetterBlockAsm8B BLO three_bytes_emit_remainder_encodeBetterBlockAsm8B three_bytes_emit_remainder_encodeBetterBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B two_bytes_emit_remainder_encodeBetterBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeBetterBlockAsm8B JMP memmove_long_emit_remainder_encodeBetterBlockAsm8B one_byte_emit_remainder_encodeBetterBlockAsm8B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeBetterBlockAsm8B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeBetterBlockAsm8B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeBetterBlockAsm8B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeBetterBlockAsm8B memmove_long_emit_remainder_encodeBetterBlockAsm8B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeBetterBlockAsm8Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeBetterBlockAsm8B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBlockAsm(dst []byte, src []byte, tmp *[65536]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBlockAsm(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000200, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBlockAsm: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBlockAsm MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBlockAsm: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x06, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBlockAsm MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 LSL $0x10, R10, R10 MUL R8, R10, R10 LSR $0x32, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeSnappyBlockAsm ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R5 MOVWU R6, R7 MOVWU 24(RSP), R16 SUBSW R16, R7, R7 BEQ repeat_extend_back_end_encodeSnappyBlockAsm repeat_extend_back_loop_encodeSnappyBlockAsm: CMPW R5, R6 BLS repeat_extend_back_end_encodeSnappyBlockAsm ADD R7, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeSnappyBlockAsm SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R7, R7 BNE repeat_extend_back_loop_encodeSnappyBlockAsm repeat_extend_back_end_encodeSnappyBlockAsm: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $5, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeSnappyBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeSnappyBlockAsm: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm MOVWU R6, R7 MOVW R6, 20(RSP) ADD R5, R3, R8 SUBW R5, R7, R7 SUB $1, R7, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeSnappyBlockAsm CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeSnappyBlockAsm CMPW $0x00010000, R5 BLO three_bytes_repeat_emit_encodeSnappyBlockAsm CMPW $0x01000000, R5 BLO four_bytes_repeat_emit_encodeSnappyBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R5, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_repeat_emit_encodeSnappyBlockAsm four_bytes_repeat_emit_encodeSnappyBlockAsm: MOVWU R5, R9 LSRW $0x10, R9, R9 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R5, 1(R1) MOVB R9, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_repeat_emit_encodeSnappyBlockAsm three_bytes_repeat_emit_encodeSnappyBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeSnappyBlockAsm two_bytes_repeat_emit_encodeSnappyBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeSnappyBlockAsm JMP memmove_long_repeat_emit_encodeSnappyBlockAsm one_byte_repeat_emit_encodeSnappyBlockAsm: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeSnappyBlockAsm: ADD R7, R1, R5 // genMemMoveShort CMP $0x08, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8 CMP $0x10, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16 CMP $0x20, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8: MOVD (R8), R9 MOVD R9, (R1) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_8through16: MOVD (R8), R9 ADD R7, R8, R15 MOVD -8(R15), R8 MOVD R9, (R1) ADD R7, R1, R15 MOVD R8, -8(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_17through32: FMOVQ (R8), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R7, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm_memmove_move_33through64: FMOVQ (R8), F0 FMOVQ 16(R8), F1 ADD R7, R8, R15 FMOVQ -32(R15), F2 ADD R7, R8, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R7, R1, R15 FMOVQ F2, -32(R15) ADD R7, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeSnappyBlockAsm: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm memmove_long_repeat_emit_encodeSnappyBlockAsm: ADD R7, R1, R5 // genMemMoveLong MOVD R8, R9 MOVD R1, R10 MOVD R7, R11 emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsmlarge_big_loop_back ADD R7, R8, R15 FMOVQ -32(R15), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 ADD R7, R1, R15 FMOVQ F0, -32(R15) ADD R7, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeSnappyBlockAsm: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R5 // matchLen MOVD $0, R10 matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm: CMPW $0x10, R7 BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm MOVD (R8)(R10), R9 ADD R10, R8, R15 MOVD 8(R15), R11 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm ADD R10, R5, R15 MOVD 8(R15), R16 EOR R16, R11, R11 TST R11, R11 BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R10, R10 MOVWU R10, R10 JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm: RBIT R11, R11 CLZ R11, R11 ASR $0x03, R11, R11 ADD R11, R10, R10 ADD $8, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm matchlen_match8_repeat_extend_encodeSnappyBlockAsm: CMPW $0x08, R7 BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm MOVD (R8)(R10), R9 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R10, R10 MOVWU R10, R10 JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm: RBIT R9, R9 CLZ R9, R9 ASR $0x03, R9, R9 ADD R9, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm matchlen_match4_repeat_extend_encodeSnappyBlockAsm: CMPW $0x04, R7 BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm MOVWU (R8)(R10), R9 MOVWU (R5)(R10), R16 CMPW R9, R16 BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R10, R10 MOVWU R10, R10 matchlen_match2_repeat_extend_encodeSnappyBlockAsm: CMPW $0x01, R7 BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm BLO repeat_extend_forward_end_encodeSnappyBlockAsm MOVHU (R8)(R10), R16 BFI $0, R16, $16, R9 ADD R10, R5, R15 MOVHU (R15), R15 AND $0xffff, R9, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm ADD $2, R10, R10 MOVWU R10, R10 SUBSW $0x02, R7, R7 BEQ repeat_extend_forward_end_encodeSnappyBlockAsm matchlen_match1_repeat_extend_encodeSnappyBlockAsm: MOVBU (R8)(R10), R16 BFI $0, R16, $8, R9 ADD R10, R5, R15 MOVBU (R15), R15 AND $0xff, R9, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeSnappyBlockAsm ADD $1, R10, R10 MOVWU R10, R10 repeat_extend_forward_end_encodeSnappyBlockAsm: ADDW R10, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 // emitCopy CMPW $0x00010000, R6 BLO two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm: CMPW $0x40, R5 BLS four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm MOVD $0xff, R16 MOVB R16, (R1) MOVW R6, 1(R1) SUB $64, R5, R5 MOVWU R5, R5 ADD $0x05, R1, R1 CMPW $0x04, R5 BLO four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm JMP four_bytes_loop_back_repeat_as_copy_encodeSnappyBlockAsm four_bytes_remain_repeat_as_copy_encodeSnappyBlockAsm: TSTW R5, R5 BEQ repeat_end_emit_encodeSnappyBlockAsm MOVD $0, R7 ADD R5<<2, R7, R5 SUB $1, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVW R6, 1(R1) ADD $0x05, R1, R1 JMP repeat_end_emit_encodeSnappyBlockAsm two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm: CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm CMPW $0x00000800, R6 BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeSnappyBlockAsm emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeSnappyBlockAsm: MOVW R2, 20(RSP) JMP search_loop_encodeSnappyBlockAsm no_repeat_found_encodeSnappyBlockAsm: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBlockAsm LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeSnappyBlockAsm MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeSnappyBlockAsm MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBlockAsm candidate3_match_encodeSnappyBlockAsm: ADDW $0x02, R2, R2 JMP candidate_match_encodeSnappyBlockAsm candidate2_match_encodeSnappyBlockAsm: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBlockAsm: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm match_extend_back_loop_encodeSnappyBlockAsm: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBlockAsm ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBlockAsm SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm JMP match_extend_back_loop_encodeSnappyBlockAsm match_extend_back_end_encodeSnappyBlockAsm: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $5, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBlockAsm: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeSnappyBlockAsm CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeSnappyBlockAsm CMPW $0x00010000, R7 BLO three_bytes_match_emit_encodeSnappyBlockAsm CMPW $0x01000000, R7 BLO four_bytes_match_emit_encodeSnappyBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R7, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_match_emit_encodeSnappyBlockAsm four_bytes_match_emit_encodeSnappyBlockAsm: MOVWU R7, R9 LSRW $0x10, R9, R9 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R7, 1(R1) MOVB R9, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_match_emit_encodeSnappyBlockAsm three_bytes_match_emit_encodeSnappyBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBlockAsm two_bytes_match_emit_encodeSnappyBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeSnappyBlockAsm JMP memmove_long_match_emit_encodeSnappyBlockAsm one_byte_match_emit_encodeSnappyBlockAsm: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBlockAsm: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm emit_lit_memmove_match_emit_encodeSnappyBlockAsm_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBlockAsm: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeSnappyBlockAsm memmove_long_match_emit_encodeSnappyBlockAsm: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsmlarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeSnappyBlockAsm: match_nolit_loop_encodeSnappyBlockAsm: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm matchlen_bsf_16match_nolit_encodeSnappyBlockAsm: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm matchlen_match8_match_nolit_encodeSnappyBlockAsm: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm matchlen_match4_match_nolit_encodeSnappyBlockAsm: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeSnappyBlockAsm: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm BLO match_nolit_end_encodeSnappyBlockAsm MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeSnappyBlockAsm matchlen_match1_match_nolit_encodeSnappyBlockAsm: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBlockAsm ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeSnappyBlockAsm: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy CMPW $0x00010000, R5 BLO two_byte_offset_match_nolit_encodeSnappyBlockAsm four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm: CMPW $0x40, R9 BLS four_bytes_remain_match_nolit_encodeSnappyBlockAsm MOVD $0xff, R16 MOVB R16, (R1) MOVW R5, 1(R1) SUB $64, R9, R9 MOVWU R9, R9 ADD $0x05, R1, R1 CMPW $0x04, R9 BLO four_bytes_remain_match_nolit_encodeSnappyBlockAsm JMP four_bytes_loop_back_match_nolit_encodeSnappyBlockAsm four_bytes_remain_match_nolit_encodeSnappyBlockAsm: TSTW R9, R9 BEQ match_nolit_emitcopy_end_encodeSnappyBlockAsm MOVD $0, R6 ADD R9<<2, R6, R9 SUB $1, R9, R9 MOVWU R9, R9 MOVB R9, (R1) MOVW R5, 1(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm two_byte_offset_match_nolit_encodeSnappyBlockAsm: CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm two_byte_offset_short_match_nolit_encodeSnappyBlockAsm: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm CMPW $0x00000800, R5 BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm emit_copy_three_match_nolit_encodeSnappyBlockAsm: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBlockAsm: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBlockAsm ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBlockAsm: MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x10, R7, R7 MUL R8, R7, R7 LSR $0x32, R7, R7 LSL $0x10, R5, R5 MUL R8, R5, R5 LSR $0x32, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeSnappyBlockAsm ADDW $1, R2, R2 JMP search_loop_encodeSnappyBlockAsm emit_remainder_encodeSnappyBlockAsm: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $5, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBlockAsm: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBlockAsm CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBlockAsm CMPW $0x00010000, R2 BLO three_bytes_emit_remainder_encodeSnappyBlockAsm CMPW $0x01000000, R2 BLO four_bytes_emit_remainder_encodeSnappyBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R2, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBlockAsm four_bytes_emit_remainder_encodeSnappyBlockAsm: MOVWU R2, R3 LSRW $0x10, R3, R3 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R2, 1(R1) MOVB R3, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBlockAsm three_bytes_emit_remainder_encodeSnappyBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBlockAsm two_bytes_emit_remainder_encodeSnappyBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBlockAsm JMP memmove_long_emit_remainder_encodeSnappyBlockAsm one_byte_emit_remainder_encodeSnappyBlockAsm: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBlockAsm: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBlockAsm: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm memmove_long_emit_remainder_encodeSnappyBlockAsm: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsmlarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBlockAsm: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBlockAsm64K(dst []byte, src []byte, tmp *[65536]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBlockAsm64K(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000200, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBlockAsm64K: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBlockAsm64K MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBlockAsm64K: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x06, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBlockAsm64K MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 LSL $0x10, R10, R10 MUL R8, R10, R10 LSR $0x32, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeSnappyBlockAsm64K ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R5 MOVWU R6, R7 MOVWU 24(RSP), R16 SUBSW R16, R7, R7 BEQ repeat_extend_back_end_encodeSnappyBlockAsm64K repeat_extend_back_loop_encodeSnappyBlockAsm64K: CMPW R5, R6 BLS repeat_extend_back_end_encodeSnappyBlockAsm64K ADD R7, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeSnappyBlockAsm64K SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R7, R7 BNE repeat_extend_back_loop_encodeSnappyBlockAsm64K repeat_extend_back_end_encodeSnappyBlockAsm64K: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $3, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeSnappyBlockAsm64K MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeSnappyBlockAsm64K: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K MOVWU R6, R7 MOVW R6, 20(RSP) ADD R5, R3, R8 SUBW R5, R7, R7 SUB $1, R7, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeSnappyBlockAsm64K CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeSnappyBlockAsm64K BLO three_bytes_repeat_emit_encodeSnappyBlockAsm64K three_bytes_repeat_emit_encodeSnappyBlockAsm64K: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K two_bytes_repeat_emit_encodeSnappyBlockAsm64K: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeSnappyBlockAsm64K JMP memmove_long_repeat_emit_encodeSnappyBlockAsm64K one_byte_repeat_emit_encodeSnappyBlockAsm64K: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeSnappyBlockAsm64K: ADD R7, R1, R5 // genMemMoveShort CMP $0x08, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8 CMP $0x10, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16 CMP $0x20, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8: MOVD (R8), R9 MOVD R9, (R1) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_8through16: MOVD (R8), R9 ADD R7, R8, R15 MOVD -8(R15), R8 MOVD R9, (R1) ADD R7, R1, R15 MOVD R8, -8(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_17through32: FMOVQ (R8), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R7, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm64K_memmove_move_33through64: FMOVQ (R8), F0 FMOVQ 16(R8), F1 ADD R7, R8, R15 FMOVQ -32(R15), F2 ADD R7, R8, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R7, R1, R15 FMOVQ F2, -32(R15) ADD R7, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeSnappyBlockAsm64K: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K memmove_long_repeat_emit_encodeSnappyBlockAsm64K: ADD R7, R1, R5 // genMemMoveLong MOVD R8, R9 MOVD R1, R10 MOVD R7, R11 emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm64Klarge_big_loop_back ADD R7, R8, R15 FMOVQ -32(R15), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 ADD R7, R1, R15 FMOVQ F0, -32(R15) ADD R7, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeSnappyBlockAsm64K: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R5 // matchLen MOVD $0, R10 matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K: CMPW $0x10, R7 BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K MOVD (R8)(R10), R9 ADD R10, R8, R15 MOVD 8(R15), R11 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K ADD R10, R5, R15 MOVD 8(R15), R16 EOR R16, R11, R11 TST R11, R11 BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R10, R10 MOVWU R10, R10 JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm64K matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm64K: RBIT R11, R11 CLZ R11, R11 ASR $0x03, R11, R11 ADD R11, R10, R10 ADD $8, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K matchlen_match8_repeat_extend_encodeSnappyBlockAsm64K: CMPW $0x08, R7 BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K MOVD (R8)(R10), R9 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R10, R10 MOVWU R10, R10 JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm64K: RBIT R9, R9 CLZ R9, R9 ASR $0x03, R9, R9 ADD R9, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm64K matchlen_match4_repeat_extend_encodeSnappyBlockAsm64K: CMPW $0x04, R7 BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K MOVWU (R8)(R10), R9 MOVWU (R5)(R10), R16 CMPW R9, R16 BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R10, R10 MOVWU R10, R10 matchlen_match2_repeat_extend_encodeSnappyBlockAsm64K: CMPW $0x01, R7 BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K BLO repeat_extend_forward_end_encodeSnappyBlockAsm64K MOVHU (R8)(R10), R16 BFI $0, R16, $16, R9 ADD R10, R5, R15 MOVHU (R15), R15 AND $0xffff, R9, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K ADD $2, R10, R10 MOVWU R10, R10 SUBSW $0x02, R7, R7 BEQ repeat_extend_forward_end_encodeSnappyBlockAsm64K matchlen_match1_repeat_extend_encodeSnappyBlockAsm64K: MOVBU (R8)(R10), R16 BFI $0, R16, $8, R9 ADD R10, R5, R15 MOVBU (R15), R15 AND $0xff, R9, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeSnappyBlockAsm64K ADD $1, R10, R10 MOVWU R10, R10 repeat_extend_forward_end_encodeSnappyBlockAsm64K: ADDW R10, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 // emitCopy two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K: CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm64K two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm64K: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K CMPW $0x00000800, R6 BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeSnappyBlockAsm64K emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm64K: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeSnappyBlockAsm64K: MOVW R2, 20(RSP) JMP search_loop_encodeSnappyBlockAsm64K no_repeat_found_encodeSnappyBlockAsm64K: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBlockAsm64K LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeSnappyBlockAsm64K MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeSnappyBlockAsm64K MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBlockAsm64K candidate3_match_encodeSnappyBlockAsm64K: ADDW $0x02, R2, R2 JMP candidate_match_encodeSnappyBlockAsm64K candidate2_match_encodeSnappyBlockAsm64K: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBlockAsm64K: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm64K match_extend_back_loop_encodeSnappyBlockAsm64K: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBlockAsm64K ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBlockAsm64K SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm64K JMP match_extend_back_loop_encodeSnappyBlockAsm64K match_extend_back_end_encodeSnappyBlockAsm64K: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBlockAsm64K MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBlockAsm64K: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm64K MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeSnappyBlockAsm64K CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeSnappyBlockAsm64K BLO three_bytes_match_emit_encodeSnappyBlockAsm64K three_bytes_match_emit_encodeSnappyBlockAsm64K: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBlockAsm64K two_bytes_match_emit_encodeSnappyBlockAsm64K: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeSnappyBlockAsm64K JMP memmove_long_match_emit_encodeSnappyBlockAsm64K one_byte_match_emit_encodeSnappyBlockAsm64K: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBlockAsm64K: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm64K emit_lit_memmove_match_emit_encodeSnappyBlockAsm64K_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBlockAsm64K: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeSnappyBlockAsm64K memmove_long_match_emit_encodeSnappyBlockAsm64K: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm64Klarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeSnappyBlockAsm64K: match_nolit_loop_encodeSnappyBlockAsm64K: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm64K MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm64K matchlen_bsf_16match_nolit_encodeSnappyBlockAsm64K: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm64K matchlen_match8_match_nolit_encodeSnappyBlockAsm64K: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm64K MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm64K matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm64K: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm64K matchlen_match4_match_nolit_encodeSnappyBlockAsm64K: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm64K MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm64K SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeSnappyBlockAsm64K: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm64K BLO match_nolit_end_encodeSnappyBlockAsm64K MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm64K ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeSnappyBlockAsm64K matchlen_match1_match_nolit_encodeSnappyBlockAsm64K: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBlockAsm64K ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeSnappyBlockAsm64K: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy two_byte_offset_match_nolit_encodeSnappyBlockAsm64K: CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm64K two_byte_offset_short_match_nolit_encodeSnappyBlockAsm64K: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm64K CMPW $0x00000800, R5 BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm64K SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm64K emit_copy_three_match_nolit_encodeSnappyBlockAsm64K: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBlockAsm64K: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBlockAsm64K ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBlockAsm64K MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBlockAsm64K: MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x10, R7, R7 MUL R8, R7, R7 LSR $0x32, R7, R7 LSL $0x10, R5, R5 MUL R8, R5, R5 LSR $0x32, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeSnappyBlockAsm64K ADDW $1, R2, R2 JMP search_loop_encodeSnappyBlockAsm64K emit_remainder_encodeSnappyBlockAsm64K: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBlockAsm64K MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBlockAsm64K: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBlockAsm64K CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBlockAsm64K BLO three_bytes_emit_remainder_encodeSnappyBlockAsm64K three_bytes_emit_remainder_encodeSnappyBlockAsm64K: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K two_bytes_emit_remainder_encodeSnappyBlockAsm64K: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBlockAsm64K JMP memmove_long_emit_remainder_encodeSnappyBlockAsm64K one_byte_emit_remainder_encodeSnappyBlockAsm64K: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBlockAsm64K: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm64K_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBlockAsm64K: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K memmove_long_emit_remainder_encodeSnappyBlockAsm64K: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm64Klarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBlockAsm64K: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBlockAsm12B(dst []byte, src []byte, tmp *[16384]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBlockAsm12B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000080, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBlockAsm12B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBlockAsm12B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBlockAsm12B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x05, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBlockAsm12B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x000000cf1bbcdcbb, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x18, R9, R9 MUL R8, R9, R9 LSR $0x34, R9, R9 LSL $0x18, R10, R10 MUL R8, R10, R10 LSR $0x34, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x18, R9, R9 MUL R8, R9, R9 LSR $0x34, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeSnappyBlockAsm12B ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R5 MOVWU R6, R7 MOVWU 24(RSP), R16 SUBSW R16, R7, R7 BEQ repeat_extend_back_end_encodeSnappyBlockAsm12B repeat_extend_back_loop_encodeSnappyBlockAsm12B: CMPW R5, R6 BLS repeat_extend_back_end_encodeSnappyBlockAsm12B ADD R7, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeSnappyBlockAsm12B SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R7, R7 BNE repeat_extend_back_loop_encodeSnappyBlockAsm12B repeat_extend_back_end_encodeSnappyBlockAsm12B: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $3, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeSnappyBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeSnappyBlockAsm12B: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B MOVWU R6, R7 MOVW R6, 20(RSP) ADD R5, R3, R8 SUBW R5, R7, R7 SUB $1, R7, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeSnappyBlockAsm12B CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeSnappyBlockAsm12B BLO three_bytes_repeat_emit_encodeSnappyBlockAsm12B three_bytes_repeat_emit_encodeSnappyBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B two_bytes_repeat_emit_encodeSnappyBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeSnappyBlockAsm12B JMP memmove_long_repeat_emit_encodeSnappyBlockAsm12B one_byte_repeat_emit_encodeSnappyBlockAsm12B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeSnappyBlockAsm12B: ADD R7, R1, R5 // genMemMoveShort CMP $0x08, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8 CMP $0x10, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16 CMP $0x20, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8: MOVD (R8), R9 MOVD R9, (R1) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_8through16: MOVD (R8), R9 ADD R7, R8, R15 MOVD -8(R15), R8 MOVD R9, (R1) ADD R7, R1, R15 MOVD R8, -8(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_17through32: FMOVQ (R8), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R7, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm12B_memmove_move_33through64: FMOVQ (R8), F0 FMOVQ 16(R8), F1 ADD R7, R8, R15 FMOVQ -32(R15), F2 ADD R7, R8, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R7, R1, R15 FMOVQ F2, -32(R15) ADD R7, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeSnappyBlockAsm12B: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B memmove_long_repeat_emit_encodeSnappyBlockAsm12B: ADD R7, R1, R5 // genMemMoveLong MOVD R8, R9 MOVD R1, R10 MOVD R7, R11 emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm12Blarge_big_loop_back ADD R7, R8, R15 FMOVQ -32(R15), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 ADD R7, R1, R15 FMOVQ F0, -32(R15) ADD R7, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeSnappyBlockAsm12B: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R5 // matchLen MOVD $0, R10 matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B: CMPW $0x10, R7 BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B MOVD (R8)(R10), R9 ADD R10, R8, R15 MOVD 8(R15), R11 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B ADD R10, R5, R15 MOVD 8(R15), R16 EOR R16, R11, R11 TST R11, R11 BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R10, R10 MOVWU R10, R10 JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm12B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm12B: RBIT R11, R11 CLZ R11, R11 ASR $0x03, R11, R11 ADD R11, R10, R10 ADD $8, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B matchlen_match8_repeat_extend_encodeSnappyBlockAsm12B: CMPW $0x08, R7 BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B MOVD (R8)(R10), R9 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R10, R10 MOVWU R10, R10 JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm12B: RBIT R9, R9 CLZ R9, R9 ASR $0x03, R9, R9 ADD R9, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm12B matchlen_match4_repeat_extend_encodeSnappyBlockAsm12B: CMPW $0x04, R7 BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B MOVWU (R8)(R10), R9 MOVWU (R5)(R10), R16 CMPW R9, R16 BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R10, R10 MOVWU R10, R10 matchlen_match2_repeat_extend_encodeSnappyBlockAsm12B: CMPW $0x01, R7 BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B BLO repeat_extend_forward_end_encodeSnappyBlockAsm12B MOVHU (R8)(R10), R16 BFI $0, R16, $16, R9 ADD R10, R5, R15 MOVHU (R15), R15 AND $0xffff, R9, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B ADD $2, R10, R10 MOVWU R10, R10 SUBSW $0x02, R7, R7 BEQ repeat_extend_forward_end_encodeSnappyBlockAsm12B matchlen_match1_repeat_extend_encodeSnappyBlockAsm12B: MOVBU (R8)(R10), R16 BFI $0, R16, $8, R9 ADD R10, R5, R15 MOVBU (R15), R15 AND $0xff, R9, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeSnappyBlockAsm12B ADD $1, R10, R10 MOVWU R10, R10 repeat_extend_forward_end_encodeSnappyBlockAsm12B: ADDW R10, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 // emitCopy two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B: CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm12B two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm12B: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B CMPW $0x00000800, R6 BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeSnappyBlockAsm12B emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm12B: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeSnappyBlockAsm12B: MOVW R2, 20(RSP) JMP search_loop_encodeSnappyBlockAsm12B no_repeat_found_encodeSnappyBlockAsm12B: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBlockAsm12B LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeSnappyBlockAsm12B MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeSnappyBlockAsm12B MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBlockAsm12B candidate3_match_encodeSnappyBlockAsm12B: ADDW $0x02, R2, R2 JMP candidate_match_encodeSnappyBlockAsm12B candidate2_match_encodeSnappyBlockAsm12B: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBlockAsm12B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm12B match_extend_back_loop_encodeSnappyBlockAsm12B: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBlockAsm12B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBlockAsm12B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm12B JMP match_extend_back_loop_encodeSnappyBlockAsm12B match_extend_back_end_encodeSnappyBlockAsm12B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBlockAsm12B: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm12B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeSnappyBlockAsm12B CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeSnappyBlockAsm12B BLO three_bytes_match_emit_encodeSnappyBlockAsm12B three_bytes_match_emit_encodeSnappyBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBlockAsm12B two_bytes_match_emit_encodeSnappyBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeSnappyBlockAsm12B JMP memmove_long_match_emit_encodeSnappyBlockAsm12B one_byte_match_emit_encodeSnappyBlockAsm12B: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBlockAsm12B: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm12B emit_lit_memmove_match_emit_encodeSnappyBlockAsm12B_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBlockAsm12B: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeSnappyBlockAsm12B memmove_long_match_emit_encodeSnappyBlockAsm12B: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm12Blarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeSnappyBlockAsm12B: match_nolit_loop_encodeSnappyBlockAsm12B: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm12B MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm12B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm12B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm12B matchlen_match8_match_nolit_encodeSnappyBlockAsm12B: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm12B MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm12B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm12B: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm12B matchlen_match4_match_nolit_encodeSnappyBlockAsm12B: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm12B MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm12B SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeSnappyBlockAsm12B: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm12B BLO match_nolit_end_encodeSnappyBlockAsm12B MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm12B ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeSnappyBlockAsm12B matchlen_match1_match_nolit_encodeSnappyBlockAsm12B: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBlockAsm12B ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeSnappyBlockAsm12B: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy two_byte_offset_match_nolit_encodeSnappyBlockAsm12B: CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm12B two_byte_offset_short_match_nolit_encodeSnappyBlockAsm12B: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm12B CMPW $0x00000800, R5 BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm12B SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm12B emit_copy_three_match_nolit_encodeSnappyBlockAsm12B: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBlockAsm12B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBlockAsm12B ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBlockAsm12B: MOVD $0x000000cf1bbcdcbb, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x18, R7, R7 MUL R8, R7, R7 LSR $0x34, R7, R7 LSL $0x18, R5, R5 MUL R8, R5, R5 LSR $0x34, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeSnappyBlockAsm12B ADDW $1, R2, R2 JMP search_loop_encodeSnappyBlockAsm12B emit_remainder_encodeSnappyBlockAsm12B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBlockAsm12B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBlockAsm12B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBlockAsm12B BLO three_bytes_emit_remainder_encodeSnappyBlockAsm12B three_bytes_emit_remainder_encodeSnappyBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B two_bytes_emit_remainder_encodeSnappyBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBlockAsm12B JMP memmove_long_emit_remainder_encodeSnappyBlockAsm12B one_byte_emit_remainder_encodeSnappyBlockAsm12B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBlockAsm12B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm12B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBlockAsm12B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B memmove_long_emit_remainder_encodeSnappyBlockAsm12B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm12Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBlockAsm12B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBlockAsm10B(dst []byte, src []byte, tmp *[4096]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBlockAsm10B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000020, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBlockAsm10B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBlockAsm10B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBlockAsm10B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x05, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBlockAsm10B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x9e3779b1, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x36, R9, R9 LSL $0x20, R10, R10 MUL R8, R10, R10 LSR $0x36, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x36, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeSnappyBlockAsm10B ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R5 MOVWU R6, R7 MOVWU 24(RSP), R16 SUBSW R16, R7, R7 BEQ repeat_extend_back_end_encodeSnappyBlockAsm10B repeat_extend_back_loop_encodeSnappyBlockAsm10B: CMPW R5, R6 BLS repeat_extend_back_end_encodeSnappyBlockAsm10B ADD R7, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeSnappyBlockAsm10B SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R7, R7 BNE repeat_extend_back_loop_encodeSnappyBlockAsm10B repeat_extend_back_end_encodeSnappyBlockAsm10B: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $3, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeSnappyBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeSnappyBlockAsm10B: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B MOVWU R6, R7 MOVW R6, 20(RSP) ADD R5, R3, R8 SUBW R5, R7, R7 SUB $1, R7, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeSnappyBlockAsm10B CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeSnappyBlockAsm10B BLO three_bytes_repeat_emit_encodeSnappyBlockAsm10B three_bytes_repeat_emit_encodeSnappyBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B two_bytes_repeat_emit_encodeSnappyBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeSnappyBlockAsm10B JMP memmove_long_repeat_emit_encodeSnappyBlockAsm10B one_byte_repeat_emit_encodeSnappyBlockAsm10B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeSnappyBlockAsm10B: ADD R7, R1, R5 // genMemMoveShort CMP $0x08, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8 CMP $0x10, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16 CMP $0x20, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8: MOVD (R8), R9 MOVD R9, (R1) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_8through16: MOVD (R8), R9 ADD R7, R8, R15 MOVD -8(R15), R8 MOVD R9, (R1) ADD R7, R1, R15 MOVD R8, -8(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_17through32: FMOVQ (R8), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R7, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm10B_memmove_move_33through64: FMOVQ (R8), F0 FMOVQ 16(R8), F1 ADD R7, R8, R15 FMOVQ -32(R15), F2 ADD R7, R8, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R7, R1, R15 FMOVQ F2, -32(R15) ADD R7, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeSnappyBlockAsm10B: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B memmove_long_repeat_emit_encodeSnappyBlockAsm10B: ADD R7, R1, R5 // genMemMoveLong MOVD R8, R9 MOVD R1, R10 MOVD R7, R11 emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm10Blarge_big_loop_back ADD R7, R8, R15 FMOVQ -32(R15), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 ADD R7, R1, R15 FMOVQ F0, -32(R15) ADD R7, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeSnappyBlockAsm10B: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R5 // matchLen MOVD $0, R10 matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B: CMPW $0x10, R7 BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B MOVD (R8)(R10), R9 ADD R10, R8, R15 MOVD 8(R15), R11 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B ADD R10, R5, R15 MOVD 8(R15), R16 EOR R16, R11, R11 TST R11, R11 BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R10, R10 MOVWU R10, R10 JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm10B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm10B: RBIT R11, R11 CLZ R11, R11 ASR $0x03, R11, R11 ADD R11, R10, R10 ADD $8, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B matchlen_match8_repeat_extend_encodeSnappyBlockAsm10B: CMPW $0x08, R7 BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B MOVD (R8)(R10), R9 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R10, R10 MOVWU R10, R10 JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm10B: RBIT R9, R9 CLZ R9, R9 ASR $0x03, R9, R9 ADD R9, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm10B matchlen_match4_repeat_extend_encodeSnappyBlockAsm10B: CMPW $0x04, R7 BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B MOVWU (R8)(R10), R9 MOVWU (R5)(R10), R16 CMPW R9, R16 BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R10, R10 MOVWU R10, R10 matchlen_match2_repeat_extend_encodeSnappyBlockAsm10B: CMPW $0x01, R7 BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B BLO repeat_extend_forward_end_encodeSnappyBlockAsm10B MOVHU (R8)(R10), R16 BFI $0, R16, $16, R9 ADD R10, R5, R15 MOVHU (R15), R15 AND $0xffff, R9, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B ADD $2, R10, R10 MOVWU R10, R10 SUBSW $0x02, R7, R7 BEQ repeat_extend_forward_end_encodeSnappyBlockAsm10B matchlen_match1_repeat_extend_encodeSnappyBlockAsm10B: MOVBU (R8)(R10), R16 BFI $0, R16, $8, R9 ADD R10, R5, R15 MOVBU (R15), R15 AND $0xff, R9, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeSnappyBlockAsm10B ADD $1, R10, R10 MOVWU R10, R10 repeat_extend_forward_end_encodeSnappyBlockAsm10B: ADDW R10, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 // emitCopy two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B: CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm10B two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm10B: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B CMPW $0x00000800, R6 BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeSnappyBlockAsm10B emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm10B: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeSnappyBlockAsm10B: MOVW R2, 20(RSP) JMP search_loop_encodeSnappyBlockAsm10B no_repeat_found_encodeSnappyBlockAsm10B: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBlockAsm10B LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeSnappyBlockAsm10B MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeSnappyBlockAsm10B MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBlockAsm10B candidate3_match_encodeSnappyBlockAsm10B: ADDW $0x02, R2, R2 JMP candidate_match_encodeSnappyBlockAsm10B candidate2_match_encodeSnappyBlockAsm10B: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBlockAsm10B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm10B match_extend_back_loop_encodeSnappyBlockAsm10B: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBlockAsm10B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBlockAsm10B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm10B JMP match_extend_back_loop_encodeSnappyBlockAsm10B match_extend_back_end_encodeSnappyBlockAsm10B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBlockAsm10B: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm10B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeSnappyBlockAsm10B CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeSnappyBlockAsm10B BLO three_bytes_match_emit_encodeSnappyBlockAsm10B three_bytes_match_emit_encodeSnappyBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBlockAsm10B two_bytes_match_emit_encodeSnappyBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeSnappyBlockAsm10B JMP memmove_long_match_emit_encodeSnappyBlockAsm10B one_byte_match_emit_encodeSnappyBlockAsm10B: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBlockAsm10B: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm10B emit_lit_memmove_match_emit_encodeSnappyBlockAsm10B_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBlockAsm10B: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeSnappyBlockAsm10B memmove_long_match_emit_encodeSnappyBlockAsm10B: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm10Blarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeSnappyBlockAsm10B: match_nolit_loop_encodeSnappyBlockAsm10B: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm10B MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm10B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm10B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm10B matchlen_match8_match_nolit_encodeSnappyBlockAsm10B: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm10B MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm10B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm10B: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm10B matchlen_match4_match_nolit_encodeSnappyBlockAsm10B: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm10B MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm10B SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeSnappyBlockAsm10B: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm10B BLO match_nolit_end_encodeSnappyBlockAsm10B MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm10B ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeSnappyBlockAsm10B matchlen_match1_match_nolit_encodeSnappyBlockAsm10B: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBlockAsm10B ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeSnappyBlockAsm10B: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy two_byte_offset_match_nolit_encodeSnappyBlockAsm10B: CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm10B two_byte_offset_short_match_nolit_encodeSnappyBlockAsm10B: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm10B CMPW $0x00000800, R5 BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm10B SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm10B emit_copy_three_match_nolit_encodeSnappyBlockAsm10B: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBlockAsm10B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBlockAsm10B ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBlockAsm10B: MOVD $0x9e3779b1, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x20, R7, R7 MUL R8, R7, R7 LSR $0x36, R7, R7 LSL $0x20, R5, R5 MUL R8, R5, R5 LSR $0x36, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeSnappyBlockAsm10B ADDW $1, R2, R2 JMP search_loop_encodeSnappyBlockAsm10B emit_remainder_encodeSnappyBlockAsm10B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBlockAsm10B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBlockAsm10B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBlockAsm10B BLO three_bytes_emit_remainder_encodeSnappyBlockAsm10B three_bytes_emit_remainder_encodeSnappyBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B two_bytes_emit_remainder_encodeSnappyBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBlockAsm10B JMP memmove_long_emit_remainder_encodeSnappyBlockAsm10B one_byte_emit_remainder_encodeSnappyBlockAsm10B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBlockAsm10B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm10B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBlockAsm10B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B memmove_long_emit_remainder_encodeSnappyBlockAsm10B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm10Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBlockAsm10B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBlockAsm8B(dst []byte, src []byte, tmp *[1024]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBlockAsm8B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000008, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBlockAsm8B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBlockAsm8B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBlockAsm8B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x04, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBlockAsm8B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x9e3779b1, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x38, R9, R9 LSL $0x20, R10, R10 MUL R8, R10, R10 LSR $0x38, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x38, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_encodeSnappyBlockAsm8B ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R5 MOVWU R6, R7 MOVWU 24(RSP), R16 SUBSW R16, R7, R7 BEQ repeat_extend_back_end_encodeSnappyBlockAsm8B repeat_extend_back_loop_encodeSnappyBlockAsm8B: CMPW R5, R6 BLS repeat_extend_back_end_encodeSnappyBlockAsm8B ADD R7, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_encodeSnappyBlockAsm8B SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R7, R7 BNE repeat_extend_back_loop_encodeSnappyBlockAsm8B repeat_extend_back_end_encodeSnappyBlockAsm8B: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $3, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_encodeSnappyBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET repeat_dst_size_check_encodeSnappyBlockAsm8B: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B MOVWU R6, R7 MOVW R6, 20(RSP) ADD R5, R3, R8 SUBW R5, R7, R7 SUB $1, R7, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_encodeSnappyBlockAsm8B CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_encodeSnappyBlockAsm8B BLO three_bytes_repeat_emit_encodeSnappyBlockAsm8B three_bytes_repeat_emit_encodeSnappyBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B two_bytes_repeat_emit_encodeSnappyBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_encodeSnappyBlockAsm8B JMP memmove_long_repeat_emit_encodeSnappyBlockAsm8B one_byte_repeat_emit_encodeSnappyBlockAsm8B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_repeat_emit_encodeSnappyBlockAsm8B: ADD R7, R1, R5 // genMemMoveShort CMP $0x08, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8 CMP $0x10, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16 CMP $0x20, R7 BLS emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64 emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8: MOVD (R8), R9 MOVD R9, (R1) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_8through16: MOVD (R8), R9 ADD R7, R8, R15 MOVD -8(R15), R8 MOVD R9, (R1) ADD R7, R1, R15 MOVD R8, -8(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_17through32: FMOVQ (R8), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R7, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B emit_lit_memmove_repeat_emit_encodeSnappyBlockAsm8B_memmove_move_33through64: FMOVQ (R8), F0 FMOVQ 16(R8), F1 ADD R7, R8, R15 FMOVQ -32(R15), F2 ADD R7, R8, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R7, R1, R15 FMOVQ F2, -32(R15) ADD R7, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_repeat_emit_encodeSnappyBlockAsm8B: MOVD R5, R1 JMP emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B memmove_long_repeat_emit_encodeSnappyBlockAsm8B: ADD R7, R1, R5 // genMemMoveLong MOVD R8, R9 MOVD R1, R10 MOVD R7, R11 emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_repeat_emit_encodeSnappyBlockAsm8Blarge_big_loop_back ADD R7, R8, R15 FMOVQ -32(R15), F0 ADD R7, R8, R15 FMOVQ -16(R15), F1 ADD R7, R1, R15 FMOVQ F0, -32(R15) ADD R7, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_repeat_emit_encodeSnappyBlockAsm8B: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R5 // matchLen MOVD $0, R10 matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B: CMPW $0x10, R7 BLO matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B MOVD (R8)(R10), R9 ADD R10, R8, R15 MOVD 8(R15), R11 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B ADD R10, R5, R15 MOVD 8(R15), R16 EOR R16, R11, R11 TST R11, R11 BNE matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R10, R10 MOVWU R10, R10 JMP matchlen_loopback_16_repeat_extend_encodeSnappyBlockAsm8B matchlen_bsf_16repeat_extend_encodeSnappyBlockAsm8B: RBIT R11, R11 CLZ R11, R11 ASR $0x03, R11, R11 ADD R11, R10, R10 ADD $8, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B matchlen_match8_repeat_extend_encodeSnappyBlockAsm8B: CMPW $0x08, R7 BLO matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B MOVD (R8)(R10), R9 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R10, R10 MOVWU R10, R10 JMP matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B matchlen_bsf_8_repeat_extend_encodeSnappyBlockAsm8B: RBIT R9, R9 CLZ R9, R9 ASR $0x03, R9, R9 ADD R9, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_encodeSnappyBlockAsm8B matchlen_match4_repeat_extend_encodeSnappyBlockAsm8B: CMPW $0x04, R7 BLO matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B MOVWU (R8)(R10), R9 MOVWU (R5)(R10), R16 CMPW R9, R16 BNE matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R10, R10 MOVWU R10, R10 matchlen_match2_repeat_extend_encodeSnappyBlockAsm8B: CMPW $0x01, R7 BEQ matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B BLO repeat_extend_forward_end_encodeSnappyBlockAsm8B MOVHU (R8)(R10), R16 BFI $0, R16, $16, R9 ADD R10, R5, R15 MOVHU (R15), R15 AND $0xffff, R9, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B ADD $2, R10, R10 MOVWU R10, R10 SUBSW $0x02, R7, R7 BEQ repeat_extend_forward_end_encodeSnappyBlockAsm8B matchlen_match1_repeat_extend_encodeSnappyBlockAsm8B: MOVBU (R8)(R10), R16 BFI $0, R16, $8, R9 ADD R10, R5, R15 MOVBU (R15), R15 AND $0xff, R9, R16 CMP R16, R15 BNE repeat_extend_forward_end_encodeSnappyBlockAsm8B ADD $1, R10, R10 MOVWU R10, R10 repeat_extend_forward_end_encodeSnappyBlockAsm8B: ADDW R10, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 // emitCopy two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B: CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B MOVD $0xee, R16 MOVB R16, (R1) MOVH R6, 1(R1) SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 JMP two_byte_offset_repeat_as_copy_encodeSnappyBlockAsm8B two_byte_offset_short_repeat_as_copy_encodeSnappyBlockAsm8B: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B SUB $15, R7, R7 MOVWU R7, R7 MOVB R6, 1(R1) LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R7, R7 MOVB R7, (R1) ADD $0x02, R1, R1 JMP repeat_end_emit_encodeSnappyBlockAsm8B emit_copy_three_repeat_as_copy_encodeSnappyBlockAsm8B: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R1) MOVH R6, 1(R1) ADD $0x03, R1, R1 repeat_end_emit_encodeSnappyBlockAsm8B: MOVW R2, 20(RSP) JMP search_loop_encodeSnappyBlockAsm8B no_repeat_found_encodeSnappyBlockAsm8B: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBlockAsm8B LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_encodeSnappyBlockAsm8B MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_encodeSnappyBlockAsm8B MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBlockAsm8B candidate3_match_encodeSnappyBlockAsm8B: ADDW $0x02, R2, R2 JMP candidate_match_encodeSnappyBlockAsm8B candidate2_match_encodeSnappyBlockAsm8B: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBlockAsm8B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm8B match_extend_back_loop_encodeSnappyBlockAsm8B: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBlockAsm8B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBlockAsm8B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBlockAsm8B JMP match_extend_back_loop_encodeSnappyBlockAsm8B match_extend_back_end_encodeSnappyBlockAsm8B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBlockAsm8B: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_encodeSnappyBlockAsm8B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R7 MOVWU R7, R7 CMPW $0x3c, R7 BLO one_byte_match_emit_encodeSnappyBlockAsm8B CMPW $0x00000100, R7 BLO two_bytes_match_emit_encodeSnappyBlockAsm8B BLO three_bytes_match_emit_encodeSnappyBlockAsm8B three_bytes_match_emit_encodeSnappyBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBlockAsm8B two_bytes_match_emit_encodeSnappyBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R7, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R7 BLO memmove_match_emit_encodeSnappyBlockAsm8B JMP memmove_long_match_emit_encodeSnappyBlockAsm8B one_byte_match_emit_encodeSnappyBlockAsm8B: LSLW $0x02, R7, R16 BFI $0, R16, $8, R7 MOVB R7, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBlockAsm8B: ADD R8, R1, R7 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8: MOVD (R6), R9 MOVD R9, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_8through16: MOVD (R6), R9 ADD R8, R6, R15 MOVD -8(R15), R6 MOVD R9, (R1) ADD R8, R1, R15 MOVD R6, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_17through32: FMOVQ (R6), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBlockAsm8B emit_lit_memmove_match_emit_encodeSnappyBlockAsm8B_memmove_move_33through64: FMOVQ (R6), F0 FMOVQ 16(R6), F1 ADD R8, R6, R15 FMOVQ -32(R15), F2 ADD R8, R6, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBlockAsm8B: MOVD R7, R1 JMP emit_literal_done_match_emit_encodeSnappyBlockAsm8B memmove_long_match_emit_encodeSnappyBlockAsm8B: ADD R8, R1, R7 // genMemMoveLong MOVD R6, R9 MOVD R1, R10 MOVD R8, R11 emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back: FMOVQ (R9), F0 FMOVQ 16(R9), F1 FMOVQ F0, (R10) FMOVQ F1, 16(R10) ADD $0x20, R9, R9 ADD $0x20, R10, R10 SUB $0x20, R11, R11 CMP $0x20, R11 BHS emit_lit_memmove_long_match_emit_encodeSnappyBlockAsm8Blarge_big_loop_back ADD R8, R6, R15 FMOVQ -32(R15), F0 ADD R8, R6, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R7, R1 emit_literal_done_match_emit_encodeSnappyBlockAsm8B: match_nolit_loop_encodeSnappyBlockAsm8B: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_encodeSnappyBlockAsm8B MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_encodeSnappyBlockAsm8B matchlen_bsf_16match_nolit_encodeSnappyBlockAsm8B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm8B matchlen_match8_match_nolit_encodeSnappyBlockAsm8B: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_encodeSnappyBlockAsm8B MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_encodeSnappyBlockAsm8B matchlen_bsf_8_match_nolit_encodeSnappyBlockAsm8B: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_encodeSnappyBlockAsm8B matchlen_match4_match_nolit_encodeSnappyBlockAsm8B: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_encodeSnappyBlockAsm8B MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_encodeSnappyBlockAsm8B SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_encodeSnappyBlockAsm8B: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_encodeSnappyBlockAsm8B BLO match_nolit_end_encodeSnappyBlockAsm8B MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBlockAsm8B ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_encodeSnappyBlockAsm8B matchlen_match1_match_nolit_encodeSnappyBlockAsm8B: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBlockAsm8B ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_encodeSnappyBlockAsm8B: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy two_byte_offset_match_nolit_encodeSnappyBlockAsm8B: CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B MOVD $0xee, R16 MOVB R16, (R1) MOVH R5, 1(R1) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBlockAsm8B two_byte_offset_short_match_nolit_encodeSnappyBlockAsm8B: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_encodeSnappyBlockAsm8B SUB $15, R6, R6 MOVWU R6, R6 MOVB R5, 1(R1) LSRW $0x08, R5, R5 LSLW $0x05, R5, R5 ORRW R5, R6, R6 MOVB R6, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBlockAsm8B emit_copy_three_match_nolit_encodeSnappyBlockAsm8B: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBlockAsm8B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBlockAsm8B ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBlockAsm8B: MOVD $0x9e3779b1, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x20, R7, R7 MUL R8, R7, R7 LSR $0x38, R7, R7 LSL $0x20, R5, R5 MUL R8, R5, R5 LSR $0x38, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_encodeSnappyBlockAsm8B ADDW $1, R2, R2 JMP search_loop_encodeSnappyBlockAsm8B emit_remainder_encodeSnappyBlockAsm8B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBlockAsm8B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBlockAsm8B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBlockAsm8B BLO three_bytes_emit_remainder_encodeSnappyBlockAsm8B three_bytes_emit_remainder_encodeSnappyBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B two_bytes_emit_remainder_encodeSnappyBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBlockAsm8B JMP memmove_long_emit_remainder_encodeSnappyBlockAsm8B one_byte_emit_remainder_encodeSnappyBlockAsm8B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBlockAsm8B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBlockAsm8B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBlockAsm8B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B memmove_long_emit_remainder_encodeSnappyBlockAsm8B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBlockAsm8Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBlockAsm8B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBetterBlockAsm(dst []byte, src []byte, tmp *[589824]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBetterBlockAsm(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00001200, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBetterBlockAsm: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBetterBlockAsm MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBetterBlockAsm: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x07, R5, R5 CMPW $0x63, R5 BLS check_maxskip_ok_encodeSnappyBetterBlockAsm ADD $100, R2, R5 MOVWU R5, R5 JMP check_maxskip_cont_encodeSnappyBetterBlockAsm check_maxskip_ok_encodeSnappyBetterBlockAsm: ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 check_maxskip_cont_encodeSnappyBetterBlockAsm: MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBetterBlockAsm MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x00cf1bbcdcbfa563, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x08, R9, R9 MUL R8, R9, R9 LSR $0x2f, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x32, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 524288(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 524288(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm CMP R6, R10 BNE no_short_found_encodeSnappyBetterBlockAsm MOVWU R7, R5 JMP candidate_match_encodeSnappyBetterBlockAsm no_short_found_encodeSnappyBetterBlockAsm: CMPW R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm CMPW R6, R10 BEQ candidateS_match_encodeSnappyBetterBlockAsm MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBetterBlockAsm candidateS_match_encodeSnappyBetterBlockAsm: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x08, R9, R9 MUL R8, R9, R9 LSR $0x2f, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBetterBlockAsm SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBetterBlockAsm: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm match_extend_back_loop_encodeSnappyBetterBlockAsm: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBetterBlockAsm ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBetterBlockAsm SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm JMP match_extend_back_loop_encodeSnappyBetterBlockAsm match_extend_back_end_encodeSnappyBetterBlockAsm: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $5, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBetterBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBetterBlockAsm: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm BLO match_nolit_end_encodeSnappyBetterBlockAsm MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeSnappyBetterBlockAsm matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBetterBlockAsm ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeSnappyBetterBlockAsm: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat CMPW $0x01, R11 BHI match_length_ok_encodeSnappyBetterBlockAsm CMPW $0x0000ffff, R7 BLS match_length_ok_encodeSnappyBetterBlockAsm MOVWU 28(RSP), R2 ADDW $1, R2, R2 JMP search_loop_encodeSnappyBetterBlockAsm match_length_ok_encodeSnappyBetterBlockAsm: MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeSnappyBetterBlockAsm CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm CMPW $0x00010000, R5 BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm CMPW $0x01000000, R5 BLO four_bytes_match_emit_encodeSnappyBetterBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R5, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm four_bytes_match_emit_encodeSnappyBetterBlockAsm: MOVWU R5, R10 LSRW $0x10, R10, R10 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R5, 1(R1) MOVB R10, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm three_bytes_match_emit_encodeSnappyBetterBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm two_bytes_match_emit_encodeSnappyBetterBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeSnappyBetterBlockAsm JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm one_byte_match_emit_encodeSnappyBetterBlockAsm: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBetterBlockAsm: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm memmove_long_match_emit_encodeSnappyBetterBlockAsm: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsmlarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeSnappyBetterBlockAsm: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy CMPW $0x00010000, R7 BLO two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm: CMPW $0x40, R11 BLS four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm MOVD $0xff, R16 MOVB R16, (R1) MOVW R7, 1(R1) SUB $64, R11, R11 MOVWU R11, R11 ADD $0x05, R1, R1 CMPW $0x04, R11 BLO four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm JMP four_bytes_loop_back_match_nolit_encodeSnappyBetterBlockAsm four_bytes_remain_match_nolit_encodeSnappyBetterBlockAsm: TSTW R11, R11 BEQ match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm MOVD $0, R5 ADD R11<<2, R5, R11 SUB $1, R11, R11 MOVWU R11, R11 MOVB R11, (R1) MOVW R7, 1(R1) ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm: CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm CMPW $0x00000800, R7 BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBetterBlockAsm MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBetterBlockAsm: MOVD $0x00cf1bbcdcbfa563, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x08, R9, R9 MUL R5, R9, R9 LSR $0x2f, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x32, R10, R10 LSL $0x08, R11, R11 MUL R5, R11, R11 LSR $0x2f, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x32, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 524288(R15) ADD R12<<2, R0, R15 MOVW R13, 524288(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeSnappyBetterBlockAsm: CMP R8, R7 BHS search_loop_encodeSnappyBetterBlockAsm MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x08, R9, R9 MUL R5, R9, R9 LSR $0x2f, R9, R9 LSL $0x08, R10, R10 MUL R5, R10, R10 LSR $0x2f, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeSnappyBetterBlockAsm emit_remainder_encodeSnappyBetterBlockAsm: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $5, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBetterBlockAsm MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBetterBlockAsm: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm CMPW $0x00010000, R2 BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm CMPW $0x01000000, R2 BLO four_bytes_emit_remainder_encodeSnappyBetterBlockAsm MOVD $0xfc, R16 MOVB R16, (R1) MOVW R2, 1(R1) ADD $0x05, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm four_bytes_emit_remainder_encodeSnappyBetterBlockAsm: MOVWU R2, R3 LSRW $0x10, R3, R3 MOVD $0xf8, R16 MOVB R16, (R1) MOVH R2, 1(R1) MOVB R3, 3(R1) ADD $0x04, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm three_bytes_emit_remainder_encodeSnappyBetterBlockAsm: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm two_bytes_emit_remainder_encodeSnappyBetterBlockAsm: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm one_byte_emit_remainder_encodeSnappyBetterBlockAsm: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBetterBlockAsm: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm memmove_long_emit_remainder_encodeSnappyBetterBlockAsm: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsmlarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBetterBlockAsm64K(dst []byte, src []byte, tmp *[294912]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBetterBlockAsm64K(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000900, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBetterBlockAsm64K: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBetterBlockAsm64K MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBetterBlockAsm64K: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x07, R5, R5 ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBetterBlockAsm64K MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x00cf1bbcdcbfa563, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x08, R9, R9 MUL R8, R9, R9 LSR $0x30, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x33, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 262144(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 262144(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm64K CMP R6, R10 BNE no_short_found_encodeSnappyBetterBlockAsm64K MOVWU R7, R5 JMP candidate_match_encodeSnappyBetterBlockAsm64K no_short_found_encodeSnappyBetterBlockAsm64K: CMPW R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm64K CMPW R6, R10 BEQ candidateS_match_encodeSnappyBetterBlockAsm64K MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBetterBlockAsm64K candidateS_match_encodeSnappyBetterBlockAsm64K: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x08, R9, R9 MUL R8, R9, R9 LSR $0x30, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBetterBlockAsm64K SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBetterBlockAsm64K: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm64K match_extend_back_loop_encodeSnappyBetterBlockAsm64K: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBetterBlockAsm64K ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBetterBlockAsm64K SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm64K JMP match_extend_back_loop_encodeSnappyBetterBlockAsm64K match_extend_back_end_encodeSnappyBetterBlockAsm64K: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBetterBlockAsm64K MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBetterBlockAsm64K: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm64K matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm64K: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm64K matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm64K: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm64K: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm64K matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm64K: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm64K: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K BLO match_nolit_end_encodeSnappyBetterBlockAsm64K MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeSnappyBetterBlockAsm64K matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm64K: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBetterBlockAsm64K ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeSnappyBetterBlockAsm64K: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeSnappyBetterBlockAsm64K CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm64K BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm64K three_bytes_match_emit_encodeSnappyBetterBlockAsm64K: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K two_bytes_match_emit_encodeSnappyBetterBlockAsm64K: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeSnappyBetterBlockAsm64K JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm64K one_byte_match_emit_encodeSnappyBetterBlockAsm64K: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBetterBlockAsm64K: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm64K_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm64K: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K memmove_long_match_emit_encodeSnappyBetterBlockAsm64K: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm64Klarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeSnappyBetterBlockAsm64K: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K: CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm64K two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm64K: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K CMPW $0x00000800, R7 BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm64K: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm64K: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBetterBlockAsm64K MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBetterBlockAsm64K: MOVD $0x00cf1bbcdcbfa563, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x08, R9, R9 MUL R5, R9, R9 LSR $0x30, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x33, R10, R10 LSL $0x08, R11, R11 MUL R5, R11, R11 LSR $0x30, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x33, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 262144(R15) ADD R12<<2, R0, R15 MOVW R13, 262144(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeSnappyBetterBlockAsm64K: CMP R8, R7 BHS search_loop_encodeSnappyBetterBlockAsm64K MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x08, R9, R9 MUL R5, R9, R9 LSR $0x30, R9, R9 LSL $0x08, R10, R10 MUL R5, R10, R10 LSR $0x30, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeSnappyBetterBlockAsm64K emit_remainder_encodeSnappyBetterBlockAsm64K: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBetterBlockAsm64K MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBetterBlockAsm64K: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K three_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K two_bytes_emit_remainder_encodeSnappyBetterBlockAsm64K: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm64K JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K one_byte_emit_remainder_encodeSnappyBetterBlockAsm64K: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBetterBlockAsm64K: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm64K_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm64K: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64K: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm64Klarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm64K: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBetterBlockAsm12B(dst []byte, src []byte, tmp *[81920]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBetterBlockAsm12B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000280, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBetterBlockAsm12B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBetterBlockAsm12B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBetterBlockAsm12B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x06, R5, R5 ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBetterBlockAsm12B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x34, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 65536(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 65536(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm12B CMP R6, R10 BNE no_short_found_encodeSnappyBetterBlockAsm12B MOVWU R7, R5 JMP candidate_match_encodeSnappyBetterBlockAsm12B no_short_found_encodeSnappyBetterBlockAsm12B: CMPW R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm12B CMPW R6, R10 BEQ candidateS_match_encodeSnappyBetterBlockAsm12B MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBetterBlockAsm12B candidateS_match_encodeSnappyBetterBlockAsm12B: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x32, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBetterBlockAsm12B SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBetterBlockAsm12B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm12B match_extend_back_loop_encodeSnappyBetterBlockAsm12B: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBetterBlockAsm12B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBetterBlockAsm12B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm12B JMP match_extend_back_loop_encodeSnappyBetterBlockAsm12B match_extend_back_end_encodeSnappyBetterBlockAsm12B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBetterBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBetterBlockAsm12B: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm12B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm12B: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm12B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm12B: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm12B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm12B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm12B: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm12B: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B BLO match_nolit_end_encodeSnappyBetterBlockAsm12B MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeSnappyBetterBlockAsm12B matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm12B: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBetterBlockAsm12B ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeSnappyBetterBlockAsm12B: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeSnappyBetterBlockAsm12B CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm12B BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm12B three_bytes_match_emit_encodeSnappyBetterBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B two_bytes_match_emit_encodeSnappyBetterBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeSnappyBetterBlockAsm12B JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm12B one_byte_match_emit_encodeSnappyBetterBlockAsm12B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBetterBlockAsm12B: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm12B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm12B: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B memmove_long_match_emit_encodeSnappyBetterBlockAsm12B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm12Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeSnappyBetterBlockAsm12B: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B: CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm12B two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm12B: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B CMPW $0x00000800, R7 BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm12B: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm12B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBetterBlockAsm12B MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBetterBlockAsm12B: MOVD $0x0000cf1bbcdcbf9b, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x32, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x34, R10, R10 LSL $0x10, R11, R11 MUL R5, R11, R11 LSR $0x32, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x34, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 65536(R15) ADD R12<<2, R0, R15 MOVW R13, 65536(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeSnappyBetterBlockAsm12B: CMP R8, R7 BHS search_loop_encodeSnappyBetterBlockAsm12B MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x32, R9, R9 LSL $0x10, R10, R10 MUL R5, R10, R10 LSR $0x32, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeSnappyBetterBlockAsm12B emit_remainder_encodeSnappyBetterBlockAsm12B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBetterBlockAsm12B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBetterBlockAsm12B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B three_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B two_bytes_emit_remainder_encodeSnappyBetterBlockAsm12B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm12B JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B one_byte_emit_remainder_encodeSnappyBetterBlockAsm12B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBetterBlockAsm12B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm12B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm12B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm12Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm12B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBetterBlockAsm10B(dst []byte, src []byte, tmp *[20480]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBetterBlockAsm10B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x000000a0, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBetterBlockAsm10B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBetterBlockAsm10B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBetterBlockAsm10B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x05, R5, R5 ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBetterBlockAsm10B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x34, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x36, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 16384(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 16384(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm10B CMP R6, R10 BNE no_short_found_encodeSnappyBetterBlockAsm10B MOVWU R7, R5 JMP candidate_match_encodeSnappyBetterBlockAsm10B no_short_found_encodeSnappyBetterBlockAsm10B: CMPW R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm10B CMPW R6, R10 BEQ candidateS_match_encodeSnappyBetterBlockAsm10B MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBetterBlockAsm10B candidateS_match_encodeSnappyBetterBlockAsm10B: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x34, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBetterBlockAsm10B SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBetterBlockAsm10B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm10B match_extend_back_loop_encodeSnappyBetterBlockAsm10B: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBetterBlockAsm10B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBetterBlockAsm10B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm10B JMP match_extend_back_loop_encodeSnappyBetterBlockAsm10B match_extend_back_end_encodeSnappyBetterBlockAsm10B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBetterBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBetterBlockAsm10B: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm10B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm10B: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm10B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm10B: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm10B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm10B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm10B: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm10B: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B BLO match_nolit_end_encodeSnappyBetterBlockAsm10B MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeSnappyBetterBlockAsm10B matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm10B: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBetterBlockAsm10B ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeSnappyBetterBlockAsm10B: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeSnappyBetterBlockAsm10B CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm10B BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm10B three_bytes_match_emit_encodeSnappyBetterBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B two_bytes_match_emit_encodeSnappyBetterBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeSnappyBetterBlockAsm10B JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm10B one_byte_match_emit_encodeSnappyBetterBlockAsm10B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBetterBlockAsm10B: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm10B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm10B: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B memmove_long_match_emit_encodeSnappyBetterBlockAsm10B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm10Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeSnappyBetterBlockAsm10B: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B: CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm10B two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm10B: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B CMPW $0x00000800, R7 BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm10B: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm10B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBetterBlockAsm10B MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBetterBlockAsm10B: MOVD $0x0000cf1bbcdcbf9b, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x34, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x36, R10, R10 LSL $0x10, R11, R11 MUL R5, R11, R11 LSR $0x34, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x36, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 16384(R15) ADD R12<<2, R0, R15 MOVW R13, 16384(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeSnappyBetterBlockAsm10B: CMP R8, R7 BHS search_loop_encodeSnappyBetterBlockAsm10B MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x34, R9, R9 LSL $0x10, R10, R10 MUL R5, R10, R10 LSR $0x34, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeSnappyBetterBlockAsm10B emit_remainder_encodeSnappyBetterBlockAsm10B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBetterBlockAsm10B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBetterBlockAsm10B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B three_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B two_bytes_emit_remainder_encodeSnappyBetterBlockAsm10B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm10B JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B one_byte_emit_remainder_encodeSnappyBetterBlockAsm10B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBetterBlockAsm10B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm10B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm10B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm10Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm10B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func encodeSnappyBetterBlockAsm8B(dst []byte, src []byte, tmp *[5120]byte) int // Requires: BMI, SSE2 TEXT ·encodeSnappyBetterBlockAsm8B(SB), $24-64 MOVD tmp+48(FP), R0 MOVD dst_base+0(FP), R1 MOVD $0x00000028, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_encodeSnappyBetterBlockAsm8B: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_encodeSnappyBetterBlockAsm8B MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+32(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVD $0x00000000, R16 MOVW R16, 24(RSP) MOVD src_base+24(FP), R3 search_loop_encodeSnappyBetterBlockAsm8B: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x04, R5, R5 ADD R5, R2, R5 ADD $1, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_encodeSnappyBetterBlockAsm8B MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD $0x9e3779b1, R5 MOVD R6, R9 MOVD R6, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x36, R9, R9 LSL $0x20, R10, R10 MUL R5, R10, R10 LSR $0x38, R10, R10 MOVWU (R0)(R9<<2), R5 ADD R10<<2, R0, R15 MOVWU 4096(R15), R7 MOVW R2, (R0)(R9<<2) ADD R10<<2, R0, R15 MOVW R2, 4096(R15) MOVD (R3)(R5), R9 MOVD (R3)(R7), R10 CMP R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm8B CMP R6, R10 BNE no_short_found_encodeSnappyBetterBlockAsm8B MOVWU R7, R5 JMP candidate_match_encodeSnappyBetterBlockAsm8B no_short_found_encodeSnappyBetterBlockAsm8B: CMPW R6, R9 BEQ candidate_match_encodeSnappyBetterBlockAsm8B CMPW R6, R10 BEQ candidateS_match_encodeSnappyBetterBlockAsm8B MOVWU 28(RSP), R2 JMP search_loop_encodeSnappyBetterBlockAsm8B candidateS_match_encodeSnappyBetterBlockAsm8B: LSR $0x08, R6, R6 MOVD R6, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x36, R9, R9 MOVWU (R0)(R9<<2), R5 ADDW $1, R2, R2 MOVW R2, (R0)(R9<<2) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_encodeSnappyBetterBlockAsm8B SUBW $1, R2, R2 MOVWU R7, R5 candidate_match_encodeSnappyBetterBlockAsm8B: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm8B match_extend_back_loop_encodeSnappyBetterBlockAsm8B: CMPW R6, R2 BLS match_extend_back_end_encodeSnappyBetterBlockAsm8B ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_encodeSnappyBetterBlockAsm8B SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_encodeSnappyBetterBlockAsm8B JMP match_extend_back_loop_encodeSnappyBetterBlockAsm8B match_extend_back_end_encodeSnappyBetterBlockAsm8B: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_encodeSnappyBetterBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_dst_size_check_encodeSnappyBetterBlockAsm8B: MOVWU R2, R6 ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+32(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R9 // matchLen MOVD $0, R11 matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B: CMPW $0x10, R7 BLO matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B MOVD (R8)(R11), R10 ADD R11, R8, R15 MOVD 8(R15), R12 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B ADD R11, R9, R15 MOVD 8(R15), R16 EOR R16, R12, R12 TST R12, R12 BNE matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R11, R11 MOVWU R11, R11 JMP matchlen_loopback_16_match_nolit_encodeSnappyBetterBlockAsm8B matchlen_bsf_16match_nolit_encodeSnappyBetterBlockAsm8B: RBIT R12, R12 CLZ R12, R12 ASR $0x03, R12, R12 ADD R12, R11, R11 ADD $8, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm8B matchlen_match8_match_nolit_encodeSnappyBetterBlockAsm8B: CMPW $0x08, R7 BLO matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B MOVD (R8)(R11), R10 MOVD (R9)(R11), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R11, R11 MOVWU R11, R11 JMP matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B matchlen_bsf_8_match_nolit_encodeSnappyBetterBlockAsm8B: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R11, R11 MOVWU R11, R11 JMP match_nolit_end_encodeSnappyBetterBlockAsm8B matchlen_match4_match_nolit_encodeSnappyBetterBlockAsm8B: CMPW $0x04, R7 BLO matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B MOVWU (R8)(R11), R10 MOVWU (R9)(R11), R16 CMPW R10, R16 BNE matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R11, R11 MOVWU R11, R11 matchlen_match2_match_nolit_encodeSnappyBetterBlockAsm8B: CMPW $0x01, R7 BEQ matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B BLO match_nolit_end_encodeSnappyBetterBlockAsm8B MOVHU (R8)(R11), R16 BFI $0, R16, $16, R10 ADD R11, R9, R15 MOVHU (R15), R15 AND $0xffff, R10, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B ADD $2, R11, R11 MOVWU R11, R11 SUBSW $0x02, R7, R7 BEQ match_nolit_end_encodeSnappyBetterBlockAsm8B matchlen_match1_match_nolit_encodeSnappyBetterBlockAsm8B: MOVBU (R8)(R11), R16 BFI $0, R16, $8, R10 ADD R11, R9, R15 MOVBU (R15), R15 AND $0xff, R10, R16 CMP R16, R15 BNE match_nolit_end_encodeSnappyBetterBlockAsm8B ADD $1, R11, R11 MOVWU R11, R11 match_nolit_end_encodeSnappyBetterBlockAsm8B: MOVWU R2, R7 SUBW R5, R7, R7 // Check if repeat MOVW R7, 24(RSP) MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B MOVWU R6, R8 MOVW R6, 20(RSP) ADD R5, R3, R9 SUBW R5, R8, R8 SUB $1, R8, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_match_emit_encodeSnappyBetterBlockAsm8B CMPW $0x00000100, R5 BLO two_bytes_match_emit_encodeSnappyBetterBlockAsm8B BLO three_bytes_match_emit_encodeSnappyBetterBlockAsm8B three_bytes_match_emit_encodeSnappyBetterBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R5, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B two_bytes_match_emit_encodeSnappyBetterBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R5, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_match_emit_encodeSnappyBetterBlockAsm8B JMP memmove_long_match_emit_encodeSnappyBetterBlockAsm8B one_byte_match_emit_encodeSnappyBetterBlockAsm8B: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R1) ADD $0x01, R1, R1 memmove_match_emit_encodeSnappyBetterBlockAsm8B: ADD R8, R1, R5 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64 emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8: MOVD (R9), R10 MOVD R10, (R1) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_8through16: MOVD (R9), R10 ADD R8, R9, R15 MOVD -8(R15), R9 MOVD R10, (R1) ADD R8, R1, R15 MOVD R9, -8(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_17through32: FMOVQ (R9), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R8, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B emit_lit_memmove_match_emit_encodeSnappyBetterBlockAsm8B_memmove_move_33through64: FMOVQ (R9), F0 FMOVQ 16(R9), F1 ADD R8, R9, R15 FMOVQ -32(R15), F2 ADD R8, R9, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R8, R1, R15 FMOVQ F2, -32(R15) ADD R8, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_match_emit_encodeSnappyBetterBlockAsm8B: MOVD R5, R1 JMP emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B memmove_long_match_emit_encodeSnappyBetterBlockAsm8B: ADD R8, R1, R5 // genMemMoveLong MOVD R9, R10 MOVD R1, R12 MOVD R8, R13 emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R10, R10 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_match_emit_encodeSnappyBetterBlockAsm8Blarge_big_loop_back ADD R8, R9, R15 FMOVQ -32(R15), F0 ADD R8, R9, R15 FMOVQ -16(R15), F1 ADD R8, R1, R15 FMOVQ F0, -32(R15) ADD R8, R1, R15 FMOVQ F1, -16(R15) MOVD R5, R1 emit_literal_done_match_emit_encodeSnappyBetterBlockAsm8B: ADDW R11, R2, R2 ADDW $0x04, R11, R11 MOVW R2, 20(RSP) // emitCopy two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B: CMPW $0x40, R11 BLS two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B MOVD $0xee, R16 MOVB R16, (R1) MOVH R7, 1(R1) SUB $60, R11, R11 MOVWU R11, R11 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_encodeSnappyBetterBlockAsm8B two_byte_offset_short_match_nolit_encodeSnappyBetterBlockAsm8B: MOVWU R11, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R11 BHS emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B SUB $15, R5, R5 MOVWU R5, R5 MOVB R7, 1(R1) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R5, R5 MOVB R5, (R1) ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B emit_copy_three_match_nolit_encodeSnappyBetterBlockAsm8B: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R1) MOVH R7, 1(R1) ADD $0x03, R1, R1 match_nolit_emitcopy_end_encodeSnappyBetterBlockAsm8B: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_encodeSnappyBetterBlockAsm8B MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET match_nolit_dst_ok_encodeSnappyBetterBlockAsm8B: MOVD $0x0000cf1bbcdcbf9b, R5 MOVD $0x9e3779b1, R7 ADD $1, R6, R6 SUB $2, R2, R8 MOVD (R3)(R6), R9 ADD R6, R3, R15 MOVD 1(R15), R10 MOVD (R3)(R8), R11 ADD R8, R3, R15 MOVD 1(R15), R12 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x36, R9, R9 LSL $0x20, R10, R10 MUL R7, R10, R10 LSR $0x38, R10, R10 LSL $0x10, R11, R11 MUL R5, R11, R11 LSR $0x36, R11, R11 LSL $0x20, R12, R12 MUL R7, R12, R12 LSR $0x38, R12, R12 ADD $1, R6, R7 ADD $1, R8, R13 MOVW R6, (R0)(R9<<2) MOVW R8, (R0)(R11<<2) ADD R10<<2, R0, R15 MOVW R7, 4096(R15) ADD R12<<2, R0, R15 MOVW R13, 4096(R15) ADD R6, R8, R7 ADD $1, R7, R7 LSR $0x01, R7, R7 ADD $0x01, R6, R6 SUB $0x01, R8, R8 index_loop_encodeSnappyBetterBlockAsm8B: CMP R8, R7 BHS search_loop_encodeSnappyBetterBlockAsm8B MOVD (R3)(R6), R9 MOVD (R3)(R7), R10 LSL $0x10, R9, R9 MUL R5, R9, R9 LSR $0x36, R9, R9 LSL $0x10, R10, R10 MUL R5, R10, R10 LSR $0x36, R10, R10 MOVW R6, (R0)(R9<<2) MOVW R7, (R0)(R10<<2) ADD $0x02, R6, R6 ADD $0x02, R7, R7 JMP index_loop_encodeSnappyBetterBlockAsm8B emit_remainder_encodeSnappyBetterBlockAsm8B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_encodeSnappyBetterBlockAsm8B MOVD $0x00000000, R16 MOVD R16, ret+56(FP) RET emit_remainder_ok_encodeSnappyBetterBlockAsm8B: MOVD src_len+32(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R2 MOVWU R2, R2 CMPW $0x3c, R2 BLO one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B CMPW $0x00000100, R2 BLO two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B BLO three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B three_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B: MOVD $0xf4, R16 MOVB R16, (R1) MOVH R2, 1(R1) ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B two_bytes_emit_remainder_encodeSnappyBetterBlockAsm8B: MOVD $0xf0, R16 MOVB R16, (R1) MOVB R2, 1(R1) ADD $0x02, R1, R1 CMPW $0x40, R2 BLO memmove_emit_remainder_encodeSnappyBetterBlockAsm8B JMP memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B one_byte_emit_remainder_encodeSnappyBetterBlockAsm8B: LSLW $0x02, R2, R16 BFI $0, R16, $8, R2 MOVB R2, (R1) ADD $0x01, R1, R1 memmove_emit_remainder_encodeSnappyBetterBlockAsm8B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveShort CMP $0x03, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2 BEQ emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3 CMP $0x08, R3 BLO emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7 CMP $0x10, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16 CMP $0x20, R3 BLS emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32 JMP emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64 emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_1or2: MOVBU (R0), R16 BFI $0, R16, $8, R5 ADD R3, R0, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R0 MOVB R5, (R1) ADD R3, R1, R15 MOVB R0, -1(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_3: MOVHU (R0), R16 BFI $0, R16, $16, R5 MOVBU 2(R0), R16 BFI $0, R16, $8, R0 MOVH R5, (R1) MOVB R0, 2(R1) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_4through7: MOVWU (R0), R5 ADD R3, R0, R15 MOVWU -4(R15), R0 MOVW R5, (R1) ADD R3, R1, R15 MOVW R0, -4(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_8through16: MOVD (R0), R5 ADD R3, R0, R15 MOVD -8(R15), R0 MOVD R5, (R1) ADD R3, R1, R15 MOVD R0, -8(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_17through32: FMOVQ (R0), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R1) ADD R3, R1, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B emit_lit_memmove_emit_remainder_encodeSnappyBetterBlockAsm8B_memmove_move_33through64: FMOVQ (R0), F0 FMOVQ 16(R0), F1 ADD R3, R0, R15 FMOVQ -32(R15), F2 ADD R3, R0, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R1) FMOVQ F1, 16(R1) ADD R3, R1, R15 FMOVQ F2, -32(R15) ADD R3, R1, R15 FMOVQ F3, -16(R15) memmove_end_copy_emit_remainder_encodeSnappyBetterBlockAsm8B: MOVD R2, R1 JMP emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8B: ADD R5, R1, R2 MOVWU R5, R3 // genMemMoveLong MOVD R0, R5 MOVD R1, R6 MOVD R3, R7 emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_emit_remainder_encodeSnappyBetterBlockAsm8Blarge_big_loop_back ADD R3, R0, R15 FMOVQ -32(R15), F0 ADD R3, R0, R15 FMOVQ -16(R15), F1 ADD R3, R1, R15 FMOVQ F0, -32(R15) ADD R3, R1, R15 FMOVQ F1, -16(R15) MOVD R2, R1 emit_literal_done_emit_remainder_encodeSnappyBetterBlockAsm8B: MOVD dst_base+0(FP), R0 SUB R0, R1, R1 MOVD R1, ret+56(FP) RET // func calcBlockSize(src []byte, tmp *[32768]byte) int // Requires: BMI, SSE2 TEXT ·calcBlockSize(SB), $24-40 MOVD tmp+24(FP), R0 MOVD $0, R1 MOVD $0x00000100, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_calcBlockSize: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_calcBlockSize MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+8(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+0(FP), R3 search_loop_calcBlockSize: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x05, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_calcBlockSize MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x33, R9, R9 LSL $0x10, R10, R10 MUL R8, R10, R10 LSR $0x33, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x10, R9, R9 MUL R8, R9, R9 LSR $0x33, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_calcBlockSize ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R5 MOVWU R6, R7 MOVWU 24(RSP), R16 SUBSW R16, R7, R7 BEQ repeat_extend_back_end_calcBlockSize repeat_extend_back_loop_calcBlockSize: CMPW R5, R6 BLS repeat_extend_back_end_calcBlockSize ADD R7, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_calcBlockSize SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R7, R7 BNE repeat_extend_back_loop_calcBlockSize repeat_extend_back_end_calcBlockSize: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $5, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_calcBlockSize MOVD $0x00000000, R16 MOVD R16, ret+32(FP) RET repeat_dst_size_check_calcBlockSize: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_calcBlockSize MOVWU R6, R7 MOVW R6, 20(RSP) ADD R5, R3, R8 SUBW R5, R7, R7 SUB $1, R7, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_calcBlockSize CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_calcBlockSize CMPW $0x00010000, R5 BLO three_bytes_repeat_emit_calcBlockSize CMPW $0x01000000, R5 BLO four_bytes_repeat_emit_calcBlockSize ADD $0x05, R1, R1 JMP memmove_long_repeat_emit_calcBlockSize four_bytes_repeat_emit_calcBlockSize: ADD $0x04, R1, R1 JMP memmove_long_repeat_emit_calcBlockSize three_bytes_repeat_emit_calcBlockSize: ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_calcBlockSize two_bytes_repeat_emit_calcBlockSize: ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_calcBlockSize JMP memmove_long_repeat_emit_calcBlockSize one_byte_repeat_emit_calcBlockSize: ADD $0x01, R1, R1 memmove_repeat_emit_calcBlockSize: ADD R7, R1, R1 JMP emit_literal_done_repeat_emit_calcBlockSize memmove_long_repeat_emit_calcBlockSize: ADD R7, R1, R1 emit_literal_done_repeat_emit_calcBlockSize: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+8(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R5 // matchLen MOVD $0, R10 matchlen_loopback_16_repeat_extend_calcBlockSize: CMPW $0x10, R7 BLO matchlen_match8_repeat_extend_calcBlockSize MOVD (R8)(R10), R9 ADD R10, R8, R15 MOVD 8(R15), R11 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_calcBlockSize ADD R10, R5, R15 MOVD 8(R15), R16 EOR R16, R11, R11 TST R11, R11 BNE matchlen_bsf_16repeat_extend_calcBlockSize SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R10, R10 MOVWU R10, R10 JMP matchlen_loopback_16_repeat_extend_calcBlockSize matchlen_bsf_16repeat_extend_calcBlockSize: RBIT R11, R11 CLZ R11, R11 ASR $0x03, R11, R11 ADD R11, R10, R10 ADD $8, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_calcBlockSize matchlen_match8_repeat_extend_calcBlockSize: CMPW $0x08, R7 BLO matchlen_match4_repeat_extend_calcBlockSize MOVD (R8)(R10), R9 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_calcBlockSize SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R10, R10 MOVWU R10, R10 JMP matchlen_match4_repeat_extend_calcBlockSize matchlen_bsf_8_repeat_extend_calcBlockSize: RBIT R9, R9 CLZ R9, R9 ASR $0x03, R9, R9 ADD R9, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_calcBlockSize matchlen_match4_repeat_extend_calcBlockSize: CMPW $0x04, R7 BLO matchlen_match2_repeat_extend_calcBlockSize MOVWU (R8)(R10), R9 MOVWU (R5)(R10), R16 CMPW R9, R16 BNE matchlen_match2_repeat_extend_calcBlockSize SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R10, R10 MOVWU R10, R10 matchlen_match2_repeat_extend_calcBlockSize: CMPW $0x01, R7 BEQ matchlen_match1_repeat_extend_calcBlockSize BLO repeat_extend_forward_end_calcBlockSize MOVHU (R8)(R10), R16 BFI $0, R16, $16, R9 ADD R10, R5, R15 MOVHU (R15), R15 AND $0xffff, R9, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_calcBlockSize ADD $2, R10, R10 MOVWU R10, R10 SUBSW $0x02, R7, R7 BEQ repeat_extend_forward_end_calcBlockSize matchlen_match1_repeat_extend_calcBlockSize: MOVBU (R8)(R10), R16 BFI $0, R16, $8, R9 ADD R10, R5, R15 MOVBU (R15), R15 AND $0xff, R9, R16 CMP R16, R15 BNE repeat_extend_forward_end_calcBlockSize ADD $1, R10, R10 MOVWU R10, R10 repeat_extend_forward_end_calcBlockSize: ADDW R10, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 // emitCopy CMPW $0x00010000, R6 BLO two_byte_offset_repeat_as_copy_calcBlockSize four_bytes_loop_back_repeat_as_copy_calcBlockSize: CMPW $0x40, R5 BLS four_bytes_remain_repeat_as_copy_calcBlockSize SUB $64, R5, R5 MOVWU R5, R5 ADD $0x05, R1, R1 CMPW $0x04, R5 BLO four_bytes_remain_repeat_as_copy_calcBlockSize JMP four_bytes_loop_back_repeat_as_copy_calcBlockSize four_bytes_remain_repeat_as_copy_calcBlockSize: TSTW R5, R5 BEQ repeat_end_emit_calcBlockSize MOVD $0, R5 ADD $0x05, R1, R1 JMP repeat_end_emit_calcBlockSize two_byte_offset_repeat_as_copy_calcBlockSize: CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_calcBlockSize SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 JMP two_byte_offset_repeat_as_copy_calcBlockSize two_byte_offset_short_repeat_as_copy_calcBlockSize: MOVWU R5, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_calcBlockSize CMPW $0x00000800, R6 BHS emit_copy_three_repeat_as_copy_calcBlockSize ADD $0x02, R1, R1 JMP repeat_end_emit_calcBlockSize emit_copy_three_repeat_as_copy_calcBlockSize: ADD $0x03, R1, R1 repeat_end_emit_calcBlockSize: MOVW R2, 20(RSP) JMP search_loop_calcBlockSize no_repeat_found_calcBlockSize: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_calcBlockSize LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_calcBlockSize MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_calcBlockSize MOVWU 28(RSP), R2 JMP search_loop_calcBlockSize candidate3_match_calcBlockSize: ADDW $0x02, R2, R2 JMP candidate_match_calcBlockSize candidate2_match_calcBlockSize: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_calcBlockSize: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_calcBlockSize match_extend_back_loop_calcBlockSize: CMPW R6, R2 BLS match_extend_back_end_calcBlockSize ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_calcBlockSize SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_calcBlockSize JMP match_extend_back_loop_calcBlockSize match_extend_back_end_calcBlockSize: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $5, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_calcBlockSize MOVD $0x00000000, R16 MOVD R16, ret+32(FP) RET match_dst_size_check_calcBlockSize: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_calcBlockSize MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R6 MOVWU R6, R6 CMPW $0x3c, R6 BLO one_byte_match_emit_calcBlockSize CMPW $0x00000100, R6 BLO two_bytes_match_emit_calcBlockSize CMPW $0x00010000, R6 BLO three_bytes_match_emit_calcBlockSize CMPW $0x01000000, R6 BLO four_bytes_match_emit_calcBlockSize ADD $0x05, R1, R1 JMP memmove_long_match_emit_calcBlockSize four_bytes_match_emit_calcBlockSize: ADD $0x04, R1, R1 JMP memmove_long_match_emit_calcBlockSize three_bytes_match_emit_calcBlockSize: ADD $0x03, R1, R1 JMP memmove_long_match_emit_calcBlockSize two_bytes_match_emit_calcBlockSize: ADD $0x02, R1, R1 CMPW $0x40, R6 BLO memmove_match_emit_calcBlockSize JMP memmove_long_match_emit_calcBlockSize one_byte_match_emit_calcBlockSize: ADD $0x01, R1, R1 memmove_match_emit_calcBlockSize: ADD R8, R1, R1 JMP emit_literal_done_match_emit_calcBlockSize memmove_long_match_emit_calcBlockSize: ADD R8, R1, R1 emit_literal_done_match_emit_calcBlockSize: match_nolit_loop_calcBlockSize: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+8(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_calcBlockSize: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_calcBlockSize MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_calcBlockSize ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_calcBlockSize SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_calcBlockSize matchlen_bsf_16match_nolit_calcBlockSize: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_calcBlockSize matchlen_match8_match_nolit_calcBlockSize: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_calcBlockSize MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_calcBlockSize SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_calcBlockSize matchlen_bsf_8_match_nolit_calcBlockSize: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_calcBlockSize matchlen_match4_match_nolit_calcBlockSize: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_calcBlockSize MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_calcBlockSize SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_calcBlockSize: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_calcBlockSize BLO match_nolit_end_calcBlockSize MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_calcBlockSize ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_calcBlockSize matchlen_match1_match_nolit_calcBlockSize: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_calcBlockSize ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_calcBlockSize: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy CMPW $0x00010000, R5 BLO two_byte_offset_match_nolit_calcBlockSize four_bytes_loop_back_match_nolit_calcBlockSize: CMPW $0x40, R9 BLS four_bytes_remain_match_nolit_calcBlockSize SUB $64, R9, R9 MOVWU R9, R9 ADD $0x05, R1, R1 CMPW $0x04, R9 BLO four_bytes_remain_match_nolit_calcBlockSize JMP four_bytes_loop_back_match_nolit_calcBlockSize four_bytes_remain_match_nolit_calcBlockSize: TSTW R9, R9 BEQ match_nolit_emitcopy_end_calcBlockSize MOVD $0, R5 ADD $0x05, R1, R1 JMP match_nolit_emitcopy_end_calcBlockSize two_byte_offset_match_nolit_calcBlockSize: CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_calcBlockSize SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_calcBlockSize two_byte_offset_short_match_nolit_calcBlockSize: MOVWU R9, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_calcBlockSize CMPW $0x00000800, R5 BHS emit_copy_three_match_nolit_calcBlockSize ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_calcBlockSize emit_copy_three_match_nolit_calcBlockSize: ADD $0x03, R1, R1 match_nolit_emitcopy_end_calcBlockSize: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_calcBlockSize ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_calcBlockSize MOVD $0x00000000, R16 MOVD R16, ret+32(FP) RET match_nolit_dst_ok_calcBlockSize: MOVD $0x0000cf1bbcdcbf9b, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x10, R7, R7 MUL R8, R7, R7 LSR $0x33, R7, R7 LSL $0x10, R5, R5 MUL R8, R5, R5 LSR $0x33, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_calcBlockSize ADDW $1, R2, R2 JMP search_loop_calcBlockSize emit_remainder_calcBlockSize: MOVD src_len+8(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $5, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_calcBlockSize MOVD $0x00000000, R16 MOVD R16, ret+32(FP) RET emit_remainder_ok_calcBlockSize: MOVD src_len+8(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_calcBlockSize MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R0 MOVWU R0, R0 CMPW $0x3c, R0 BLO one_byte_emit_remainder_calcBlockSize CMPW $0x00000100, R0 BLO two_bytes_emit_remainder_calcBlockSize CMPW $0x00010000, R0 BLO three_bytes_emit_remainder_calcBlockSize CMPW $0x01000000, R0 BLO four_bytes_emit_remainder_calcBlockSize ADD $0x05, R1, R1 JMP memmove_long_emit_remainder_calcBlockSize four_bytes_emit_remainder_calcBlockSize: ADD $0x04, R1, R1 JMP memmove_long_emit_remainder_calcBlockSize three_bytes_emit_remainder_calcBlockSize: ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_calcBlockSize two_bytes_emit_remainder_calcBlockSize: ADD $0x02, R1, R1 CMPW $0x40, R0 BLO memmove_emit_remainder_calcBlockSize JMP memmove_long_emit_remainder_calcBlockSize one_byte_emit_remainder_calcBlockSize: ADD $0x01, R1, R1 memmove_emit_remainder_calcBlockSize: ADD R5, R1, R0 MOVD R0, R1 JMP emit_literal_done_emit_remainder_calcBlockSize memmove_long_emit_remainder_calcBlockSize: ADD R5, R1, R0 MOVD R0, R1 emit_literal_done_emit_remainder_calcBlockSize: MOVD R1, ret+32(FP) RET // func calcBlockSizeSmall(src []byte, tmp *[2048]byte) int // Requires: BMI, SSE2 TEXT ·calcBlockSizeSmall(SB), $24-40 MOVD tmp+24(FP), R0 MOVD $0, R1 MOVD $0x00000010, R2 MOVD R0, R3 VEOR V0.B16, V0.B16, V0.B16 zero_loop_calcBlockSizeSmall: FMOVQ F0, (R3) FMOVQ F0, 16(R3) FMOVQ F0, 32(R3) FMOVQ F0, 48(R3) FMOVQ F0, 64(R3) FMOVQ F0, 80(R3) FMOVQ F0, 96(R3) FMOVQ F0, 112(R3) ADD $0x80, R3, R3 SUBS $1, R2, R2 BNE zero_loop_calcBlockSizeSmall MOVD $0x00000000, R16 MOVW R16, 20(RSP) MOVD src_len+8(FP), R2 SUB $9, R2, R3 SUB $8, R2, R5 MOVW R5, 16(RSP) LSR $0x05, R2, R2 SUBW R2, R3, R3 ADD R3, R1, R3 MOVD R3, 8(RSP) MOVD $0x00000001, R2 MOVW R2, 24(RSP) MOVD src_base+0(FP), R3 search_loop_calcBlockSizeSmall: MOVWU R2, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 LSRW $0x04, R5, R5 ADD R5, R2, R5 ADD $4, R5, R5 MOVWU R5, R5 MOVWU 16(RSP), R16 CMPW R16, R5 BHS emit_remainder_calcBlockSizeSmall MOVD (R3)(R2), R6 MOVW R5, 28(RSP) MOVD $0x9e3779b1, R8 MOVD R6, R9 MOVD R6, R10 LSR $0x08, R10, R10 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x37, R9, R9 LSL $0x20, R10, R10 MUL R8, R10, R10 LSR $0x37, R10, R10 MOVWU (R0)(R9<<2), R5 MOVWU (R0)(R10<<2), R7 MOVW R2, (R0)(R9<<2) ADD $1, R2, R9 MOVWU R9, R9 MOVW R9, (R0)(R10<<2) MOVD R6, R9 LSR $0x10, R9, R9 LSL $0x20, R9, R9 MUL R8, R9, R9 LSR $0x37, R9, R9 MOVWU R2, R8 MOVWU 24(RSP), R16 SUBW R16, R8, R8 ADD R8, R3, R15 MOVWU 1(R15), R10 MOVD R6, R8 LSR $0x08, R8, R8 CMPW R10, R8 BNE no_repeat_found_calcBlockSizeSmall ADD $1, R2, R6 MOVWU R6, R6 MOVWU 20(RSP), R5 MOVWU R6, R7 MOVWU 24(RSP), R16 SUBSW R16, R7, R7 BEQ repeat_extend_back_end_calcBlockSizeSmall repeat_extend_back_loop_calcBlockSizeSmall: CMPW R5, R6 BLS repeat_extend_back_end_calcBlockSizeSmall ADD R7, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 ADD R6, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R9 AND $0xff, R9, R16 AND $0xff, R8, R15 CMP R16, R15 BNE repeat_extend_back_end_calcBlockSizeSmall SUB $1, R6, R6 MOVWU R6, R6 SUBSW $1, R7, R7 BNE repeat_extend_back_loop_calcBlockSizeSmall repeat_extend_back_end_calcBlockSizeSmall: MOVWU R6, R5 MOVWU 20(RSP), R16 SUBW R16, R5, R5 ADD R5, R1, R5 ADD $3, R5, R5 MOVD 8(RSP), R16 CMP R16, R5 BLO repeat_dst_size_check_calcBlockSizeSmall MOVD $0x00000000, R16 MOVD R16, ret+32(FP) RET repeat_dst_size_check_calcBlockSizeSmall: MOVWU 20(RSP), R5 CMPW R6, R5 BEQ emit_literal_done_repeat_emit_calcBlockSizeSmall MOVWU R6, R7 MOVW R6, 20(RSP) ADD R5, R3, R8 SUBW R5, R7, R7 SUB $1, R7, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_repeat_emit_calcBlockSizeSmall CMPW $0x00000100, R5 BLO two_bytes_repeat_emit_calcBlockSizeSmall BLO three_bytes_repeat_emit_calcBlockSizeSmall three_bytes_repeat_emit_calcBlockSizeSmall: ADD $0x03, R1, R1 JMP memmove_long_repeat_emit_calcBlockSizeSmall two_bytes_repeat_emit_calcBlockSizeSmall: ADD $0x02, R1, R1 CMPW $0x40, R5 BLO memmove_repeat_emit_calcBlockSizeSmall JMP memmove_long_repeat_emit_calcBlockSizeSmall one_byte_repeat_emit_calcBlockSizeSmall: ADD $0x01, R1, R1 memmove_repeat_emit_calcBlockSizeSmall: ADD R7, R1, R1 JMP emit_literal_done_repeat_emit_calcBlockSizeSmall memmove_long_repeat_emit_calcBlockSizeSmall: ADD R7, R1, R1 emit_literal_done_repeat_emit_calcBlockSizeSmall: ADDW $0x05, R2, R2 MOVWU R2, R5 MOVWU 24(RSP), R16 SUBW R16, R5, R5 MOVD src_len+8(FP), R7 SUBW R2, R7, R7 ADD R2, R3, R8 ADD R5, R3, R5 // matchLen MOVD $0, R10 matchlen_loopback_16_repeat_extend_calcBlockSizeSmall: CMPW $0x10, R7 BLO matchlen_match8_repeat_extend_calcBlockSizeSmall MOVD (R8)(R10), R9 ADD R10, R8, R15 MOVD 8(R15), R11 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_calcBlockSizeSmall ADD R10, R5, R15 MOVD 8(R15), R16 EOR R16, R11, R11 TST R11, R11 BNE matchlen_bsf_16repeat_extend_calcBlockSizeSmall SUB $16, R7, R7 MOVWU R7, R7 ADD $16, R10, R10 MOVWU R10, R10 JMP matchlen_loopback_16_repeat_extend_calcBlockSizeSmall matchlen_bsf_16repeat_extend_calcBlockSizeSmall: RBIT R11, R11 CLZ R11, R11 ASR $0x03, R11, R11 ADD R11, R10, R10 ADD $8, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_calcBlockSizeSmall matchlen_match8_repeat_extend_calcBlockSizeSmall: CMPW $0x08, R7 BLO matchlen_match4_repeat_extend_calcBlockSizeSmall MOVD (R8)(R10), R9 MOVD (R5)(R10), R16 EOR R16, R9, R9 TST R9, R9 BNE matchlen_bsf_8_repeat_extend_calcBlockSizeSmall SUB $8, R7, R7 MOVWU R7, R7 ADD $8, R10, R10 MOVWU R10, R10 JMP matchlen_match4_repeat_extend_calcBlockSizeSmall matchlen_bsf_8_repeat_extend_calcBlockSizeSmall: RBIT R9, R9 CLZ R9, R9 ASR $0x03, R9, R9 ADD R9, R10, R10 MOVWU R10, R10 JMP repeat_extend_forward_end_calcBlockSizeSmall matchlen_match4_repeat_extend_calcBlockSizeSmall: CMPW $0x04, R7 BLO matchlen_match2_repeat_extend_calcBlockSizeSmall MOVWU (R8)(R10), R9 MOVWU (R5)(R10), R16 CMPW R9, R16 BNE matchlen_match2_repeat_extend_calcBlockSizeSmall SUB $4, R7, R7 MOVWU R7, R7 ADD $4, R10, R10 MOVWU R10, R10 matchlen_match2_repeat_extend_calcBlockSizeSmall: CMPW $0x01, R7 BEQ matchlen_match1_repeat_extend_calcBlockSizeSmall BLO repeat_extend_forward_end_calcBlockSizeSmall MOVHU (R8)(R10), R16 BFI $0, R16, $16, R9 ADD R10, R5, R15 MOVHU (R15), R15 AND $0xffff, R9, R16 CMP R16, R15 BNE matchlen_match1_repeat_extend_calcBlockSizeSmall ADD $2, R10, R10 MOVWU R10, R10 SUBSW $0x02, R7, R7 BEQ repeat_extend_forward_end_calcBlockSizeSmall matchlen_match1_repeat_extend_calcBlockSizeSmall: MOVBU (R8)(R10), R16 BFI $0, R16, $8, R9 ADD R10, R5, R15 MOVBU (R15), R15 AND $0xff, R9, R16 CMP R16, R15 BNE repeat_extend_forward_end_calcBlockSizeSmall ADD $1, R10, R10 MOVWU R10, R10 repeat_extend_forward_end_calcBlockSizeSmall: ADDW R10, R2, R2 MOVWU R2, R5 SUBW R6, R5, R5 MOVWU 24(RSP), R6 // emitCopy two_byte_offset_repeat_as_copy_calcBlockSizeSmall: CMPW $0x40, R5 BLS two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall SUB $60, R5, R5 MOVWU R5, R5 ADD $0x03, R1, R1 JMP two_byte_offset_repeat_as_copy_calcBlockSizeSmall two_byte_offset_short_repeat_as_copy_calcBlockSizeSmall: MOVWU R5, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R5 BHS emit_copy_three_repeat_as_copy_calcBlockSizeSmall ADD $0x02, R1, R1 JMP repeat_end_emit_calcBlockSizeSmall emit_copy_three_repeat_as_copy_calcBlockSizeSmall: ADD $0x03, R1, R1 repeat_end_emit_calcBlockSizeSmall: MOVW R2, 20(RSP) JMP search_loop_calcBlockSizeSmall no_repeat_found_calcBlockSizeSmall: MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate_match_calcBlockSizeSmall LSR $0x08, R6, R6 MOVWU (R0)(R9<<2), R5 ADD $2, R2, R8 MOVWU R8, R8 MOVWU (R3)(R7), R16 CMPW R6, R16 BEQ candidate2_match_calcBlockSizeSmall MOVW R8, (R0)(R9<<2) LSR $0x08, R6, R6 MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ candidate3_match_calcBlockSizeSmall MOVWU 28(RSP), R2 JMP search_loop_calcBlockSizeSmall candidate3_match_calcBlockSizeSmall: ADDW $0x02, R2, R2 JMP candidate_match_calcBlockSizeSmall candidate2_match_calcBlockSizeSmall: MOVW R8, (R0)(R9<<2) ADDW $1, R2, R2 MOVWU R7, R5 candidate_match_calcBlockSizeSmall: MOVWU 20(RSP), R6 TSTW R5, R5 BEQ match_extend_back_end_calcBlockSizeSmall match_extend_back_loop_calcBlockSizeSmall: CMPW R6, R2 BLS match_extend_back_end_calcBlockSizeSmall ADD R5, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R7 ADD R2, R3, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R8 AND $0xff, R8, R16 AND $0xff, R7, R15 CMP R16, R15 BNE match_extend_back_end_calcBlockSizeSmall SUB $1, R2, R2 MOVWU R2, R2 SUBSW $1, R5, R5 BEQ match_extend_back_end_calcBlockSizeSmall JMP match_extend_back_loop_calcBlockSizeSmall match_extend_back_end_calcBlockSizeSmall: MOVWU R2, R6 MOVWU 20(RSP), R16 SUBW R16, R6, R6 ADD R6, R1, R6 ADD $3, R6, R6 MOVD 8(RSP), R16 CMP R16, R6 BLO match_dst_size_check_calcBlockSizeSmall MOVD $0x00000000, R16 MOVD R16, ret+32(FP) RET match_dst_size_check_calcBlockSizeSmall: MOVWU R2, R6 MOVWU 20(RSP), R7 CMPW R6, R7 BEQ emit_literal_done_match_emit_calcBlockSizeSmall MOVWU R6, R8 MOVW R6, 20(RSP) ADD R7, R3, R6 SUBW R7, R8, R8 SUB $1, R8, R6 MOVWU R6, R6 CMPW $0x3c, R6 BLO one_byte_match_emit_calcBlockSizeSmall CMPW $0x00000100, R6 BLO two_bytes_match_emit_calcBlockSizeSmall BLO three_bytes_match_emit_calcBlockSizeSmall three_bytes_match_emit_calcBlockSizeSmall: ADD $0x03, R1, R1 JMP memmove_long_match_emit_calcBlockSizeSmall two_bytes_match_emit_calcBlockSizeSmall: ADD $0x02, R1, R1 CMPW $0x40, R6 BLO memmove_match_emit_calcBlockSizeSmall JMP memmove_long_match_emit_calcBlockSizeSmall one_byte_match_emit_calcBlockSizeSmall: ADD $0x01, R1, R1 memmove_match_emit_calcBlockSizeSmall: ADD R8, R1, R1 JMP emit_literal_done_match_emit_calcBlockSizeSmall memmove_long_match_emit_calcBlockSizeSmall: ADD R8, R1, R1 emit_literal_done_match_emit_calcBlockSizeSmall: match_nolit_loop_calcBlockSizeSmall: MOVWU R2, R6 SUBW R5, R6, R6 MOVW R6, 24(RSP) ADDW $0x04, R2, R2 ADDW $0x04, R5, R5 MOVD src_len+8(FP), R6 SUBW R2, R6, R6 ADD R2, R3, R7 ADD R5, R3, R5 // matchLen MOVD $0, R9 matchlen_loopback_16_match_nolit_calcBlockSizeSmall: CMPW $0x10, R6 BLO matchlen_match8_match_nolit_calcBlockSizeSmall MOVD (R7)(R9), R8 ADD R9, R7, R15 MOVD 8(R15), R10 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_calcBlockSizeSmall ADD R9, R5, R15 MOVD 8(R15), R16 EOR R16, R10, R10 TST R10, R10 BNE matchlen_bsf_16match_nolit_calcBlockSizeSmall SUB $16, R6, R6 MOVWU R6, R6 ADD $16, R9, R9 MOVWU R9, R9 JMP matchlen_loopback_16_match_nolit_calcBlockSizeSmall matchlen_bsf_16match_nolit_calcBlockSizeSmall: RBIT R10, R10 CLZ R10, R10 ASR $0x03, R10, R10 ADD R10, R9, R9 ADD $8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_calcBlockSizeSmall matchlen_match8_match_nolit_calcBlockSizeSmall: CMPW $0x08, R6 BLO matchlen_match4_match_nolit_calcBlockSizeSmall MOVD (R7)(R9), R8 MOVD (R5)(R9), R16 EOR R16, R8, R8 TST R8, R8 BNE matchlen_bsf_8_match_nolit_calcBlockSizeSmall SUB $8, R6, R6 MOVWU R6, R6 ADD $8, R9, R9 MOVWU R9, R9 JMP matchlen_match4_match_nolit_calcBlockSizeSmall matchlen_bsf_8_match_nolit_calcBlockSizeSmall: RBIT R8, R8 CLZ R8, R8 ASR $0x03, R8, R8 ADD R8, R9, R9 MOVWU R9, R9 JMP match_nolit_end_calcBlockSizeSmall matchlen_match4_match_nolit_calcBlockSizeSmall: CMPW $0x04, R6 BLO matchlen_match2_match_nolit_calcBlockSizeSmall MOVWU (R7)(R9), R8 MOVWU (R5)(R9), R16 CMPW R8, R16 BNE matchlen_match2_match_nolit_calcBlockSizeSmall SUB $4, R6, R6 MOVWU R6, R6 ADD $4, R9, R9 MOVWU R9, R9 matchlen_match2_match_nolit_calcBlockSizeSmall: CMPW $0x01, R6 BEQ matchlen_match1_match_nolit_calcBlockSizeSmall BLO match_nolit_end_calcBlockSizeSmall MOVHU (R7)(R9), R16 BFI $0, R16, $16, R8 ADD R9, R5, R15 MOVHU (R15), R15 AND $0xffff, R8, R16 CMP R16, R15 BNE matchlen_match1_match_nolit_calcBlockSizeSmall ADD $2, R9, R9 MOVWU R9, R9 SUBSW $0x02, R6, R6 BEQ match_nolit_end_calcBlockSizeSmall matchlen_match1_match_nolit_calcBlockSizeSmall: MOVBU (R7)(R9), R16 BFI $0, R16, $8, R8 ADD R9, R5, R15 MOVBU (R15), R15 AND $0xff, R8, R16 CMP R16, R15 BNE match_nolit_end_calcBlockSizeSmall ADD $1, R9, R9 MOVWU R9, R9 match_nolit_end_calcBlockSizeSmall: ADDW R9, R2, R2 MOVWU 24(RSP), R5 ADDW $0x04, R9, R9 MOVW R2, 20(RSP) // emitCopy two_byte_offset_match_nolit_calcBlockSizeSmall: CMPW $0x40, R9 BLS two_byte_offset_short_match_nolit_calcBlockSizeSmall SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R1, R1 JMP two_byte_offset_match_nolit_calcBlockSizeSmall two_byte_offset_short_match_nolit_calcBlockSizeSmall: MOVWU R9, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R9 BHS emit_copy_three_match_nolit_calcBlockSizeSmall ADD $0x02, R1, R1 JMP match_nolit_emitcopy_end_calcBlockSizeSmall emit_copy_three_match_nolit_calcBlockSizeSmall: ADD $0x03, R1, R1 match_nolit_emitcopy_end_calcBlockSizeSmall: MOVWU 16(RSP), R16 CMPW R16, R2 BHS emit_remainder_calcBlockSizeSmall ADD R2, R3, R15 MOVD -2(R15), R6 MOVD 8(RSP), R16 CMP R16, R1 BLO match_nolit_dst_ok_calcBlockSizeSmall MOVD $0x00000000, R16 MOVD R16, ret+32(FP) RET match_nolit_dst_ok_calcBlockSizeSmall: MOVD $0x9e3779b1, R8 MOVD R6, R7 LSR $0x10, R6, R6 MOVD R6, R5 LSL $0x20, R7, R7 MUL R8, R7, R7 LSR $0x37, R7, R7 LSL $0x20, R5, R5 MUL R8, R5, R5 LSR $0x37, R5, R5 SUB $2, R2, R8 MOVWU R8, R8 ADD R5<<2, R0, R9 MOVWU (R9), R5 MOVW R8, (R0)(R7<<2) MOVW R2, (R9) MOVWU (R3)(R5), R16 CMPW R6, R16 BEQ match_nolit_loop_calcBlockSizeSmall ADDW $1, R2, R2 JMP search_loop_calcBlockSizeSmall emit_remainder_calcBlockSizeSmall: MOVD src_len+8(FP), R0 MOVWU 20(RSP), R16 SUBW R16, R0, R0 ADD R0, R1, R0 ADD $3, R0, R0 MOVD 8(RSP), R16 CMP R16, R0 BLO emit_remainder_ok_calcBlockSizeSmall MOVD $0x00000000, R16 MOVD R16, ret+32(FP) RET emit_remainder_ok_calcBlockSizeSmall: MOVD src_len+8(FP), R0 MOVWU 20(RSP), R2 CMPW R0, R2 BEQ emit_literal_done_emit_remainder_calcBlockSizeSmall MOVWU R0, R5 MOVW R0, 20(RSP) ADD R2, R3, R0 SUBW R2, R5, R5 SUB $1, R5, R0 MOVWU R0, R0 CMPW $0x3c, R0 BLO one_byte_emit_remainder_calcBlockSizeSmall CMPW $0x00000100, R0 BLO two_bytes_emit_remainder_calcBlockSizeSmall BLO three_bytes_emit_remainder_calcBlockSizeSmall three_bytes_emit_remainder_calcBlockSizeSmall: ADD $0x03, R1, R1 JMP memmove_long_emit_remainder_calcBlockSizeSmall two_bytes_emit_remainder_calcBlockSizeSmall: ADD $0x02, R1, R1 CMPW $0x40, R0 BLO memmove_emit_remainder_calcBlockSizeSmall JMP memmove_long_emit_remainder_calcBlockSizeSmall one_byte_emit_remainder_calcBlockSizeSmall: ADD $0x01, R1, R1 memmove_emit_remainder_calcBlockSizeSmall: ADD R5, R1, R0 MOVD R0, R1 JMP emit_literal_done_emit_remainder_calcBlockSizeSmall memmove_long_emit_remainder_calcBlockSizeSmall: ADD R5, R1, R0 MOVD R0, R1 emit_literal_done_emit_remainder_calcBlockSizeSmall: MOVD R1, ret+32(FP) RET // func emitLiteral(dst []byte, lit []byte) int // Requires: SSE2 TEXT ·emitLiteral(SB), NOSPLIT, $0-56 MOVD lit_len+32(FP), R2 MOVD dst_base+0(FP), R0 MOVD lit_base+24(FP), R1 TST R2, R2 BEQ emit_literal_end_standalone_skip MOVWU R2, R3 SUB $1, R2, R5 MOVWU R5, R5 CMPW $0x3c, R5 BLO one_byte_standalone CMPW $0x00000100, R5 BLO two_bytes_standalone CMPW $0x00010000, R5 BLO three_bytes_standalone CMPW $0x01000000, R5 BLO four_bytes_standalone MOVD $0xfc, R16 MOVB R16, (R0) MOVW R5, 1(R0) ADD $0x05, R3, R3 ADD $0x05, R0, R0 JMP memmove_long_standalone four_bytes_standalone: MOVWU R5, R6 LSRW $0x10, R6, R6 MOVD $0xf8, R16 MOVB R16, (R0) MOVH R5, 1(R0) MOVB R6, 3(R0) ADD $0x04, R3, R3 ADD $0x04, R0, R0 JMP memmove_long_standalone three_bytes_standalone: MOVD $0xf4, R16 MOVB R16, (R0) MOVH R5, 1(R0) ADD $0x03, R3, R3 ADD $0x03, R0, R0 JMP memmove_long_standalone two_bytes_standalone: MOVD $0xf0, R16 MOVB R16, (R0) MOVB R5, 1(R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 CMPW $0x40, R5 BLO memmove_standalone JMP memmove_long_standalone one_byte_standalone: LSLW $0x02, R5, R16 BFI $0, R16, $8, R5 MOVB R5, (R0) ADD $0x01, R3, R3 ADD $0x01, R0, R0 memmove_standalone: // genMemMoveShort CMP $0x03, R2 BLO emit_lit_memmove_standalone_memmove_move_1or2 BEQ emit_lit_memmove_standalone_memmove_move_3 CMP $0x08, R2 BLO emit_lit_memmove_standalone_memmove_move_4through7 CMP $0x10, R2 BLS emit_lit_memmove_standalone_memmove_move_8through16 CMP $0x20, R2 BLS emit_lit_memmove_standalone_memmove_move_17through32 JMP emit_lit_memmove_standalone_memmove_move_33through64 emit_lit_memmove_standalone_memmove_move_1or2: MOVBU (R1), R16 BFI $0, R16, $8, R5 ADD R2, R1, R15 MOVBU -1(R15), R16 BFI $0, R16, $8, R1 MOVB R5, (R0) ADD R2, R0, R15 MOVB R1, -1(R15) JMP emit_literal_end_standalone emit_lit_memmove_standalone_memmove_move_3: MOVHU (R1), R16 BFI $0, R16, $16, R5 MOVBU 2(R1), R16 BFI $0, R16, $8, R1 MOVH R5, (R0) MOVB R1, 2(R0) JMP emit_literal_end_standalone emit_lit_memmove_standalone_memmove_move_4through7: MOVWU (R1), R5 ADD R2, R1, R15 MOVWU -4(R15), R1 MOVW R5, (R0) ADD R2, R0, R15 MOVW R1, -4(R15) JMP emit_literal_end_standalone emit_lit_memmove_standalone_memmove_move_8through16: MOVD (R1), R5 ADD R2, R1, R15 MOVD -8(R15), R1 MOVD R5, (R0) ADD R2, R0, R15 MOVD R1, -8(R15) JMP emit_literal_end_standalone emit_lit_memmove_standalone_memmove_move_17through32: FMOVQ (R1), F0 ADD R2, R1, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R0) ADD R2, R0, R15 FMOVQ F1, -16(R15) JMP emit_literal_end_standalone emit_lit_memmove_standalone_memmove_move_33through64: FMOVQ (R1), F0 FMOVQ 16(R1), F1 ADD R2, R1, R15 FMOVQ -32(R15), F2 ADD R2, R1, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R0) FMOVQ F1, 16(R0) ADD R2, R0, R15 FMOVQ F2, -32(R15) ADD R2, R0, R15 FMOVQ F3, -16(R15) JMP emit_literal_end_standalone JMP emit_literal_end_standalone memmove_long_standalone: // genMemMoveLong MOVD R1, R5 MOVD R0, R6 MOVD R2, R7 emit_lit_memmove_long_standalonelarge_big_loop_back: FMOVQ (R5), F0 FMOVQ 16(R5), F1 FMOVQ F0, (R6) FMOVQ F1, 16(R6) ADD $0x20, R5, R5 ADD $0x20, R6, R6 SUB $0x20, R7, R7 CMP $0x20, R7 BHS emit_lit_memmove_long_standalonelarge_big_loop_back ADD R2, R1, R15 FMOVQ -32(R15), F0 ADD R2, R1, R15 FMOVQ -16(R15), F1 ADD R2, R0, R15 FMOVQ F0, -32(R15) ADD R2, R0, R15 FMOVQ F1, -16(R15) JMP emit_literal_end_standalone JMP emit_literal_end_standalone emit_literal_end_standalone_skip: MOVD $0, R3 emit_literal_end_standalone: MOVD R3, ret+48(FP) RET // func emitRepeat(dst []byte, offset int, length int) int TEXT ·emitRepeat(SB), NOSPLIT, $0-48 MOVD $0, R3 MOVD dst_base+0(FP), R0 MOVD offset+24(FP), R1 MOVD length+32(FP), R2 // emitRepeat emit_repeat_again_standalone: MOVWU R2, R5 SUB $4, R2, R2 MOVWU R2, R2 CMPW $0x08, R5 BLS repeat_two_standalone CMPW $0x0c, R5 BHS cant_repeat_two_offset_standalone CMPW $0x00000800, R1 BLO repeat_two_offset_standalone cant_repeat_two_offset_standalone: CMPW $0x00000104, R2 BLO repeat_three_standalone CMPW $0x00010100, R2 BLO repeat_four_standalone CMPW $0x0100ffff, R2 BLO repeat_five_standalone SUB $16842747, R2, R2 MOVWU R2, R2 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 ADD $0x05, R3, R3 JMP emit_repeat_again_standalone repeat_five_standalone: SUB $65536, R2, R2 MOVWU R2, R2 MOVWU R2, R1 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R2, 2(R0) ASRW $0x10, R1, R1 MOVB R1, 4(R0) ADD $0x05, R3, R3 ADD $0x05, R0, R0 JMP gen_emit_repeat_end repeat_four_standalone: SUB $256, R2, R2 MOVWU R2, R2 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R2, 2(R0) ADD $0x04, R3, R3 ADD $0x04, R0, R0 JMP gen_emit_repeat_end repeat_three_standalone: SUB $4, R2, R2 MOVWU R2, R2 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R2, 2(R0) ADD $0x03, R3, R3 ADD $0x03, R0, R0 JMP gen_emit_repeat_end repeat_two_standalone: LSLW $0x02, R2, R2 ORRW $0x01, R2, R2 MOVH R2, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 JMP gen_emit_repeat_end repeat_two_offset_standalone: MOVD $0, R5 ADD R2<<2, R5, R2 ADD $1, R2, R2 MOVWU R2, R2 MOVB R1, 1(R0) ASRW $0x08, R1, R1 LSLW $0x05, R1, R1 ORRW R1, R2, R2 MOVB R2, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 gen_emit_repeat_end: MOVD R3, ret+40(FP) RET // func emitCopy(dst []byte, offset int, length int) int TEXT ·emitCopy(SB), NOSPLIT, $0-48 MOVD $0, R3 MOVD dst_base+0(FP), R0 MOVD offset+24(FP), R1 MOVD length+32(FP), R2 // emitCopy CMPW $0x00010000, R1 BLO two_byte_offset_standalone CMPW $0x40, R2 BLS four_bytes_remain_standalone MOVD $0xff, R16 MOVB R16, (R0) MOVW R1, 1(R0) SUB $64, R2, R2 MOVWU R2, R2 ADD $0x05, R3, R3 ADD $0x05, R0, R0 CMPW $0x04, R2 BLO four_bytes_remain_standalone // emitRepeat emit_repeat_again_standalone_emit_copy: MOVWU R2, R5 SUB $4, R2, R2 MOVWU R2, R2 CMPW $0x08, R5 BLS repeat_two_standalone_emit_copy CMPW $0x0c, R5 BHS cant_repeat_two_offset_standalone_emit_copy CMPW $0x00000800, R1 BLO repeat_two_offset_standalone_emit_copy cant_repeat_two_offset_standalone_emit_copy: CMPW $0x00000104, R2 BLO repeat_three_standalone_emit_copy CMPW $0x00010100, R2 BLO repeat_four_standalone_emit_copy CMPW $0x0100ffff, R2 BLO repeat_five_standalone_emit_copy SUB $16842747, R2, R2 MOVWU R2, R2 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 ADD $0x05, R3, R3 JMP emit_repeat_again_standalone_emit_copy repeat_five_standalone_emit_copy: SUB $65536, R2, R2 MOVWU R2, R2 MOVWU R2, R1 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R2, 2(R0) ASRW $0x10, R1, R1 MOVB R1, 4(R0) ADD $0x05, R3, R3 ADD $0x05, R0, R0 JMP gen_emit_copy_end repeat_four_standalone_emit_copy: SUB $256, R2, R2 MOVWU R2, R2 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R2, 2(R0) ADD $0x04, R3, R3 ADD $0x04, R0, R0 JMP gen_emit_copy_end repeat_three_standalone_emit_copy: SUB $4, R2, R2 MOVWU R2, R2 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R2, 2(R0) ADD $0x03, R3, R3 ADD $0x03, R0, R0 JMP gen_emit_copy_end repeat_two_standalone_emit_copy: LSLW $0x02, R2, R2 ORRW $0x01, R2, R2 MOVH R2, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 JMP gen_emit_copy_end repeat_two_offset_standalone_emit_copy: MOVD $0, R5 ADD R2<<2, R5, R2 ADD $1, R2, R2 MOVWU R2, R2 MOVB R1, 1(R0) ASRW $0x08, R1, R1 LSLW $0x05, R1, R1 ORRW R1, R2, R2 MOVB R2, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 JMP gen_emit_copy_end four_bytes_remain_standalone: TSTW R2, R2 BEQ gen_emit_copy_end MOVD $0, R5 ADD R2<<2, R5, R2 SUB $1, R2, R2 MOVWU R2, R2 MOVB R2, (R0) MOVW R1, 1(R0) ADD $0x05, R3, R3 ADD $0x05, R0, R0 JMP gen_emit_copy_end two_byte_offset_standalone: CMPW $0x40, R2 BLS two_byte_offset_short_standalone CMPW $0x00000800, R1 BHS long_offset_short_standalone MOVD $0x00000001, R5 ADD $16, R5, R5 MOVWU R5, R5 MOVB R1, 1(R0) MOVWU R1, R6 LSRW $0x08, R6, R6 LSLW $0x05, R6, R6 ORRW R6, R5, R5 MOVB R5, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 SUBW $0x08, R2, R2 // emitRepeat SUB $4, R2, R2 MOVWU R2, R2 JMP cant_repeat_two_offset_standalone_emit_copy_short_2b emit_repeat_again_standalone_emit_copy_short_2b: MOVWU R2, R5 SUB $4, R2, R2 MOVWU R2, R2 CMPW $0x08, R5 BLS repeat_two_standalone_emit_copy_short_2b CMPW $0x0c, R5 BHS cant_repeat_two_offset_standalone_emit_copy_short_2b CMPW $0x00000800, R1 BLO repeat_two_offset_standalone_emit_copy_short_2b cant_repeat_two_offset_standalone_emit_copy_short_2b: CMPW $0x00000104, R2 BLO repeat_three_standalone_emit_copy_short_2b CMPW $0x00010100, R2 BLO repeat_four_standalone_emit_copy_short_2b CMPW $0x0100ffff, R2 BLO repeat_five_standalone_emit_copy_short_2b SUB $16842747, R2, R2 MOVWU R2, R2 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 ADD $0x05, R3, R3 JMP emit_repeat_again_standalone_emit_copy_short_2b repeat_five_standalone_emit_copy_short_2b: SUB $65536, R2, R2 MOVWU R2, R2 MOVWU R2, R1 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R2, 2(R0) ASRW $0x10, R1, R1 MOVB R1, 4(R0) ADD $0x05, R3, R3 ADD $0x05, R0, R0 JMP gen_emit_copy_end repeat_four_standalone_emit_copy_short_2b: SUB $256, R2, R2 MOVWU R2, R2 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R2, 2(R0) ADD $0x04, R3, R3 ADD $0x04, R0, R0 JMP gen_emit_copy_end repeat_three_standalone_emit_copy_short_2b: SUB $4, R2, R2 MOVWU R2, R2 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R2, 2(R0) ADD $0x03, R3, R3 ADD $0x03, R0, R0 JMP gen_emit_copy_end repeat_two_standalone_emit_copy_short_2b: LSLW $0x02, R2, R2 ORRW $0x01, R2, R2 MOVH R2, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 JMP gen_emit_copy_end repeat_two_offset_standalone_emit_copy_short_2b: MOVD $0, R5 ADD R2<<2, R5, R2 ADD $1, R2, R2 MOVWU R2, R2 MOVB R1, 1(R0) ASRW $0x08, R1, R1 LSLW $0x05, R1, R1 ORRW R1, R2, R2 MOVB R2, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 JMP gen_emit_copy_end long_offset_short_standalone: MOVD $0xee, R16 MOVB R16, (R0) MOVH R1, 1(R0) SUB $60, R2, R2 MOVWU R2, R2 ADD $0x03, R0, R0 ADD $0x03, R3, R3 // emitRepeat emit_repeat_again_standalone_emit_copy_short: MOVWU R2, R5 SUB $4, R2, R2 MOVWU R2, R2 CMPW $0x08, R5 BLS repeat_two_standalone_emit_copy_short CMPW $0x0c, R5 BHS cant_repeat_two_offset_standalone_emit_copy_short CMPW $0x00000800, R1 BLO repeat_two_offset_standalone_emit_copy_short cant_repeat_two_offset_standalone_emit_copy_short: CMPW $0x00000104, R2 BLO repeat_three_standalone_emit_copy_short CMPW $0x00010100, R2 BLO repeat_four_standalone_emit_copy_short CMPW $0x0100ffff, R2 BLO repeat_five_standalone_emit_copy_short SUB $16842747, R2, R2 MOVWU R2, R2 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 ADD $0x05, R3, R3 JMP emit_repeat_again_standalone_emit_copy_short repeat_five_standalone_emit_copy_short: SUB $65536, R2, R2 MOVWU R2, R2 MOVWU R2, R1 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R2, 2(R0) ASRW $0x10, R1, R1 MOVB R1, 4(R0) ADD $0x05, R3, R3 ADD $0x05, R0, R0 JMP gen_emit_copy_end repeat_four_standalone_emit_copy_short: SUB $256, R2, R2 MOVWU R2, R2 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R2, 2(R0) ADD $0x04, R3, R3 ADD $0x04, R0, R0 JMP gen_emit_copy_end repeat_three_standalone_emit_copy_short: SUB $4, R2, R2 MOVWU R2, R2 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R2, 2(R0) ADD $0x03, R3, R3 ADD $0x03, R0, R0 JMP gen_emit_copy_end repeat_two_standalone_emit_copy_short: LSLW $0x02, R2, R2 ORRW $0x01, R2, R2 MOVH R2, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 JMP gen_emit_copy_end repeat_two_offset_standalone_emit_copy_short: MOVD $0, R5 ADD R2<<2, R5, R2 ADD $1, R2, R2 MOVWU R2, R2 MOVB R1, 1(R0) ASRW $0x08, R1, R1 LSLW $0x05, R1, R1 ORRW R1, R2, R2 MOVB R2, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 JMP gen_emit_copy_end two_byte_offset_short_standalone: MOVWU R2, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R2 BHS emit_copy_three_standalone CMPW $0x00000800, R1 BHS emit_copy_three_standalone SUB $15, R5, R5 MOVWU R5, R5 MOVB R1, 1(R0) LSRW $0x08, R1, R1 LSLW $0x05, R1, R1 ORRW R1, R5, R5 MOVB R5, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 JMP gen_emit_copy_end emit_copy_three_standalone: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R0) MOVH R1, 1(R0) ADD $0x03, R3, R3 ADD $0x03, R0, R0 gen_emit_copy_end: MOVD R3, ret+40(FP) RET // func emitCopyNoRepeat(dst []byte, offset int, length int) int TEXT ·emitCopyNoRepeat(SB), NOSPLIT, $0-48 MOVD $0, R3 MOVD dst_base+0(FP), R0 MOVD offset+24(FP), R1 MOVD length+32(FP), R2 // emitCopy CMPW $0x00010000, R1 BLO two_byte_offset_standalone_snappy four_bytes_loop_back_standalone_snappy: CMPW $0x40, R2 BLS four_bytes_remain_standalone_snappy MOVD $0xff, R16 MOVB R16, (R0) MOVW R1, 1(R0) SUB $64, R2, R2 MOVWU R2, R2 ADD $0x05, R3, R3 ADD $0x05, R0, R0 CMPW $0x04, R2 BLO four_bytes_remain_standalone_snappy JMP four_bytes_loop_back_standalone_snappy four_bytes_remain_standalone_snappy: TSTW R2, R2 BEQ gen_emit_copy_end_snappy MOVD $0, R5 ADD R2<<2, R5, R2 SUB $1, R2, R2 MOVWU R2, R2 MOVB R2, (R0) MOVW R1, 1(R0) ADD $0x05, R3, R3 ADD $0x05, R0, R0 JMP gen_emit_copy_end_snappy two_byte_offset_standalone_snappy: CMPW $0x40, R2 BLS two_byte_offset_short_standalone_snappy MOVD $0xee, R16 MOVB R16, (R0) MOVH R1, 1(R0) SUB $60, R2, R2 MOVWU R2, R2 ADD $0x03, R0, R0 ADD $0x03, R3, R3 JMP two_byte_offset_standalone_snappy two_byte_offset_short_standalone_snappy: MOVWU R2, R5 LSLW $0x02, R5, R5 CMPW $0x0c, R2 BHS emit_copy_three_standalone_snappy CMPW $0x00000800, R1 BHS emit_copy_three_standalone_snappy SUB $15, R5, R5 MOVWU R5, R5 MOVB R1, 1(R0) LSRW $0x08, R1, R1 LSLW $0x05, R1, R1 ORRW R1, R5, R5 MOVB R5, (R0) ADD $0x02, R3, R3 ADD $0x02, R0, R0 JMP gen_emit_copy_end_snappy emit_copy_three_standalone_snappy: SUB $2, R5, R5 MOVWU R5, R5 MOVB R5, (R0) MOVH R1, 1(R0) ADD $0x03, R3, R3 ADD $0x03, R0, R0 gen_emit_copy_end_snappy: MOVD R3, ret+40(FP) RET // func matchLen(a []byte, b []byte) int // Requires: BMI TEXT ·matchLen(SB), NOSPLIT, $0-56 MOVD a_base+0(FP), R0 MOVD b_base+24(FP), R1 MOVD a_len+8(FP), R2 // matchLen MOVD $0, R5 matchlen_loopback_16_standalone: CMPW $0x10, R2 BLO matchlen_match8_standalone MOVD (R0)(R5), R3 ADD R5, R0, R15 MOVD 8(R15), R6 MOVD (R1)(R5), R16 EOR R16, R3, R3 TST R3, R3 BNE matchlen_bsf_8_standalone ADD R5, R1, R15 MOVD 8(R15), R16 EOR R16, R6, R6 TST R6, R6 BNE matchlen_bsf_16standalone SUB $16, R2, R2 MOVWU R2, R2 ADD $16, R5, R5 MOVWU R5, R5 JMP matchlen_loopback_16_standalone matchlen_bsf_16standalone: RBIT R6, R6 CLZ R6, R6 ASR $0x03, R6, R6 ADD R6, R5, R5 ADD $8, R5, R5 MOVWU R5, R5 JMP gen_match_len_end matchlen_match8_standalone: CMPW $0x08, R2 BLO matchlen_match4_standalone MOVD (R0)(R5), R3 MOVD (R1)(R5), R16 EOR R16, R3, R3 TST R3, R3 BNE matchlen_bsf_8_standalone SUB $8, R2, R2 MOVWU R2, R2 ADD $8, R5, R5 MOVWU R5, R5 JMP matchlen_match4_standalone matchlen_bsf_8_standalone: RBIT R3, R3 CLZ R3, R3 ASR $0x03, R3, R3 ADD R3, R5, R5 MOVWU R5, R5 JMP gen_match_len_end matchlen_match4_standalone: CMPW $0x04, R2 BLO matchlen_match2_standalone MOVWU (R0)(R5), R3 MOVWU (R1)(R5), R16 CMPW R3, R16 BNE matchlen_match2_standalone SUB $4, R2, R2 MOVWU R2, R2 ADD $4, R5, R5 MOVWU R5, R5 matchlen_match2_standalone: CMPW $0x01, R2 BEQ matchlen_match1_standalone BLO gen_match_len_end MOVHU (R0)(R5), R16 BFI $0, R16, $16, R3 ADD R5, R1, R15 MOVHU (R15), R15 AND $0xffff, R3, R16 CMP R16, R15 BNE matchlen_match1_standalone ADD $2, R5, R5 MOVWU R5, R5 SUBSW $0x02, R2, R2 BEQ gen_match_len_end matchlen_match1_standalone: MOVBU (R0)(R5), R16 BFI $0, R16, $8, R3 ADD R5, R1, R15 MOVBU (R15), R15 AND $0xff, R3, R16 CMP R16, R15 BNE gen_match_len_end ADD $1, R5, R5 MOVWU R5, R5 gen_match_len_end: MOVD R5, ret+48(FP) RET // func cvtLZ4BlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 TEXT ·cvtLZ4BlockAsm(SB), NOSPLIT, $0-64 MOVD $0, R5 MOVD dst_base+0(FP), R0 MOVD dst_len+8(FP), R1 MOVD src_base+24(FP), R2 MOVD src_len+32(FP), R3 ADD R3, R2, R3 ADD R1, R0, R1 SUB $8, R1, R1 MOVD $0, R6 lz4_s2_loop: CMP R3, R2 BHS lz4_s2_corrupt CMP R1, R0 BHS lz4_s2_dstfull MOVBU (R2), R7 MOVD R7, R8 MOVD R7, R9 LSR $0x04, R8, R8 AND $0x0f, R9, R9 CMP $0xf0, R7 BLO lz4_s2_ll_end lz4_s2_ll_loop: ADD $1, R2, R2 CMP R3, R2 BHS lz4_s2_corrupt MOVBU (R2), R7 ADD R7, R8, R8 CMP $0xff, R7 BEQ lz4_s2_ll_loop lz4_s2_ll_end: ADD R8, R2, R7 ADD $0x04, R9, R9 CMP R3, R7 BHS lz4_s2_corrupt ADD $1, R2, R2 ADD $1, R7, R7 TST R8, R8 BEQ lz4_s2_lits_done ADD R8, R0, R10 CMP R1, R10 BHS lz4_s2_dstfull ADD R8, R5, R5 SUB $1, R8, R10 MOVWU R10, R10 CMPW $0x3c, R10 BLO one_byte_lz4_s2 CMPW $0x00000100, R10 BLO two_bytes_lz4_s2 CMPW $0x00010000, R10 BLO three_bytes_lz4_s2 CMPW $0x01000000, R10 BLO four_bytes_lz4_s2 MOVD $0xfc, R16 MOVB R16, (R0) MOVW R10, 1(R0) ADD $0x05, R0, R0 JMP memmove_long_lz4_s2 four_bytes_lz4_s2: MOVWU R10, R11 LSRW $0x10, R11, R11 MOVD $0xf8, R16 MOVB R16, (R0) MOVH R10, 1(R0) MOVB R11, 3(R0) ADD $0x04, R0, R0 JMP memmove_long_lz4_s2 three_bytes_lz4_s2: MOVD $0xf4, R16 MOVB R16, (R0) MOVH R10, 1(R0) ADD $0x03, R0, R0 JMP memmove_long_lz4_s2 two_bytes_lz4_s2: MOVD $0xf0, R16 MOVB R16, (R0) MOVB R10, 1(R0) ADD $0x02, R0, R0 CMPW $0x40, R10 BLO memmove_lz4_s2 JMP memmove_long_lz4_s2 one_byte_lz4_s2: LSLW $0x02, R10, R16 BFI $0, R16, $8, R10 MOVB R10, (R0) ADD $0x01, R0, R0 memmove_lz4_s2: ADD R8, R0, R10 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_lz4_s2_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_lz4_s2_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_lz4_s2_memmove_move_17through32 JMP emit_lit_memmove_lz4_s2_memmove_move_33through64 emit_lit_memmove_lz4_s2_memmove_move_8: MOVD (R2), R11 MOVD R11, (R0) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_8through16: MOVD (R2), R11 ADD R8, R2, R15 MOVD -8(R15), R2 MOVD R11, (R0) ADD R8, R0, R15 MOVD R2, -8(R15) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_17through32: FMOVQ (R2), F0 ADD R8, R2, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R0) ADD R8, R0, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_lz4_s2 emit_lit_memmove_lz4_s2_memmove_move_33through64: FMOVQ (R2), F0 FMOVQ 16(R2), F1 ADD R8, R2, R15 FMOVQ -32(R15), F2 ADD R8, R2, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R0) FMOVQ F1, 16(R0) ADD R8, R0, R15 FMOVQ F2, -32(R15) ADD R8, R0, R15 FMOVQ F3, -16(R15) memmove_end_copy_lz4_s2: MOVD R10, R0 JMP lz4_s2_lits_emit_done memmove_long_lz4_s2: ADD R8, R0, R10 // genMemMoveLong MOVD R2, R11 MOVD R0, R12 MOVD R8, R13 emit_lit_memmove_long_lz4_s2large_big_loop_back: FMOVQ (R11), F0 FMOVQ 16(R11), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R11, R11 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_lz4_s2large_big_loop_back ADD R8, R2, R15 FMOVQ -32(R15), F0 ADD R8, R2, R15 FMOVQ -16(R15), F1 ADD R8, R0, R15 FMOVQ F0, -32(R15) ADD R8, R0, R15 FMOVQ F1, -16(R15) MOVD R10, R0 lz4_s2_lits_emit_done: MOVD R7, R2 lz4_s2_lits_done: CMP R3, R2 BNE lz4_s2_match CMP $0x04, R9 BEQ lz4_s2_done JMP lz4_s2_corrupt lz4_s2_match: ADD $2, R2, R7 CMP R3, R7 BHS lz4_s2_corrupt MOVHU (R2), R8 MOVD R7, R2 TST R8, R8 BEQ lz4_s2_corrupt CMP R5, R8 BHI lz4_s2_corrupt CMP $0x13, R9 BNE lz4_s2_ml_done lz4_s2_ml_loop: MOVBU (R2), R7 ADD $1, R2, R2 ADD R7, R9, R9 CMP R3, R2 BHS lz4_s2_corrupt CMP $0xff, R7 BEQ lz4_s2_ml_loop lz4_s2_ml_done: ADD R9, R5, R5 CMP R6, R8 BNE lz4_s2_docopy // emitRepeat emit_repeat_again_lz4_s2: MOVWU R9, R7 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R7 BLS repeat_two_lz4_s2 CMPW $0x0c, R7 BHS cant_repeat_two_offset_lz4_s2 CMPW $0x00000800, R8 BLO repeat_two_offset_lz4_s2 cant_repeat_two_offset_lz4_s2: CMPW $0x00000104, R9 BLO repeat_three_lz4_s2 CMPW $0x00010100, R9 BLO repeat_four_lz4_s2 CMPW $0x0100ffff, R9 BLO repeat_five_lz4_s2 SUB $16842747, R9, R9 MOVWU R9, R9 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 JMP emit_repeat_again_lz4_s2 repeat_five_lz4_s2: SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R8 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R9, 2(R0) ASRW $0x10, R8, R8 MOVB R8, 4(R0) ADD $0x05, R0, R0 JMP lz4_s2_loop repeat_four_lz4_s2: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R9, 2(R0) ADD $0x04, R0, R0 JMP lz4_s2_loop repeat_three_lz4_s2: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R9, 2(R0) ADD $0x03, R0, R0 JMP lz4_s2_loop repeat_two_lz4_s2: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R0) ADD $0x02, R0, R0 JMP lz4_s2_loop repeat_two_offset_lz4_s2: MOVD $0, R7 ADD R9<<2, R7, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R8, 1(R0) ASRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R9, R9 MOVB R9, (R0) ADD $0x02, R0, R0 JMP lz4_s2_loop lz4_s2_docopy: MOVD R8, R6 // emitCopy CMPW $0x40, R9 BLS two_byte_offset_short_lz4_s2 CMPW $0x00000800, R8 BHS long_offset_short_lz4_s2 MOVD $0x00000001, R7 ADD $16, R7, R7 MOVWU R7, R7 MOVB R8, 1(R0) MOVWU R8, R10 LSRW $0x08, R10, R10 LSLW $0x05, R10, R10 ORRW R10, R7, R7 MOVB R7, (R0) ADD $0x02, R0, R0 SUBW $0x08, R9, R9 // emitRepeat SUB $4, R9, R9 MOVWU R9, R9 JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b emit_repeat_again_lz4_s2_emit_copy_short_2b: MOVWU R9, R7 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R7 BLS repeat_two_lz4_s2_emit_copy_short_2b CMPW $0x0c, R7 BHS cant_repeat_two_offset_lz4_s2_emit_copy_short_2b CMPW $0x00000800, R8 BLO repeat_two_offset_lz4_s2_emit_copy_short_2b cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: CMPW $0x00000104, R9 BLO repeat_three_lz4_s2_emit_copy_short_2b CMPW $0x00010100, R9 BLO repeat_four_lz4_s2_emit_copy_short_2b CMPW $0x0100ffff, R9 BLO repeat_five_lz4_s2_emit_copy_short_2b SUB $16842747, R9, R9 MOVWU R9, R9 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 JMP emit_repeat_again_lz4_s2_emit_copy_short_2b repeat_five_lz4_s2_emit_copy_short_2b: SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R8 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R9, 2(R0) ASRW $0x10, R8, R8 MOVB R8, 4(R0) ADD $0x05, R0, R0 JMP lz4_s2_loop repeat_four_lz4_s2_emit_copy_short_2b: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R9, 2(R0) ADD $0x04, R0, R0 JMP lz4_s2_loop repeat_three_lz4_s2_emit_copy_short_2b: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R9, 2(R0) ADD $0x03, R0, R0 JMP lz4_s2_loop repeat_two_lz4_s2_emit_copy_short_2b: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R0) ADD $0x02, R0, R0 JMP lz4_s2_loop repeat_two_offset_lz4_s2_emit_copy_short_2b: MOVD $0, R7 ADD R9<<2, R7, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R8, 1(R0) ASRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R9, R9 MOVB R9, (R0) ADD $0x02, R0, R0 JMP lz4_s2_loop long_offset_short_lz4_s2: MOVD $0xee, R16 MOVB R16, (R0) MOVH R8, 1(R0) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R0, R0 // emitRepeat emit_repeat_again_lz4_s2_emit_copy_short: MOVWU R9, R7 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R7 BLS repeat_two_lz4_s2_emit_copy_short CMPW $0x0c, R7 BHS cant_repeat_two_offset_lz4_s2_emit_copy_short CMPW $0x00000800, R8 BLO repeat_two_offset_lz4_s2_emit_copy_short cant_repeat_two_offset_lz4_s2_emit_copy_short: CMPW $0x00000104, R9 BLO repeat_three_lz4_s2_emit_copy_short CMPW $0x00010100, R9 BLO repeat_four_lz4_s2_emit_copy_short CMPW $0x0100ffff, R9 BLO repeat_five_lz4_s2_emit_copy_short SUB $16842747, R9, R9 MOVWU R9, R9 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 JMP emit_repeat_again_lz4_s2_emit_copy_short repeat_five_lz4_s2_emit_copy_short: SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R8 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R9, 2(R0) ASRW $0x10, R8, R8 MOVB R8, 4(R0) ADD $0x05, R0, R0 JMP lz4_s2_loop repeat_four_lz4_s2_emit_copy_short: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R9, 2(R0) ADD $0x04, R0, R0 JMP lz4_s2_loop repeat_three_lz4_s2_emit_copy_short: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R9, 2(R0) ADD $0x03, R0, R0 JMP lz4_s2_loop repeat_two_lz4_s2_emit_copy_short: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R0) ADD $0x02, R0, R0 JMP lz4_s2_loop repeat_two_offset_lz4_s2_emit_copy_short: MOVD $0, R7 ADD R9<<2, R7, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R8, 1(R0) ASRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R9, R9 MOVB R9, (R0) ADD $0x02, R0, R0 JMP lz4_s2_loop two_byte_offset_short_lz4_s2: MOVWU R9, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R9 BHS emit_copy_three_lz4_s2 CMPW $0x00000800, R8 BHS emit_copy_three_lz4_s2 SUB $15, R7, R7 MOVWU R7, R7 MOVB R8, 1(R0) LSRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R7, R7 MOVB R7, (R0) ADD $0x02, R0, R0 JMP lz4_s2_loop emit_copy_three_lz4_s2: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R0) MOVH R8, 1(R0) ADD $0x03, R0, R0 JMP lz4_s2_loop lz4_s2_done: MOVD dst_base+0(FP), R1 SUB R1, R0, R0 MOVD R5, uncompressed+48(FP) MOVD R0, dstUsed+56(FP) RET lz4_s2_corrupt: MOVD $0, R0 SUB $1, R0, R5 MOVD R5, uncompressed+48(FP) RET lz4_s2_dstfull: MOVD $0, R0 SUB $2, R0, R5 MOVD R5, uncompressed+48(FP) RET // func cvtLZ4sBlockAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 TEXT ·cvtLZ4sBlockAsm(SB), NOSPLIT, $0-64 MOVD $0, R5 MOVD dst_base+0(FP), R0 MOVD dst_len+8(FP), R1 MOVD src_base+24(FP), R2 MOVD src_len+32(FP), R3 ADD R3, R2, R3 ADD R1, R0, R1 SUB $8, R1, R1 MOVD $0, R6 lz4s_s2_loop: CMP R3, R2 BHS lz4s_s2_corrupt CMP R1, R0 BHS lz4s_s2_dstfull MOVBU (R2), R7 MOVD R7, R8 MOVD R7, R9 LSR $0x04, R8, R8 AND $0x0f, R9, R9 CMP $0xf0, R7 BLO lz4s_s2_ll_end lz4s_s2_ll_loop: ADD $1, R2, R2 CMP R3, R2 BHS lz4s_s2_corrupt MOVBU (R2), R7 ADD R7, R8, R8 CMP $0xff, R7 BEQ lz4s_s2_ll_loop lz4s_s2_ll_end: ADD R8, R2, R7 ADD $0x03, R9, R9 CMP R3, R7 BHS lz4s_s2_corrupt ADD $1, R2, R2 ADD $1, R7, R7 TST R8, R8 BEQ lz4s_s2_lits_done ADD R8, R0, R10 CMP R1, R10 BHS lz4s_s2_dstfull ADD R8, R5, R5 SUB $1, R8, R10 MOVWU R10, R10 CMPW $0x3c, R10 BLO one_byte_lz4s_s2 CMPW $0x00000100, R10 BLO two_bytes_lz4s_s2 CMPW $0x00010000, R10 BLO three_bytes_lz4s_s2 CMPW $0x01000000, R10 BLO four_bytes_lz4s_s2 MOVD $0xfc, R16 MOVB R16, (R0) MOVW R10, 1(R0) ADD $0x05, R0, R0 JMP memmove_long_lz4s_s2 four_bytes_lz4s_s2: MOVWU R10, R11 LSRW $0x10, R11, R11 MOVD $0xf8, R16 MOVB R16, (R0) MOVH R10, 1(R0) MOVB R11, 3(R0) ADD $0x04, R0, R0 JMP memmove_long_lz4s_s2 three_bytes_lz4s_s2: MOVD $0xf4, R16 MOVB R16, (R0) MOVH R10, 1(R0) ADD $0x03, R0, R0 JMP memmove_long_lz4s_s2 two_bytes_lz4s_s2: MOVD $0xf0, R16 MOVB R16, (R0) MOVB R10, 1(R0) ADD $0x02, R0, R0 CMPW $0x40, R10 BLO memmove_lz4s_s2 JMP memmove_long_lz4s_s2 one_byte_lz4s_s2: LSLW $0x02, R10, R16 BFI $0, R16, $8, R10 MOVB R10, (R0) ADD $0x01, R0, R0 memmove_lz4s_s2: ADD R8, R0, R10 // genMemMoveShort CMP $0x08, R8 BLS emit_lit_memmove_lz4s_s2_memmove_move_8 CMP $0x10, R8 BLS emit_lit_memmove_lz4s_s2_memmove_move_8through16 CMP $0x20, R8 BLS emit_lit_memmove_lz4s_s2_memmove_move_17through32 JMP emit_lit_memmove_lz4s_s2_memmove_move_33through64 emit_lit_memmove_lz4s_s2_memmove_move_8: MOVD (R2), R11 MOVD R11, (R0) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_8through16: MOVD (R2), R11 ADD R8, R2, R15 MOVD -8(R15), R2 MOVD R11, (R0) ADD R8, R0, R15 MOVD R2, -8(R15) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_17through32: FMOVQ (R2), F0 ADD R8, R2, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R0) ADD R8, R0, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_lz4s_s2 emit_lit_memmove_lz4s_s2_memmove_move_33through64: FMOVQ (R2), F0 FMOVQ 16(R2), F1 ADD R8, R2, R15 FMOVQ -32(R15), F2 ADD R8, R2, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R0) FMOVQ F1, 16(R0) ADD R8, R0, R15 FMOVQ F2, -32(R15) ADD R8, R0, R15 FMOVQ F3, -16(R15) memmove_end_copy_lz4s_s2: MOVD R10, R0 JMP lz4s_s2_lits_emit_done memmove_long_lz4s_s2: ADD R8, R0, R10 // genMemMoveLong MOVD R2, R11 MOVD R0, R12 MOVD R8, R13 emit_lit_memmove_long_lz4s_s2large_big_loop_back: FMOVQ (R11), F0 FMOVQ 16(R11), F1 FMOVQ F0, (R12) FMOVQ F1, 16(R12) ADD $0x20, R11, R11 ADD $0x20, R12, R12 SUB $0x20, R13, R13 CMP $0x20, R13 BHS emit_lit_memmove_long_lz4s_s2large_big_loop_back ADD R8, R2, R15 FMOVQ -32(R15), F0 ADD R8, R2, R15 FMOVQ -16(R15), F1 ADD R8, R0, R15 FMOVQ F0, -32(R15) ADD R8, R0, R15 FMOVQ F1, -16(R15) MOVD R10, R0 lz4s_s2_lits_emit_done: MOVD R7, R2 lz4s_s2_lits_done: CMP R3, R2 BNE lz4s_s2_match CMP $0x03, R9 BEQ lz4s_s2_done JMP lz4s_s2_corrupt lz4s_s2_match: CMP $0x03, R9 BEQ lz4s_s2_loop ADD $2, R2, R7 CMP R3, R7 BHS lz4s_s2_corrupt MOVHU (R2), R8 MOVD R7, R2 TST R8, R8 BEQ lz4s_s2_corrupt CMP R5, R8 BHI lz4s_s2_corrupt CMP $0x12, R9 BNE lz4s_s2_ml_done lz4s_s2_ml_loop: MOVBU (R2), R7 ADD $1, R2, R2 ADD R7, R9, R9 CMP R3, R2 BHS lz4s_s2_corrupt CMP $0xff, R7 BEQ lz4s_s2_ml_loop lz4s_s2_ml_done: ADD R9, R5, R5 CMP R6, R8 BNE lz4s_s2_docopy // emitRepeat emit_repeat_again_lz4_s2: MOVWU R9, R7 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R7 BLS repeat_two_lz4_s2 CMPW $0x0c, R7 BHS cant_repeat_two_offset_lz4_s2 CMPW $0x00000800, R8 BLO repeat_two_offset_lz4_s2 cant_repeat_two_offset_lz4_s2: CMPW $0x00000104, R9 BLO repeat_three_lz4_s2 CMPW $0x00010100, R9 BLO repeat_four_lz4_s2 CMPW $0x0100ffff, R9 BLO repeat_five_lz4_s2 SUB $16842747, R9, R9 MOVWU R9, R9 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 JMP emit_repeat_again_lz4_s2 repeat_five_lz4_s2: SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R8 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R9, 2(R0) ASRW $0x10, R8, R8 MOVB R8, 4(R0) ADD $0x05, R0, R0 JMP lz4s_s2_loop repeat_four_lz4_s2: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R9, 2(R0) ADD $0x04, R0, R0 JMP lz4s_s2_loop repeat_three_lz4_s2: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R9, 2(R0) ADD $0x03, R0, R0 JMP lz4s_s2_loop repeat_two_lz4_s2: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R0) ADD $0x02, R0, R0 JMP lz4s_s2_loop repeat_two_offset_lz4_s2: MOVD $0, R7 ADD R9<<2, R7, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R8, 1(R0) ASRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R9, R9 MOVB R9, (R0) ADD $0x02, R0, R0 JMP lz4s_s2_loop lz4s_s2_docopy: MOVD R8, R6 // emitCopy CMPW $0x40, R9 BLS two_byte_offset_short_lz4_s2 CMPW $0x00000800, R8 BHS long_offset_short_lz4_s2 MOVD $0x00000001, R7 ADD $16, R7, R7 MOVWU R7, R7 MOVB R8, 1(R0) MOVWU R8, R10 LSRW $0x08, R10, R10 LSLW $0x05, R10, R10 ORRW R10, R7, R7 MOVB R7, (R0) ADD $0x02, R0, R0 SUBW $0x08, R9, R9 // emitRepeat SUB $4, R9, R9 MOVWU R9, R9 JMP cant_repeat_two_offset_lz4_s2_emit_copy_short_2b emit_repeat_again_lz4_s2_emit_copy_short_2b: MOVWU R9, R7 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R7 BLS repeat_two_lz4_s2_emit_copy_short_2b CMPW $0x0c, R7 BHS cant_repeat_two_offset_lz4_s2_emit_copy_short_2b CMPW $0x00000800, R8 BLO repeat_two_offset_lz4_s2_emit_copy_short_2b cant_repeat_two_offset_lz4_s2_emit_copy_short_2b: CMPW $0x00000104, R9 BLO repeat_three_lz4_s2_emit_copy_short_2b CMPW $0x00010100, R9 BLO repeat_four_lz4_s2_emit_copy_short_2b CMPW $0x0100ffff, R9 BLO repeat_five_lz4_s2_emit_copy_short_2b SUB $16842747, R9, R9 MOVWU R9, R9 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 JMP emit_repeat_again_lz4_s2_emit_copy_short_2b repeat_five_lz4_s2_emit_copy_short_2b: SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R8 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R9, 2(R0) ASRW $0x10, R8, R8 MOVB R8, 4(R0) ADD $0x05, R0, R0 JMP lz4s_s2_loop repeat_four_lz4_s2_emit_copy_short_2b: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R9, 2(R0) ADD $0x04, R0, R0 JMP lz4s_s2_loop repeat_three_lz4_s2_emit_copy_short_2b: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R9, 2(R0) ADD $0x03, R0, R0 JMP lz4s_s2_loop repeat_two_lz4_s2_emit_copy_short_2b: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R0) ADD $0x02, R0, R0 JMP lz4s_s2_loop repeat_two_offset_lz4_s2_emit_copy_short_2b: MOVD $0, R7 ADD R9<<2, R7, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R8, 1(R0) ASRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R9, R9 MOVB R9, (R0) ADD $0x02, R0, R0 JMP lz4s_s2_loop long_offset_short_lz4_s2: MOVD $0xee, R16 MOVB R16, (R0) MOVH R8, 1(R0) SUB $60, R9, R9 MOVWU R9, R9 ADD $0x03, R0, R0 // emitRepeat emit_repeat_again_lz4_s2_emit_copy_short: MOVWU R9, R7 SUB $4, R9, R9 MOVWU R9, R9 CMPW $0x08, R7 BLS repeat_two_lz4_s2_emit_copy_short CMPW $0x0c, R7 BHS cant_repeat_two_offset_lz4_s2_emit_copy_short CMPW $0x00000800, R8 BLO repeat_two_offset_lz4_s2_emit_copy_short cant_repeat_two_offset_lz4_s2_emit_copy_short: CMPW $0x00000104, R9 BLO repeat_three_lz4_s2_emit_copy_short CMPW $0x00010100, R9 BLO repeat_four_lz4_s2_emit_copy_short CMPW $0x0100ffff, R9 BLO repeat_five_lz4_s2_emit_copy_short SUB $16842747, R9, R9 MOVWU R9, R9 MOVD $0xfffb001d, R16 MOVW R16, (R0) MOVD $0xff, R16 MOVB R16, 4(R0) ADD $0x05, R0, R0 JMP emit_repeat_again_lz4_s2_emit_copy_short repeat_five_lz4_s2_emit_copy_short: SUB $65536, R9, R9 MOVWU R9, R9 MOVWU R9, R8 MOVD $0x001d, R16 MOVH R16, (R0) MOVH R9, 2(R0) ASRW $0x10, R8, R8 MOVB R8, 4(R0) ADD $0x05, R0, R0 JMP lz4s_s2_loop repeat_four_lz4_s2_emit_copy_short: SUB $256, R9, R9 MOVWU R9, R9 MOVD $0x0019, R16 MOVH R16, (R0) MOVH R9, 2(R0) ADD $0x04, R0, R0 JMP lz4s_s2_loop repeat_three_lz4_s2_emit_copy_short: SUB $4, R9, R9 MOVWU R9, R9 MOVD $0x0015, R16 MOVH R16, (R0) MOVB R9, 2(R0) ADD $0x03, R0, R0 JMP lz4s_s2_loop repeat_two_lz4_s2_emit_copy_short: LSLW $0x02, R9, R9 ORRW $0x01, R9, R9 MOVH R9, (R0) ADD $0x02, R0, R0 JMP lz4s_s2_loop repeat_two_offset_lz4_s2_emit_copy_short: MOVD $0, R7 ADD R9<<2, R7, R9 ADD $1, R9, R9 MOVWU R9, R9 MOVB R8, 1(R0) ASRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R9, R9 MOVB R9, (R0) ADD $0x02, R0, R0 JMP lz4s_s2_loop two_byte_offset_short_lz4_s2: MOVWU R9, R7 LSLW $0x02, R7, R7 CMPW $0x0c, R9 BHS emit_copy_three_lz4_s2 CMPW $0x00000800, R8 BHS emit_copy_three_lz4_s2 SUB $15, R7, R7 MOVWU R7, R7 MOVB R8, 1(R0) LSRW $0x08, R8, R8 LSLW $0x05, R8, R8 ORRW R8, R7, R7 MOVB R7, (R0) ADD $0x02, R0, R0 JMP lz4s_s2_loop emit_copy_three_lz4_s2: SUB $2, R7, R7 MOVWU R7, R7 MOVB R7, (R0) MOVH R8, 1(R0) ADD $0x03, R0, R0 JMP lz4s_s2_loop lz4s_s2_done: MOVD dst_base+0(FP), R1 SUB R1, R0, R0 MOVD R5, uncompressed+48(FP) MOVD R0, dstUsed+56(FP) RET lz4s_s2_corrupt: MOVD $0, R0 SUB $1, R0, R5 MOVD R5, uncompressed+48(FP) RET lz4s_s2_dstfull: MOVD $0, R0 SUB $2, R0, R5 MOVD R5, uncompressed+48(FP) RET // func cvtLZ4BlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 TEXT ·cvtLZ4BlockSnappyAsm(SB), NOSPLIT, $0-64 MOVD $0, R5 MOVD dst_base+0(FP), R0 MOVD dst_len+8(FP), R1 MOVD src_base+24(FP), R2 MOVD src_len+32(FP), R3 ADD R3, R2, R3 ADD R1, R0, R1 SUB $8, R1, R1 lz4_snappy_loop: CMP R3, R2 BHS lz4_snappy_corrupt CMP R1, R0 BHS lz4_snappy_dstfull MOVBU (R2), R6 MOVD R6, R7 MOVD R6, R8 LSR $0x04, R7, R7 AND $0x0f, R8, R8 CMP $0xf0, R6 BLO lz4_snappy_ll_end lz4_snappy_ll_loop: ADD $1, R2, R2 CMP R3, R2 BHS lz4_snappy_corrupt MOVBU (R2), R6 ADD R6, R7, R7 CMP $0xff, R6 BEQ lz4_snappy_ll_loop lz4_snappy_ll_end: ADD R7, R2, R6 ADD $0x04, R8, R8 CMP R3, R6 BHS lz4_snappy_corrupt ADD $1, R2, R2 ADD $1, R6, R6 TST R7, R7 BEQ lz4_snappy_lits_done ADD R7, R0, R9 CMP R1, R9 BHS lz4_snappy_dstfull ADD R7, R5, R5 SUB $1, R7, R9 MOVWU R9, R9 CMPW $0x3c, R9 BLO one_byte_lz4_snappy CMPW $0x00000100, R9 BLO two_bytes_lz4_snappy CMPW $0x00010000, R9 BLO three_bytes_lz4_snappy CMPW $0x01000000, R9 BLO four_bytes_lz4_snappy MOVD $0xfc, R16 MOVB R16, (R0) MOVW R9, 1(R0) ADD $0x05, R0, R0 JMP memmove_long_lz4_snappy four_bytes_lz4_snappy: MOVWU R9, R10 LSRW $0x10, R10, R10 MOVD $0xf8, R16 MOVB R16, (R0) MOVH R9, 1(R0) MOVB R10, 3(R0) ADD $0x04, R0, R0 JMP memmove_long_lz4_snappy three_bytes_lz4_snappy: MOVD $0xf4, R16 MOVB R16, (R0) MOVH R9, 1(R0) ADD $0x03, R0, R0 JMP memmove_long_lz4_snappy two_bytes_lz4_snappy: MOVD $0xf0, R16 MOVB R16, (R0) MOVB R9, 1(R0) ADD $0x02, R0, R0 CMPW $0x40, R9 BLO memmove_lz4_snappy JMP memmove_long_lz4_snappy one_byte_lz4_snappy: LSLW $0x02, R9, R16 BFI $0, R16, $8, R9 MOVB R9, (R0) ADD $0x01, R0, R0 memmove_lz4_snappy: ADD R7, R0, R9 // genMemMoveShort CMP $0x08, R7 BLS emit_lit_memmove_lz4_snappy_memmove_move_8 CMP $0x10, R7 BLS emit_lit_memmove_lz4_snappy_memmove_move_8through16 CMP $0x20, R7 BLS emit_lit_memmove_lz4_snappy_memmove_move_17through32 JMP emit_lit_memmove_lz4_snappy_memmove_move_33through64 emit_lit_memmove_lz4_snappy_memmove_move_8: MOVD (R2), R10 MOVD R10, (R0) JMP memmove_end_copy_lz4_snappy emit_lit_memmove_lz4_snappy_memmove_move_8through16: MOVD (R2), R10 ADD R7, R2, R15 MOVD -8(R15), R2 MOVD R10, (R0) ADD R7, R0, R15 MOVD R2, -8(R15) JMP memmove_end_copy_lz4_snappy emit_lit_memmove_lz4_snappy_memmove_move_17through32: FMOVQ (R2), F0 ADD R7, R2, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R0) ADD R7, R0, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_lz4_snappy emit_lit_memmove_lz4_snappy_memmove_move_33through64: FMOVQ (R2), F0 FMOVQ 16(R2), F1 ADD R7, R2, R15 FMOVQ -32(R15), F2 ADD R7, R2, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R0) FMOVQ F1, 16(R0) ADD R7, R0, R15 FMOVQ F2, -32(R15) ADD R7, R0, R15 FMOVQ F3, -16(R15) memmove_end_copy_lz4_snappy: MOVD R9, R0 JMP lz4_snappy_lits_emit_done memmove_long_lz4_snappy: ADD R7, R0, R9 // genMemMoveLong MOVD R2, R10 MOVD R0, R11 MOVD R7, R12 emit_lit_memmove_long_lz4_snappylarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R11) FMOVQ F1, 16(R11) ADD $0x20, R10, R10 ADD $0x20, R11, R11 SUB $0x20, R12, R12 CMP $0x20, R12 BHS emit_lit_memmove_long_lz4_snappylarge_big_loop_back ADD R7, R2, R15 FMOVQ -32(R15), F0 ADD R7, R2, R15 FMOVQ -16(R15), F1 ADD R7, R0, R15 FMOVQ F0, -32(R15) ADD R7, R0, R15 FMOVQ F1, -16(R15) MOVD R9, R0 lz4_snappy_lits_emit_done: MOVD R6, R2 lz4_snappy_lits_done: CMP R3, R2 BNE lz4_snappy_match CMP $0x04, R8 BEQ lz4_snappy_done JMP lz4_snappy_corrupt lz4_snappy_match: ADD $2, R2, R6 CMP R3, R6 BHS lz4_snappy_corrupt MOVHU (R2), R7 MOVD R6, R2 TST R7, R7 BEQ lz4_snappy_corrupt CMP R5, R7 BHI lz4_snappy_corrupt CMP $0x13, R8 BNE lz4_snappy_ml_done lz4_snappy_ml_loop: MOVBU (R2), R6 ADD $1, R2, R2 ADD R6, R8, R8 CMP R3, R2 BHS lz4_snappy_corrupt CMP $0xff, R6 BEQ lz4_snappy_ml_loop lz4_snappy_ml_done: ADD R8, R5, R5 // emitCopy two_byte_offset_lz4_s2: CMPW $0x40, R8 BLS two_byte_offset_short_lz4_s2 MOVD $0xee, R16 MOVB R16, (R0) MOVH R7, 1(R0) SUB $60, R8, R8 MOVWU R8, R8 ADD $0x03, R0, R0 CMP R1, R0 BHS lz4_snappy_loop JMP two_byte_offset_lz4_s2 two_byte_offset_short_lz4_s2: MOVWU R8, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R8 BHS emit_copy_three_lz4_s2 CMPW $0x00000800, R7 BHS emit_copy_three_lz4_s2 SUB $15, R6, R6 MOVWU R6, R6 MOVB R7, 1(R0) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R6, R6 MOVB R6, (R0) ADD $0x02, R0, R0 JMP lz4_snappy_loop emit_copy_three_lz4_s2: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R0) MOVH R7, 1(R0) ADD $0x03, R0, R0 JMP lz4_snappy_loop lz4_snappy_done: MOVD dst_base+0(FP), R1 SUB R1, R0, R0 MOVD R5, uncompressed+48(FP) MOVD R0, dstUsed+56(FP) RET lz4_snappy_corrupt: MOVD $0, R0 SUB $1, R0, R5 MOVD R5, uncompressed+48(FP) RET lz4_snappy_dstfull: MOVD $0, R0 SUB $2, R0, R5 MOVD R5, uncompressed+48(FP) RET // func cvtLZ4sBlockSnappyAsm(dst []byte, src []byte) (uncompressed int, dstUsed int) // Requires: SSE2 TEXT ·cvtLZ4sBlockSnappyAsm(SB), NOSPLIT, $0-64 MOVD $0, R5 MOVD dst_base+0(FP), R0 MOVD dst_len+8(FP), R1 MOVD src_base+24(FP), R2 MOVD src_len+32(FP), R3 ADD R3, R2, R3 ADD R1, R0, R1 SUB $8, R1, R1 lz4s_snappy_loop: CMP R3, R2 BHS lz4s_snappy_corrupt CMP R1, R0 BHS lz4s_snappy_dstfull MOVBU (R2), R6 MOVD R6, R7 MOVD R6, R8 LSR $0x04, R7, R7 AND $0x0f, R8, R8 CMP $0xf0, R6 BLO lz4s_snappy_ll_end lz4s_snappy_ll_loop: ADD $1, R2, R2 CMP R3, R2 BHS lz4s_snappy_corrupt MOVBU (R2), R6 ADD R6, R7, R7 CMP $0xff, R6 BEQ lz4s_snappy_ll_loop lz4s_snappy_ll_end: ADD R7, R2, R6 ADD $0x03, R8, R8 CMP R3, R6 BHS lz4s_snappy_corrupt ADD $1, R2, R2 ADD $1, R6, R6 TST R7, R7 BEQ lz4s_snappy_lits_done ADD R7, R0, R9 CMP R1, R9 BHS lz4s_snappy_dstfull ADD R7, R5, R5 SUB $1, R7, R9 MOVWU R9, R9 CMPW $0x3c, R9 BLO one_byte_lz4s_snappy CMPW $0x00000100, R9 BLO two_bytes_lz4s_snappy CMPW $0x00010000, R9 BLO three_bytes_lz4s_snappy CMPW $0x01000000, R9 BLO four_bytes_lz4s_snappy MOVD $0xfc, R16 MOVB R16, (R0) MOVW R9, 1(R0) ADD $0x05, R0, R0 JMP memmove_long_lz4s_snappy four_bytes_lz4s_snappy: MOVWU R9, R10 LSRW $0x10, R10, R10 MOVD $0xf8, R16 MOVB R16, (R0) MOVH R9, 1(R0) MOVB R10, 3(R0) ADD $0x04, R0, R0 JMP memmove_long_lz4s_snappy three_bytes_lz4s_snappy: MOVD $0xf4, R16 MOVB R16, (R0) MOVH R9, 1(R0) ADD $0x03, R0, R0 JMP memmove_long_lz4s_snappy two_bytes_lz4s_snappy: MOVD $0xf0, R16 MOVB R16, (R0) MOVB R9, 1(R0) ADD $0x02, R0, R0 CMPW $0x40, R9 BLO memmove_lz4s_snappy JMP memmove_long_lz4s_snappy one_byte_lz4s_snappy: LSLW $0x02, R9, R16 BFI $0, R16, $8, R9 MOVB R9, (R0) ADD $0x01, R0, R0 memmove_lz4s_snappy: ADD R7, R0, R9 // genMemMoveShort CMP $0x08, R7 BLS emit_lit_memmove_lz4s_snappy_memmove_move_8 CMP $0x10, R7 BLS emit_lit_memmove_lz4s_snappy_memmove_move_8through16 CMP $0x20, R7 BLS emit_lit_memmove_lz4s_snappy_memmove_move_17through32 JMP emit_lit_memmove_lz4s_snappy_memmove_move_33through64 emit_lit_memmove_lz4s_snappy_memmove_move_8: MOVD (R2), R10 MOVD R10, (R0) JMP memmove_end_copy_lz4s_snappy emit_lit_memmove_lz4s_snappy_memmove_move_8through16: MOVD (R2), R10 ADD R7, R2, R15 MOVD -8(R15), R2 MOVD R10, (R0) ADD R7, R0, R15 MOVD R2, -8(R15) JMP memmove_end_copy_lz4s_snappy emit_lit_memmove_lz4s_snappy_memmove_move_17through32: FMOVQ (R2), F0 ADD R7, R2, R15 FMOVQ -16(R15), F1 FMOVQ F0, (R0) ADD R7, R0, R15 FMOVQ F1, -16(R15) JMP memmove_end_copy_lz4s_snappy emit_lit_memmove_lz4s_snappy_memmove_move_33through64: FMOVQ (R2), F0 FMOVQ 16(R2), F1 ADD R7, R2, R15 FMOVQ -32(R15), F2 ADD R7, R2, R15 FMOVQ -16(R15), F3 FMOVQ F0, (R0) FMOVQ F1, 16(R0) ADD R7, R0, R15 FMOVQ F2, -32(R15) ADD R7, R0, R15 FMOVQ F3, -16(R15) memmove_end_copy_lz4s_snappy: MOVD R9, R0 JMP lz4s_snappy_lits_emit_done memmove_long_lz4s_snappy: ADD R7, R0, R9 // genMemMoveLong MOVD R2, R10 MOVD R0, R11 MOVD R7, R12 emit_lit_memmove_long_lz4s_snappylarge_big_loop_back: FMOVQ (R10), F0 FMOVQ 16(R10), F1 FMOVQ F0, (R11) FMOVQ F1, 16(R11) ADD $0x20, R10, R10 ADD $0x20, R11, R11 SUB $0x20, R12, R12 CMP $0x20, R12 BHS emit_lit_memmove_long_lz4s_snappylarge_big_loop_back ADD R7, R2, R15 FMOVQ -32(R15), F0 ADD R7, R2, R15 FMOVQ -16(R15), F1 ADD R7, R0, R15 FMOVQ F0, -32(R15) ADD R7, R0, R15 FMOVQ F1, -16(R15) MOVD R9, R0 lz4s_snappy_lits_emit_done: MOVD R6, R2 lz4s_snappy_lits_done: CMP R3, R2 BNE lz4s_snappy_match CMP $0x03, R8 BEQ lz4s_snappy_done JMP lz4s_snappy_corrupt lz4s_snappy_match: CMP $0x03, R8 BEQ lz4s_snappy_loop ADD $2, R2, R6 CMP R3, R6 BHS lz4s_snappy_corrupt MOVHU (R2), R7 MOVD R6, R2 TST R7, R7 BEQ lz4s_snappy_corrupt CMP R5, R7 BHI lz4s_snappy_corrupt CMP $0x12, R8 BNE lz4s_snappy_ml_done lz4s_snappy_ml_loop: MOVBU (R2), R6 ADD $1, R2, R2 ADD R6, R8, R8 CMP R3, R2 BHS lz4s_snappy_corrupt CMP $0xff, R6 BEQ lz4s_snappy_ml_loop lz4s_snappy_ml_done: ADD R8, R5, R5 // emitCopy two_byte_offset_lz4_s2: CMPW $0x40, R8 BLS two_byte_offset_short_lz4_s2 MOVD $0xee, R16 MOVB R16, (R0) MOVH R7, 1(R0) SUB $60, R8, R8 MOVWU R8, R8 ADD $0x03, R0, R0 CMP R1, R0 BHS lz4s_snappy_loop JMP two_byte_offset_lz4_s2 two_byte_offset_short_lz4_s2: MOVWU R8, R6 LSLW $0x02, R6, R6 CMPW $0x0c, R8 BHS emit_copy_three_lz4_s2 CMPW $0x00000800, R7 BHS emit_copy_three_lz4_s2 SUB $15, R6, R6 MOVWU R6, R6 MOVB R7, 1(R0) LSRW $0x08, R7, R7 LSLW $0x05, R7, R7 ORRW R7, R6, R6 MOVB R6, (R0) ADD $0x02, R0, R0 JMP lz4s_snappy_loop emit_copy_three_lz4_s2: SUB $2, R6, R6 MOVWU R6, R6 MOVB R6, (R0) MOVH R7, 1(R0) ADD $0x03, R0, R0 JMP lz4s_snappy_loop lz4s_snappy_done: MOVD dst_base+0(FP), R1 SUB R1, R0, R0 MOVD R5, uncompressed+48(FP) MOVD R0, dstUsed+56(FP) RET lz4s_snappy_corrupt: MOVD $0, R0 SUB $1, R0, R5 MOVD R5, uncompressed+48(FP) RET lz4s_snappy_dstfull: MOVD $0, R0 SUB $2, R0, R5 MOVD R5, uncompressed+48(FP) RET