srctree

const compress = if (builtin.cpu.arch == .x86_64 and

(builtin.zig_backend != .stage2_x86_64 or std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)))

const SalsaImpl = if (builtin.cpu.arch == .x86_64 and

(builtin.zig_backend != .stage2_x86_64 or std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)))

SalsaVecImpl

else

SalsaNonVecImpl;

try testing.expect(hasUniqueRepresentation(@Vector(std.simd.suggestVectorLength(u8) orelse 1, u8)));

try testing.expect(@sizeOf(@Vector(3, u8)) == 3 or !hasUniqueRepresentation(@Vector(3, u8)));

if (std.simd.suggestVectorLength(u8)) |chunk_len| {

const Chunk = @Vector(chunk_len, u8);

// Fast path. Check for and skip ASCII characters at the start of the input.

while (remaining.len >= chunk_len) {

const chunk: Chunk = remaining[0..chunk_len].*;

const mask: Chunk = @splat(0x80);

if (@reduce(.Or, chunk & mask == mask)) {

// found a non ASCII byte

break;

}

remaining = remaining[chunk_len..];

if (builtin.zig_backend != .stage2_x86_64 or

comptime (std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3) and

!std.Target.x86.featureSetHasAny(builtin.cpu.features, .{ .prefer_256_bit, .avx })))

vectorized: {

const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;

if (builtin.zig_backend != .stage2_x86_64 or

comptime (std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3) and

!std.Target.x86.featureSetHasAny(builtin.cpu.features, .{ .prefer_256_bit, .avx })))

vectorized: {

const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;

if ((builtin.zig_backend != .stage2_x86_64 or

comptime (std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3) and

!std.Target.x86.featureSetHasAny(builtin.cpu.features, .{ .prefer_256_bit, .avx }))) and

comptime !builtin.cpu.arch.isMIPS())

vectorized: {

const chunk_len = @divExact(std.simd.suggestVectorLength(u8) orelse break :vectorized, 2);

if ((builtin.zig_backend != .stage2_x86_64 or

comptime (std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3) and

!std.Target.x86.featureSetHasAny(builtin.cpu.features, .{ .prefer_256_bit, .avx }))) and

comptime !builtin.cpu.arch.isMIPS())

vectorized: {

const chunk_len = @divExact(std.simd.suggestVectorLength(u8) orelse break :vectorized, 2);

/// negative values for `this_index` indicate "don't care".

const positive_index = std.math.cast(usize, this_index) orelse return undefined;

_ = shuffleVectorIndex(-1, vector_len);

@"64" = 6,

if (opts.update_tracking)

{} else std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);

1...8 => switch (ty.vectorLen(mod)) {

1...16 => return .{ .move = if (self.hasFeature(.avx))

9...16 => switch (ty.vectorLen(mod)) {

1...8 => return .{ .move = if (self.hasFeature(.avx))

17...32 => switch (ty.vectorLen(mod)) {

1...4 => return .{ .move = if (self.hasFeature(.avx))

33...64 => switch (ty.vectorLen(mod)) {

1...2 => return .{ .move = if (self.hasFeature(.avx))

65...128 => switch (ty.vectorLen(mod)) {

129...256 => switch (ty.vectorLen(mod)) {

1...2 => return .{ .move = if (self.hasFeature(.avx))

1...8 => return .{ .move = if (self.hasFeature(.avx))

1...4 => return .{ .move = if (self.hasFeature(.avx))

1...2 => return .{ .move = if (self.hasFeature(.avx))

const abi_size: u32 = @intCast(ty.abiSize(mod));

const mod = self.bin_file.comp.module.?;

const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data;

 

const dst_ty = self.typeOfIndex(inst);

const elem_ty = dst_ty.childType(mod);

const elem_abi_size: u32 = @intCast(elem_ty.abiSize(mod));

const dst_abi_size: u32 = @intCast(dst_ty.abiSize(mod));

const lhs_ty = self.typeOf(extra.a);

const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(mod));

const rhs_ty = self.typeOf(extra.b);

const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(mod));

const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size);

 

const ExpectedContents = [32]?i32;

var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) =

std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa);

const allocator = stack.get();

 

const mask_elems = try allocator.alloc(?i32, extra.mask_len);

defer allocator.free(mask_elems);

for (mask_elems, 0..) |*mask_elem, elem_index| {

const mask_elem_val = Value

.fromInterned(extra.mask).elemValue(mod, elem_index) catch unreachable;

mask_elem.* = if (mask_elem_val.isUndef(mod))

null

else

@intCast(mask_elem_val.toSignedInt(mod));

}

 

const result = @as(?MCValue, result: {

for (mask_elems) |mask_elem| {

if (mask_elem) |_| break;

} else break :result try self.allocRegOrMem(inst, true);

 

for (mask_elems, 0..) |mask_elem, elem_index| {

if (mask_elem orelse continue != @as(i32, @intCast(elem_index))) break;

} else {

const lhs_mcv = try self.resolveInst(extra.a);

if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv;

const dst_mcv = try self.allocRegOrMem(inst, true);

try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{});

break :result dst_mcv;

}

 

for (mask_elems, 0..) |mask_elem, elem_index| {

if (mask_elem orelse continue != ~@as(i32, @intCast(elem_index))) break;

} else {

const rhs_mcv = try self.resolveInst(extra.b);

if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv;

const dst_mcv = try self.allocRegOrMem(inst, true);

try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{});

break :result dst_mcv;

}

 

const has_avx = self.hasFeature(.avx);

shufpd: {

if (elem_abi_size != 8) break :shufpd;

if (max_abi_size > @as(u32, if (has_avx) 32 else 16)) break :shufpd;

 

var control: u4 = 0b0_0_0_0;

var sources = [1]?u1{null} ** 2;

for (mask_elems, 0..) |maybe_mask_elem, elem_index| {

const mask_elem = maybe_mask_elem orelse continue;

const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem);

if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd;

 

const source = @intFromBool(mask_elem < 0);

if (sources[elem_index & 0b01]) |prev_source| {

if (source != prev_source) break :shufpd;

} else sources[elem_index & 0b01] = source;

 

control |= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index);

}

if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd;

 

const operands = [2]Air.Inst.Ref{ extra.a, extra.b };

const operand_tys = [2]Type{ lhs_ty, rhs_ty };

const lhs_mcv = try self.resolveInst(operands[sources[0].?]);

const rhs_mcv = try self.resolveInst(operands[sources[1].?]);

 

const dst_mcv: MCValue = if (lhs_mcv.isRegister() and

self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv))

lhs_mcv

else if (has_avx and lhs_mcv.isRegister())

.{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) }

else

try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv);

const dst_reg = dst_mcv.getReg().?;

const dst_alias = registerAlias(dst_reg, max_abi_size);

 

if (has_avx) if (rhs_mcv.isMemory()) try self.asmRegisterRegisterMemoryImmediate(

.{ .v_pd, .shuf },

dst_alias,

registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),

try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),

Immediate.u(control),

) else try self.asmRegisterRegisterRegisterImmediate(

.{ .v_pd, .shuf },

dst_alias,

registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size),

registerAlias(if (rhs_mcv.isRegister())

rhs_mcv.getReg().?

else

try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),

Immediate.u(control),

) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate(

.{ ._pd, .shuf },

dst_alias,

try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)),

Immediate.u(control),

) else try self.asmRegisterRegisterImmediate(

.{ ._pd, .shuf },

dst_alias,

registerAlias(if (rhs_mcv.isRegister())

rhs_mcv.getReg().?

else

try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size),

Immediate.u(control),

);

break :result dst_mcv;

}

 

pshufb: {

if (max_abi_size > 16) break :pshufb;

if (!self.hasFeature(.ssse3)) break :pshufb;

 

const temp_regs =

try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse);

const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs);

defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);

 

const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size);

try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{});

 

const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size);

try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{});

 

var lhs_mask_elems: [16]InternPool.Index = undefined;

for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| {

const elem_index = byte_index / elem_abi_size;

lhs_mask_elem.* = try mod.intern(.{ .int = .{

.ty = .u8_type,

.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {

const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;

if (mask_elem < 0) break :elem 0b1_00_00000;

const mask_elem_index: u31 = @intCast(mask_elem);

const byte_off: u32 = @intCast(byte_index % elem_abi_size);

break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);

} },

} });

}

const lhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });

const lhs_mask_mcv = try self.genTypedValue(.{

.ty = lhs_mask_ty,

.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{

.ty = lhs_mask_ty.toIntern(),

.storage = .{ .elems = lhs_mask_elems[0..max_abi_size] },

} })),

});

const lhs_mask_mem: Memory = .{

.base = .{ .reg = try self.copyToTmpRegister(Type.usize, lhs_mask_mcv.address()) },

.mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },

};

if (has_avx) try self.asmRegisterRegisterMemory(

.{ .vp_b, .shuf },

lhs_temp_alias,

lhs_temp_alias,

lhs_mask_mem,

) else try self.asmRegisterMemory(

.{ .p_b, .shuf },

lhs_temp_alias,

lhs_mask_mem,

);

 

var rhs_mask_elems: [16]InternPool.Index = undefined;

for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| {

const elem_index = byte_index / elem_abi_size;

rhs_mask_elem.* = try mod.intern(.{ .int = .{

.ty = .u8_type,

.storage = .{ .u64 = if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: {

const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000;

if (mask_elem >= 0) break :elem 0b1_00_00000;

const mask_elem_index: u31 = @intCast(~mask_elem);

const byte_off: u32 = @intCast(byte_index % elem_abi_size);

break :elem @intCast(mask_elem_index * elem_abi_size + byte_off);

} },

} });

}

const rhs_mask_ty = try mod.vectorType(.{ .len = max_abi_size, .child = .u8_type });

const rhs_mask_mcv = try self.genTypedValue(.{

.ty = rhs_mask_ty,

.val = Value.fromInterned(try mod.intern(.{ .aggregate = .{

.ty = rhs_mask_ty.toIntern(),

.storage = .{ .elems = rhs_mask_elems[0..max_abi_size] },

} })),

});

const rhs_mask_mem: Memory = .{

.base = .{ .reg = try self.copyToTmpRegister(Type.usize, rhs_mask_mcv.address()) },

.mod = .{ .rm = .{ .size = Memory.Size.fromSize(@max(max_abi_size, 16)) } },

};

if (has_avx) try self.asmRegisterRegisterMemory(

.{ .vp_b, .shuf },

rhs_temp_alias,

rhs_temp_alias,

rhs_mask_mem,

) else try self.asmRegisterMemory(

.{ .p_b, .shuf },

rhs_temp_alias,

rhs_mask_mem,

);

 

if (has_avx) try self.asmRegisterRegisterRegister(

.{ switch (elem_ty.zigTypeTag(mod)) {

else => break :result null,

.Int => .vp_,

.Float => switch (elem_ty.floatBits(self.target.*)) {

32 => .v_ps,

64 => .v_pd,

16, 80, 128 => break :result null,

else => unreachable,

},

}, .@"or" },

lhs_temp_alias,

lhs_temp_alias,

rhs_temp_alias,

) else try self.asmRegisterRegister(

.{ switch (elem_ty.zigTypeTag(mod)) {

else => break :result null,

.Int => .p_,

.Float => switch (elem_ty.floatBits(self.target.*)) {

32 => ._ps,

64 => ._pd,

16, 80, 128 => break :result null,

else => unreachable,

},

}, .@"or" },

lhs_temp_alias,

rhs_temp_alias,

);

break :result .{ .register = temp_regs[0] };

}

 

if (max_abi_size <= 16) {

const lhs_mcv = try self.resolveInst(extra.a);

const lhs_reg = if (lhs_mcv.isRegister())

lhs_mcv.getReg().?

else

try self.copyToTmpRegister(lhs_ty, lhs_mcv);

const lhs_lock = self.register_manager.lockRegAssumeUnused(lhs_reg);

defer self.register_manager.unlockReg(lhs_lock);

 

const rhs_mcv = try self.resolveInst(extra.b);

const rhs_reg = if (rhs_mcv.isRegister())

rhs_mcv.getReg().?

else

try self.copyToTmpRegister(rhs_ty, rhs_mcv);

const rhs_lock = self.register_manager.lockReg(rhs_reg);

defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);

 

//const dst_mcv = try self.register_manager.allocReg(inst, abi.RegisterClass.sse);

switch (elem_ty.zigTypeTag(mod)) {

.Float => switch (elem_ty.floatBits(self.target.*)) {

16, 32 => {},

64 => unreachable, // fully handled by shufpd

80, 128 => unreachable, // all possible masks already handled

else => unreachable,

},

else => {},

}

}

 

break :result null;

}) orelse return self.fail("TODO implement airShuffle from {} and {} to {}", .{

lhs_ty.fmt(mod), rhs_ty.fmt(mod), dst_ty.fmt(mod),

});

return self.finishAir(inst, result, .{ extra.a, extra.b, .none });

if (result_ty.isVector(mod) and elem_ty.toIntern() == .bool_type) {

.float_combine => try mod.arrayType(.{ .len = 2, .child = .f32_type }),

pabsb, pabsd, pabsw, palignr, pshufb,

vpshufb, vpshufd, vpshufhw, vpshuflw,

.{ .pshufb, .rm, &.{ .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .none, .ssse3 },

 

.{ .vpshufb, .rvm, &.{ .xmm, .xmm, .xmm_m128 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_128_wig, .avx },

 

.{ .vpshufb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x00 }, 0, .vex_256_wig, .avx2 },

if (vector_type.child == .bool_type) {

} else {

switch (aggregate.storage) {

.bytes => |bytes| try code.appendSlice(bytes),

.elems, .repeated_elem => {

var index: u64 = 0;

while (index < vector_type.len) : (index += 1) {

switch (try generateSymbol(bin_file, src_loc, .{

.ty = Type.fromInterned(vector_type.child),

.val = Value.fromInterned(switch (aggregate.storage) {

.bytes => unreachable,

.elems => |elems| elems[

math.cast(usize, index) orelse return error.Overflow

],

.repeated_elem => |elem| elem,

}),

}, code, debug_output, reloc_info)) {

.ok => {},

.fail => |em| return .{ .fail = em },

}

},

}

const padding = abi_size -

(math.cast(usize, Type.fromInterned(vector_type.child).abiSize(mod) * vector_type.len) orelse

return error.Overflow);

if (padding > 0) try code.appendNTimes(0, padding);

}

if (vector_type.len == 0) return .{ .scalar = .@"1" };

switch (mod.comp.getZigBackend()) {

else => {

const elem_bits: u32 = @intCast(try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema));

if (elem_bits == 0) return .{ .scalar = .@"1" };

const bytes = ((elem_bits * vector_type.len) + 7) / 8;

const alignment = std.math.ceilPowerOfTwoAssert(u32, bytes);

return .{ .scalar = Alignment.fromByteUnits(alignment) };

},

.stage2_x86_64 => {

if (vector_type.child == .bool_type) return .{ .scalar = intAbiAlignment(@intCast(vector_type.len), target) };

const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);

if (elem_bytes == 0) return .{ .scalar = .@"1" };

const bytes = elem_bytes * vector_type.len;

if (bytes > 32 and std.Target.x86.featureSetHas(target.cpu.features, .avx512f)) return .{ .scalar = .@"64" };

if (bytes > 16 and std.Target.x86.featureSetHas(

target.cpu.features,

if (Type.fromInterned(vector_type.child).isRuntimeFloat()) .avx else .avx2,

)) return .{ .scalar = .@"32" };

return .{ .scalar = .@"16" };

},

}

const total_bytes = switch (mod.comp.getZigBackend()) {

else => total_bytes: {

const elem_bits = try Type.fromInterned(vector_type.child).bitSizeAdvanced(mod, opt_sema);

const total_bits = elem_bits * vector_type.len;

break :total_bytes (total_bits + 7) / 8;

},

.stage2_x86_64 => total_bytes: {

if (vector_type.child == .bool_type) break :total_bytes std.math.divCeil(u32, vector_type.len, 8) catch unreachable;

const elem_bytes: u32 = @intCast((try Type.fromInterned(vector_type.child).abiSizeAdvanced(mod, strat)).scalar);

break :total_bytes elem_bytes * vector_type.len;

},

};

if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;

if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;

if (builtin.zig_backend == .stage2_x86_64 and

!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;

!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sse4_1)) return error.SkipZigTest;

if (builtin.zig_backend == .stage2_x86_64 and

!comptime std.Target.x86.featureSetHas(builtin.cpu.features, .ssse3)) return error.SkipZigTest;

var v4_a = __v4hi{ 1, 2, 3, 4 };

var v4_b = __v4hi{ 5, 6, 7, 8 };

std.zig.c_translation.shuffleVectorIndex(2, @typeInfo(@TypeOf(v4_a)).Vector.len),

std.zig.c_translation.shuffleVectorIndex(4, @typeInfo(@TypeOf(v4_a)).Vector.len),

std.zig.c_translation.shuffleVectorIndex(6, @typeInfo(@TypeOf(v4_a)).Vector.len),

try expect(shuffled[0] == 1);

try expect(shuffled[1] == 3);

try expect(shuffled[2] == 5);

try expect(shuffled[3] == 7);

try expect(@alignOf(@Vector(2, u8)) == switch (builtin.zig_backend) {

else => 2,

.stage2_x86_64 => 16,

});

try expect(@alignOf(@Vector(2, u1)) == switch (builtin.zig_backend) {

else => 1,

.stage2_x86_64 => 16,

});

try expect(@alignOf(@Vector(1, u1)) == switch (builtin.zig_backend) {

else => 1,

.stage2_x86_64 => 16,

});

try expect(@alignOf(@Vector(2, u16)) == switch (builtin.zig_backend) {

else => 4,

.stage2_x86_64 => 16,

});

if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;