Transactions: move loops inside

Rewrite loops in assembler (minor optimization)

Utilities/JIT.cpp

@@ -7,7 +7,7 @@ asmjit::JitRuntime& asmjit::get_global_runtime()
 	return g_rt;
 }
 
-void asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label fallback)
+asmjit::Label asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label fallback)
 {
 	Label fall = c.newLabel();
 	Label begin = c.newLabel();
@@ -18,6 +18,7 @@ void asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label fall
 	c.align(kAlignCode, 16);
 	c.bind(begin);
 	c.xbegin(fall);
+	return begin;
 }
 
 void asmjit::build_transaction_abort(asmjit::X86Assembler& c, unsigned char code)

Utilities/JIT.h

@@ -12,8 +12,8 @@ namespace asmjit
 	// Should only be used to build global functions
 	JitRuntime& get_global_runtime();
 
-	// Emit xbegin and adjacent loop
+	// Emit xbegin and adjacent loop, return label at xbegin
-	void build_transaction_enter(X86Assembler& c, Label fallback);
+	Label build_transaction_enter(X86Assembler& c, Label fallback);
 
 	// Emit xabort
 	void build_transaction_abort(X86Assembler& c, unsigned char code);

rpcs3/Emu/Cell/PPUThread.cpp

@@ -944,8 +944,23 @@ static T ppu_load_acquire_reservation(ppu_thread& ppu, u32 addr)
 
 	ppu.raddr = addr;
 
+	while (g_use_rtm)
+	{
+		ppu.rtime = vm::reservation_acquire(addr, sizeof(T));
+		ppu.rdata = data;
+
+		if (LIKELY(vm::reservation_acquire(addr, sizeof(T)) == ppu.rtime))
+		{
+			return static_cast<T>(ppu.rdata);
+		}
+		else
+		{
+			_mm_pause();
+		}
+	}
+
 	// Do several attemps
-	for (uint i = 0; g_use_rtm || i < 5; i++)
+	for (uint i = 0; i < 5; i++)
 	{
 		ppu.rtime = vm::reservation_acquire(addr, sizeof(T));
 		_mm_lfence();
@@ -981,7 +996,7 @@ extern u64 ppu_ldarx(ppu_thread& ppu, u32 addr)
 	return ppu_load_acquire_reservation<u64>(ppu, addr);
 }
 
-const auto ppu_stwcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rdata, u32 value)>([](asmjit::X86Assembler& c, auto& args)
+const auto ppu_stwcx_tx = build_function_asm<bool(*)(u32 raddr, u64 rtime, u64 rdata, u32 value)>([](asmjit::X86Assembler& c, auto& args)
 {
 	using namespace asmjit;
 
@@ -998,9 +1013,10 @@ const auto ppu_stwcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rd
 	c.lea(x86::r10, x86::qword_ptr(x86::r10, args[0], 3));
 	c.bswap(args[2].r32());
 	c.bswap(args[3].r32());
+	c.mov(args[0].r32(), 5);
 
 	// Begin transaction
-	build_transaction_enter(c, fall);
+	Label begin = build_transaction_enter(c, fall);
 	c.cmp(x86::qword_ptr(x86::r10), args[1]);
 	c.jne(fail);
 	c.cmp(x86::dword_ptr(x86::r11), args[2].r32());
@@ -1017,14 +1033,18 @@ const auto ppu_stwcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rd
 
 	// Touch memory after transaction failure
 	c.bind(fall);
+	c.sub(args[0].r32(), 1);
+	c.jz(fail);
+	c.sar(x86::eax, 24);
+	c.js(fail);
 	c.lock().add(x86::dword_ptr(x86::r11), 0);
 	c.lock().add(x86::qword_ptr(x86::r10), 0);
-	c.sar(x86::eax, 24);
+	c.jmp(begin);
-	c.ret();
 
 	c.bind(fail);
 	build_transaction_abort(c, 0xff);
-	c.int3();
+	c.xor_(x86::eax, x86::eax);
+	c.ret();
 });
 
 extern bool ppu_stwcx(ppu_thread& ppu, u32 addr, u32 reg_value)
@@ -1039,30 +1059,17 @@ extern bool ppu_stwcx(ppu_thread& ppu, u32 addr, u32 reg_value)
 
 	if (g_use_rtm)
 	{
-		// Do several attempts (TODO)
+		if (ppu_stwcx_tx(addr, ppu.rtime, ppu.rdata, reg_value))
-		for (u32 i = 0; i < 5; i++)
-		{
-			const int r = ppu_stwcx_tx(addr, ppu.rtime, ppu.rdata, reg_value);
-
-			if (r > 0)
 		{
 			vm::reservation_notifier(addr, sizeof(u32)).notify_all();
 			ppu.raddr = 0;
 			return true;
 		}
 
-			if (r < 0)
-			{
 		// Reservation lost
 		ppu.raddr = 0;
 		return false;
 	}
-		}
-
-		// Give up
-		ppu.raddr = 0;
-		return false;
-	}
 
 	vm::writer_lock lock(0);
 
@@ -1078,7 +1085,7 @@ extern bool ppu_stwcx(ppu_thread& ppu, u32 addr, u32 reg_value)
 	return result;
 }
 
-const auto ppu_stdcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rdata, u64 value)>([](asmjit::X86Assembler& c, auto& args)
+const auto ppu_stdcx_tx = build_function_asm<bool(*)(u32 raddr, u64 rtime, u64 rdata, u64 value)>([](asmjit::X86Assembler& c, auto& args)
 {
 	using namespace asmjit;
 
@@ -1095,9 +1102,10 @@ const auto ppu_stdcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rd
 	c.lea(x86::r10, x86::qword_ptr(x86::r10, args[0], 3));
 	c.bswap(args[2]);
 	c.bswap(args[3]);
+	c.mov(args[0].r32(), 5);
 
 	// Begin transaction
-	build_transaction_enter(c, fall);
+	Label begin = build_transaction_enter(c, fall);
 	c.cmp(x86::qword_ptr(x86::r10), args[1]);
 	c.jne(fail);
 	c.cmp(x86::qword_ptr(x86::r11), args[2]);
@@ -1114,14 +1122,18 @@ const auto ppu_stdcx_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, u64 rd
 
 	// Touch memory after transaction failure
 	c.bind(fall);
+	c.sub(args[0].r32(), 1);
+	c.jz(fail);
+	c.sar(x86::eax, 24);
+	c.js(fail);
 	c.lock().add(x86::qword_ptr(x86::r11), 0);
 	c.lock().add(x86::qword_ptr(x86::r10), 0);
-	c.sar(x86::eax, 24);
+	c.jmp(begin);
-	c.ret();
 
 	c.bind(fail);
 	build_transaction_abort(c, 0xff);
-	c.int3();
+	c.xor_(x86::eax, x86::eax);
+	c.ret();
 });
 
 extern bool ppu_stdcx(ppu_thread& ppu, u32 addr, u64 reg_value)
@@ -1136,30 +1148,17 @@ extern bool ppu_stdcx(ppu_thread& ppu, u32 addr, u64 reg_value)
 
 	if (g_use_rtm)
 	{
-		// Do several attempts (TODO)
+		if (ppu_stdcx_tx(addr, ppu.rtime, ppu.rdata, reg_value))
-		for (u32 i = 0; i < 5; i++)
-		{
-			const int r = ppu_stdcx_tx(addr, ppu.rtime, ppu.rdata, reg_value);
-
-			if (r > 0)
 		{
 			vm::reservation_notifier(addr, sizeof(u64)).notify_all();
 			ppu.raddr = 0;
 			return true;
 		}
 
-			if (r < 0)
-			{
 		// Reservation lost
 		ppu.raddr = 0;
 		return false;
 	}
-		}
-
-		// Give up
-		ppu.raddr = 0;
-		return false;
-	}
 
 	vm::writer_lock lock(0);
 

rpcs3/Emu/Cell/SPUThread.cpp

@@ -213,7 +213,7 @@ namespace spu
 	}
 }
 
-const auto spu_putllc_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, const void* _old, const void* _new)>([](asmjit::X86Assembler& c, auto& args)
+const auto spu_putllc_tx = build_function_asm<bool(*)(u32 raddr, u64 rtime, const void* _old, const void* _new)>([](asmjit::X86Assembler& c, auto& args)
 {
 	using namespace asmjit;
 
@@ -228,6 +228,7 @@ const auto spu_putllc_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, const
 	c.lea(x86::r11, x86::qword_ptr(x86::r11, args[0]));
 	c.shr(args[0], 4);
 	c.lea(x86::r10, x86::qword_ptr(x86::r10, args[0]));
+	c.mov(args[0].r32(), 3);
 
 	// Prepare data (Windows has only 6 volatile vector registers)
 	c.vmovups(x86::ymm0, x86::yword_ptr(args[2], 0));
@@ -245,7 +246,7 @@ const auto spu_putllc_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, const
 #endif
 
 	// Begin transaction
-	build_transaction_enter(c, fall);
+	Label begin = build_transaction_enter(c, fall);
 	c.cmp(x86::qword_ptr(x86::r10), args[1]);
 	c.jne(fail);
 	c.vxorps(x86::ymm0, x86::ymm0, x86::yword_ptr(x86::r11, 0));
@@ -282,14 +283,22 @@ const auto spu_putllc_tx = build_function_asm<int(*)(u32 raddr, u64 rtime, const
 
 	// Touch memory after transaction failure
 	c.bind(fall);
+	c.sub(args[0].r32(), 1);
+	c.jz(fail);
+	c.sar(x86::eax, 24);
+	c.js(fail);
 	c.lock().add(x86::qword_ptr(x86::r11), 0);
 	c.lock().add(x86::qword_ptr(x86::r10), 0);
-	c.sar(x86::eax, 24);
+#ifdef _WIN32
-	c.ret();
+	c.vmovups(x86::ymm4, x86::yword_ptr(args[3], 0));
+	c.vmovups(x86::ymm5, x86::yword_ptr(args[3], 96));
+#endif
+	c.jmp(begin);
 
 	c.bind(fail);
 	build_transaction_abort(c, 0xff);
-	c.int3();
+	c.xor_(x86::eax, x86::eax);
+	c.ret();
 });
 
 const auto spu_getll_tx = build_function_asm<u64(*)(u32 raddr, void* rdata)>([](asmjit::X86Assembler& c, auto& args)
@@ -308,7 +317,7 @@ const auto spu_getll_tx = build_function_asm<u64(*)(u32 raddr, void* rdata)>([](
 	c.lea(x86::r10, x86::qword_ptr(x86::r10, args[0]));
 
 	// Begin transaction
-	build_transaction_enter(c, fall);
+	Label begin = build_transaction_enter(c, fall);
 	c.mov(x86::rax, x86::qword_ptr(x86::r10));
 	c.vmovaps(x86::ymm0, x86::yword_ptr(x86::r11, 0));
 	c.vmovaps(x86::ymm1, x86::yword_ptr(x86::r11, 32));
@@ -324,13 +333,13 @@ const auto spu_getll_tx = build_function_asm<u64(*)(u32 raddr, void* rdata)>([](
 
 	// Touch memory after transaction failure
 	c.bind(fall);
+	c.pause();
 	c.mov(x86::rax, x86::qword_ptr(x86::r11));
 	c.mov(x86::rax, x86::qword_ptr(x86::r10));
-	c.mov(x86::eax, 1);
+	c.jmp(begin);
-	c.ret();
 });
 
-const auto spu_putlluc_tx = build_function_asm<bool(*)(u32 raddr, const void* rdata)>([](asmjit::X86Assembler& c, auto& args)
+const auto spu_putlluc_tx = build_function_asm<void(*)(u32 raddr, const void* rdata)>([](asmjit::X86Assembler& c, auto& args)
 {
 	using namespace asmjit;
 
@@ -352,7 +361,7 @@ const auto spu_putlluc_tx = build_function_asm<bool(*)(u32 raddr, const void* rd
 	c.vmovups(x86::ymm3, x86::yword_ptr(args[1], 96));
 
 	// Begin transaction
-	build_transaction_enter(c, fall);
+	Label begin = build_transaction_enter(c, fall);
 	c.vmovaps(x86::yword_ptr(x86::r11, 0), x86::ymm0);
 	c.vmovaps(x86::yword_ptr(x86::r11, 32), x86::ymm1);
 	c.vmovaps(x86::yword_ptr(x86::r11, 64), x86::ymm2);
@@ -364,15 +373,14 @@ const auto spu_putlluc_tx = build_function_asm<bool(*)(u32 raddr, const void* rd
 	c.mov(x86::qword_ptr(x86::r10), x86::rax);
 	c.xend();
 	c.vzeroupper();
-	c.mov(x86::eax, 1);
 	c.ret();
 
 	// Touch memory after transaction failure
 	c.bind(fall);
+	c.pause();
 	c.lock().add(x86::qword_ptr(x86::r11), 0);
 	c.lock().add(x86::qword_ptr(x86::r10), 0);
-	c.xor_(x86::eax, x86::eax);
+	c.jmp(begin);
-	c.ret();
 });
 
 void spu_int_ctrl_t::set(u64 ints)
@@ -1042,22 +1050,14 @@ void SPUThread::do_putlluc(const spu_mfc_cmd& args)
 	auto& data = vm::_ref<decltype(rdata)>(addr);
 	const auto to_write = _ref<decltype(rdata)>(args.lsa & 0x3ffff);
 
-	vm::reservation_acquire(addr, 128);
-
 	// Store unconditionally
-	while (g_use_rtm)
+	if (g_use_rtm)
-	{
-		if (spu_putlluc_tx(addr, to_write.data()))
 	{
+		spu_putlluc_tx(addr, to_write.data());
 		vm::reservation_notifier(addr, 128).notify_all();
-			tx_success++;
 		return;
 	}
 
-		busy_wait(300);
-		tx_failure++;
-	}
-
 	vm::writer_lock lock(0);
 	vm::reservation_update(addr, 128, true);
 	_mm_sfence();
@@ -1248,19 +1248,9 @@ bool SPUThread::process_mfc_cmd(spu_mfc_cmd args)
 			}
 		}
 
-		while (g_use_rtm)
+		if (g_use_rtm)
 		{
 			rtime = spu_getll_tx(raddr, rdata.data());
-
-			if (rtime & 1)
-			{
-				tx_failure++;
-				busy_wait(300);
-				continue;
-			}
-
-			tx_success++;
-			break;
 		}
 
 		// Do several attemps
@@ -1312,28 +1302,13 @@ bool SPUThread::process_mfc_cmd(spu_mfc_cmd args)
 		{
 			if (g_use_rtm)
 			{
-				// Do several attempts (TODO)
+				if (spu_putllc_tx(raddr, rtime, rdata.data(), to_write.data()))
-				for (u32 i = 0; i < 3; i++)
-				{
-					const int r = spu_putllc_tx(raddr, rtime, rdata.data(), to_write.data());
-
-					if (r > 0)
 				{
 					vm::reservation_notifier(raddr, 128).notify_all();
 					result = true;
-						tx_success++;
-						break;
-					}
-
-					if (r < 0)
-					{
-						// Reservation lost
-						break;
 				}
 
 				// Don't fallback to heavyweight lock, just give up
-					tx_failure++;
-				}
 			}
 			else if (rdata == data)
 			{