Add vertex shader dumping/override

This change generalizes the pixel shader dumping and override support to
also include vertex shaders.

Author: niligulmohar

DATA/DSfix.ini

@@ -201,14 +201,17 @@ enableTexturePrefetch 0
 # Shader Override Options
 ###############################################################################
 
-# Enable pixel shader dumping
-# Disassembled pixel shaders will be dumped to "dsfix\pixelshader_dump\[hash].asm".
-enablePixelShaderDumping 0
+# Enable shader dumping
+# Disassembled pixel shaders will be dumped to
+# "dsfix\pixelshader_dump\[hash].asm", vertex shaders to
+# "dsfix\vertexshader_dump\[hash].asm".
+enableShaderDumping 0
 
 # Enable pixel shader override
-# Pixel shaders named "dsfix\pixelshader_override\[hash].asm" will
-# replace the corresponding originals.
-enablePixelShaderOverride 0
+# Shaders from files named "dsfix\pixelshader_override\[hash].asm" or
+# "dsfix\vertexshader_override\[hash].asm" will replace the
+# corresponding originals.
+enableShaderOverride 0
 
 ###############################################################################
 # Other Options

RenderstateManager.cpp

@@ -15,8 +15,10 @@
 
 RSManager RSManager::instance;
 
-#define PIXEL_SHADER_DUMP_DIR "dsfix/pixelshader_dump"
-#define PIXEL_SHADER_OVERRIDE_DIR "dsfix/pixelshader_override"
+static const char *PIXEL_SHADER_DUMP_DIR = "dsfix/pixelshader_dump";
+static const char *PIXEL_SHADER_OVERRIDE_DIR = "dsfix/pixelshader_override";
+static const char *VERTEX_SHADER_DUMP_DIR = "dsfix/vertexshader_dump";
+static const char *VERTEX_SHADER_OVERRIDE_DIR = "dsfix/vertexshader_override";
 
 void RSManager::initResources() {
 	SDLOG(0, "RenderstateManager resource initialization started\n");
@@ -40,12 +42,9 @@ void RSManager::initResources() {
     if (Settings::get().getEnableTextureOverride() && Settings::get().getEnableTexturePrefetch())
         prefetchTextures();
 
-	if (Settings::get().getEnablePixelShaderDumping()) {
-		CreateDirectory(GetDirectoryFile(PIXEL_SHADER_DUMP_DIR), nullptr);
-		DWORD error = GetLastError();
-		if (error && error != ERROR_ALREADY_EXISTS) {
-			SDLOG(0, "Failed to create %s: %s\n", PIXEL_SHADER_DUMP_DIR, formatMessage(error));
-		}
+	if (Settings::get().getEnableShaderDumping()) {
+		createDirectory(PIXEL_SHADER_DUMP_DIR);
+		createDirectory(VERTEX_SHADER_DUMP_DIR);
 	}
 
 	SDLOG(0, "RenderstateManager resource initialization completed\n");
@@ -869,14 +868,14 @@ HRESULT RSManager::redirectSetRenderState(D3DRENDERSTATETYPE State, DWORD Value)
 
 HRESULT RSManager::redirectCreatePixelShader(CONST DWORD* pFunction, IDirect3DPixelShader9** ppShader)
 {
-	bool shouldDump = Settings::get().getEnablePixelShaderDumping();
-	bool shouldOverride = Settings::get().getEnablePixelShaderOverride();
+	bool shouldDump = Settings::get().getEnableShaderDumping();
+	bool shouldOverride = Settings::get().getEnableShaderOverride();
 	LPD3DXBUFFER pAssemblerBuffer = nullptr;
 	LPD3DXBUFFER pFunctionBuffer = nullptr;
 	UINT32 hash;
 
 	if (shouldDump || shouldOverride) {
-		if (disassemblePixelShader(pFunction, &pAssemblerBuffer)) {
+		if (disassembleShader(pFunction, &pAssemblerBuffer)) {
 			hash = SuperFastHash(static_cast<char *>(pAssemblerBuffer->GetBufferPointer()), pAssemblerBuffer->GetBufferSize());
 		}
 		else {
@@ -886,11 +885,11 @@ HRESULT RSManager::redirectCreatePixelShader(CONST DWORD* pFunction, IDirect3DPi
 	}
 
 	if (shouldDump) {
-		dumpPixelShader(hash, pAssemblerBuffer);
+		dumpShader(hash, PIXEL_SHADER_DUMP_DIR, pAssemblerBuffer);
 	}
 
 	if (shouldOverride) {
-		if (getOverridePixelShader(hash, &pFunctionBuffer)) {
+		if (getOverrideShader(hash, PIXEL_SHADER_OVERRIDE_DIR, &pFunctionBuffer)) {
 			pFunction = static_cast<DWORD *>(pFunctionBuffer->GetBufferPointer());
 		}
 	}
@@ -904,33 +903,70 @@ HRESULT RSManager::redirectCreatePixelShader(CONST DWORD* pFunction, IDirect3DPi
 	return result;
 }
 
-bool RSManager::disassemblePixelShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer) {
+HRESULT RSManager::redirectCreateVertexShader(CONST DWORD* pFunction, IDirect3DVertexShader9** ppShader)
+{
+	bool shouldDump = Settings::get().getEnableShaderDumping();
+	bool shouldOverride = Settings::get().getEnableShaderOverride();
+	LPD3DXBUFFER pAssemblerBuffer = nullptr;
+	LPD3DXBUFFER pFunctionBuffer = nullptr;
+	UINT32 hash;
+
+	if (shouldDump || shouldOverride) {
+		if (disassembleShader(pFunction, &pAssemblerBuffer)) {
+			hash = SuperFastHash(static_cast<char *>(pAssemblerBuffer->GetBufferPointer()), pAssemblerBuffer->GetBufferSize());
+		}
+		else {
+			shouldDump = false;
+			shouldOverride = false;
+		}
+	}
+
+	if (shouldDump) {
+		dumpShader(hash, VERTEX_SHADER_DUMP_DIR, pAssemblerBuffer);
+	}
+
+	if (shouldOverride) {
+		if (getOverrideShader(hash, VERTEX_SHADER_OVERRIDE_DIR, &pFunctionBuffer)) {
+			pFunction = static_cast<DWORD *>(pFunctionBuffer->GetBufferPointer());
+		}
+	}
+
+	HRESULT result = d3ddev->CreateVertexShader(pFunction, ppShader);
+	if (shouldDump || shouldOverride) {
+		SDLOG(1, "Created vertex shader for hash %08x: 0x%p\n", hash, *ppShader);
+	}
+	SAFERELEASE(pAssemblerBuffer);
+	SAFERELEASE(pFunctionBuffer);
+	return result;
+}
+
+bool RSManager::disassembleShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer) {
 	LPD3DXBUFFER buffer = nullptr;
 	HRESULT result = D3DXDisassembleShader(pFunction, false, NULL, ppBuffer);
 	if (result != D3D_OK) {
-		SDLOG(0, "Failed to disassemble pixel shader\n");
+		SDLOG(0, "Failed to disassemble shader\n");
 	}
 	return result == D3D_OK;
 }
 
-void RSManager::dumpPixelShader(UINT32 hash, LPD3DXBUFFER pBuffer) {
+void RSManager::dumpShader(UINT32 hash, const char *directory, LPD3DXBUFFER pBuffer) {
 	char fileNameBuffer[64];
-	sprintf_s(fileNameBuffer, PIXEL_SHADER_DUMP_DIR "/%08x.asm", hash);
+	sprintf_s(fileNameBuffer, "%s/%08x.asm", directory, hash);
 	const char *fileName = GetDirectoryFile(fileNameBuffer);
-	if (writeFile(fileName, static_cast<char *>(pBuffer->GetBufferPointer()), pBuffer->GetBufferSize())) {
-		SDLOG(0, "Wrote disassembled pixel shader to %s\n", fileNameBuffer);
+	if (writeFile(fileName, static_cast<char *>(pBuffer->GetBufferPointer()), pBuffer->GetBufferSize() - 1)) {
+		SDLOG(0, "Wrote disassembled shader to %s\n", fileNameBuffer);
 	}
 	else {
-		SDLOG(0, "Failed to write disassembled pixel shader to %s\n", fileNameBuffer);
+		SDLOG(0, "Failed to write disassembled shader to %s\n", fileNameBuffer);
 	}
 }
 
-bool RSManager::getOverridePixelShader(UINT32 hash, LPD3DXBUFFER *ppBuffer) {
+bool RSManager::getOverrideShader(UINT32 hash, const char *directory, LPD3DXBUFFER *ppBuffer) {
 	char fileNameBuffer[64];
-	sprintf_s(fileNameBuffer, PIXEL_SHADER_OVERRIDE_DIR "/%08x.asm", hash);
+	sprintf_s(fileNameBuffer, "%s/%08x.asm", directory, hash);
 	const char * fileName = GetDirectoryFile(fileNameBuffer);
 	if (fileExists(fileName)) {
-		SDLOG(1, "Pixel shader override: %08x\n", hash);
+		SDLOG(1, "Shader override: %s\n", fileName);
 		LPD3DXBUFFER errors = nullptr;
 		HRESULT assembleResult = D3DXAssembleShaderFromFile(fileName, nullptr, nullptr, 0, ppBuffer, &errors);
 		if (assembleResult != D3D_OK) {

RenderstateManager.h

@@ -113,9 +113,9 @@ class RSManager {
 
     std::map<UINT32, MemData> cachedTexFiles;
 
-	bool disassemblePixelShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer);
-	void dumpPixelShader(UINT32 hash, LPD3DXBUFFER pBuffer);
-	bool getOverridePixelShader(UINT32 hash, LPD3DXBUFFER *ppBuffer);
+	bool disassembleShader(CONST DWORD *pFunction, LPD3DXBUFFER *ppBuffer);
+	void dumpShader(UINT32 hash, const char *directory, LPD3DXBUFFER pBuffer);
+	bool getOverrideShader(UINT32 hash, const char *directory, LPD3DXBUFFER *ppBuffer);
 
 private:
     ~RSManager();
@@ -194,4 +194,5 @@ class RSManager {
 	HRESULT redirectSetTextureStageState(DWORD Stage, D3DTEXTURESTAGESTATETYPE Type, DWORD Value);
 	HRESULT redirectSetRenderState(D3DRENDERSTATETYPE State, DWORD Value);
 	HRESULT redirectCreatePixelShader(CONST DWORD *pfunction, IDirect3DPixelShader9 **ppShader);
+	HRESULT redirectCreateVertexShader(CONST DWORD *pfunction, IDirect3DVertexShader9 **ppShader);
 };

Settings.def

@@ -52,9 +52,9 @@ SETTING(bool, EnableTextureDumping, "enableTextureDumping", false);
 SETTING(bool, EnableTextureOverride, "enableTextureOverride", false);
 SETTING(bool, EnableTexturePrefetch, "enableTexturePrefetch", false);
 
-// Pixel Shader Override Options
-SETTING(bool, EnablePixelShaderDumping, "enablePixelShaderDumping", false);
-SETTING(bool, EnablePixelShaderOverride, "enablePixelShaderOverride", false);
+// Shader Override Options
+SETTING(bool, EnableShaderDumping, "enableShaderDumping", false);
+SETTING(bool, EnableShaderOverride, "enableShaderOverride", false);
 
 // HUD options
 SETTING(bool, EnableHudMod, "enableHudMod", false)

d3d9dev.cpp

@@ -253,14 +253,7 @@ HRESULT APIENTRY hkIDirect3DDevice9::CreateVertexDeclaration(CONST D3DVERTEXELEM
 }
 
 HRESULT APIENTRY hkIDirect3DDevice9::CreateVertexShader(CONST DWORD* pFunction,IDirect3DVertexShader9** ppShader) {
-	HRESULT res = m_pD3Ddev->CreateVertexShader(pFunction, ppShader);
-	if(Settings::get().getLogLevel() > 12) {
-		SDLOG(12, "CreateVertexShader data: %p : shader: %p\n", pFunction, *ppShader);
-		LPD3DXBUFFER buffer;
-		D3DXDisassembleShader(pFunction, false, NULL, &buffer);
-		SDLOG(12, "===== disassembly:\n%s\n==============\n", buffer->GetBufferPointer());
-	}
-	return res;
+	return RSManager::get().redirectCreateVertexShader(pFunction, ppShader);
 }
 
 HRESULT APIENTRY hkIDirect3DDevice9::CreateVolumeTexture(UINT Width,UINT Height,UINT Depth,UINT Levels,DWORD Usage,D3DFORMAT Format,D3DPOOL Pool,IDirect3DVolumeTexture9** ppVolumeTexture,HANDLE* pSharedHandle) {

main.cpp

@@ -79,7 +79,7 @@ bool WINAPI DllMain(HMODULE hDll, DWORD dwReason, PVOID pvReserved) {
     return false;
 }
 
-char *GetDirectoryFile(char *filename) {
+char *GetDirectoryFile(const char *filename) {
 	static char path[320];
 	strcpy_s(path, dlldir);
 	strcat_s(path, filename);
@@ -136,6 +136,14 @@ bool fileExists(const char *filename) {
   return std::ifstream(filename).good();
 }
 
+void createDirectory(const char *fileName) {
+	CreateDirectory(GetDirectoryFile(fileName), nullptr);
+	DWORD error = GetLastError();
+	if (error && error != ERROR_ALREADY_EXISTS) {
+		SDLOG(0, "Failed to create %s: %s\n", fileName, formatMessage(error));
+	}
+}
+
 bool writeFile(const char *filename, const char *data, size_t length) {
 	std::ofstream file(filename, std::ios::out | std::ios::binary);
 	if (!file) {

main.h

@@ -36,8 +36,9 @@
 #include "dinput.h"
 #include <string>
 
-char *GetDirectoryFile(char *filename);
+char *GetDirectoryFile(const char *filename);
 bool fileExists(const char *filename);
+void createDirectory(const char *filename);
 bool writeFile(const char *filename, const char *data, size_t length);
 std::string formatMessage(DWORD messageId);
 std::string strError(int err);