initial

utils/shadercompile/_vpc_/manifest_shadercompile_dll/win32/manifest.txt

@@ -0,0 +1,88 @@
+// ----------------------------------------- //
+//           File generated by VPC           //
+// ----------------------------------------- //
+
+Source file:            F:\csgo_64\cstrike15_src\utils\shadercompile\cfgprocessor.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\shadercompile\cfgprocessor.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\shadercompile\cfgprocessor.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\common\cmdlib.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\common\cmdlib.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\common\cmdlib.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\shadercompile\cmdsink.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\shadercompile\cmdsink.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\shadercompile\cmdsink.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\shadercompile\d3dxfxc.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\shadercompile\d3dxfxc.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\shadercompile\d3dxfxc.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\common\debug_dll_check.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\common\debug_dll_check.cpp
+Release output file:    F:\csgo_64\cstrike15_src\common\debug_dll_check.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\public\filesystem_helpers.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\public\filesystem_helpers.cpp
+Release output file:    F:\csgo_64\cstrike15_src\public\filesystem_helpers.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\public\tier0\memoverride.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\public\tier0\memoverride.cpp
+Release output file:    F:\csgo_64\cstrike15_src\public\tier0\memoverride.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\common\pacifier.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\common\pacifier.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\common\pacifier.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\shadercompile\shadercompile.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\shadercompile\shadercompile.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\shadercompile\shadercompile.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\shadercompile\shadercompile_ps3_helpers.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\shadercompile\shadercompile_ps3_helpers.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\shadercompile\shadercompile_ps3_helpers.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\shadercompile\subprocess.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\shadercompile\subprocess.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\shadercompile\subprocess.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\common\threads.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\common\threads.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\common\threads.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\common\tools_minidump.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\common\tools_minidump.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\common\tools_minidump.cpp
+Containing unity file:  
+PCH file:               
+
+Source file:            F:\csgo_64\cstrike15_src\utils\common\vmpi_tools_shared.cpp
+Debug output file:      F:\csgo_64\cstrike15_src\utils\common\vmpi_tools_shared.cpp
+Release output file:    F:\csgo_64\cstrike15_src\utils\common\vmpi_tools_shared.cpp
+Containing unity file:  
+PCH file:               
+

utils/shadercompile/cfgprocessor.h

@@ -0,0 +1,76 @@
+//====== Copyright c 1996-2007, Valve Corporation, All rights reserved. =======//
+//
+// Purpose: 
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#ifndef CFGPROCESSOR_H
+#define CFGPROCESSOR_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "tier1/smartptr.h"
+
+
+/*
+
+Layout of the internal structures is as follows:
+
+|-------- shader1.fxc ---------||--- shader2.fxc ---||--------- shader3.fxc -----||-...
+| 0 s s 3 s s s s 8 s 10 s s s || s s 2 3 4 s s s 8 || 0 s s s 4 s s s 8 9 s s s ||-...
+| 0 1 2 3 4 5 6 7 8 9 10 * * *   14 * * * * *20 * *   23 * * *27 * * * * * * *35    * * *
+
+GetSection( 10 ) -> shader1.fxc
+GetSection( 27 ) -> shader3.fxc
+
+GetNextCombo(  3,  3, 14 ) -> shader1.fxc : ( riCommandNumber =  8, rhCombo =  "8" )
+GetNextCombo( 10, 10, 14 ) ->   NULL      : ( riCommandNumber = 14, rhCombo = NULL )
+GetNextCombo( 22,  8, 36 ) -> shader3.fxc : ( riCommandNumber = 23, rhCombo =  "0" )
+GetNextCombo( 29, -1, 36 ) -> shader3.fxc : ( riCommandNumber = 31, rhCombo =  "8" )
+
+*/
+
+class CUtlInplaceBuffer;
+
+namespace CfgProcessor
+{
+
+// Working with configuration
+void ReadConfiguration( FILE *fInputStream );
+void ReadConfiguration( CUtlInplaceBuffer *fInputStream );
+
+struct CfgEntryInfo
+{
+	char const *m_szName;				// Name of the shader, e.g. "shader_ps20b"
+	char const *m_szShaderFileName;		// Name of the src file, e.g. "shader_psxx.fxc"
+	uint64 m_numCombos;					// Total possible num of combos, e.g. 1024
+	uint64 m_numDynamicCombos;			// Num of dynamic combos, e.g. 4
+	uint64 m_numStaticCombos;			// Num of static combos, e.g. 256
+	uint64 m_iCommandStart;				// Start command, e.g. 0
+	uint64 m_iCommandEnd;				// End command, e.g. 1024
+};
+
+void DescribeConfiguration( CArrayAutoPtr < CfgEntryInfo > &rarrEntries );
+
+
+// Working with combos
+typedef struct {} * ComboHandle;
+
+ComboHandle Combo_GetCombo( uint64 iCommandNumber );
+ComboHandle Combo_GetNext( uint64 &riCommandNumber, ComboHandle &rhCombo, uint64 iCommandEnd );
+void Combo_FormatCommand( ComboHandle hCombo, char *pchBuffer );
+uint64 Combo_GetCommandNum( ComboHandle hCombo );
+uint64 Combo_GetComboNum( ComboHandle hCombo );
+CfgEntryInfo const *Combo_GetEntryInfo( ComboHandle hCombo );
+
+ComboHandle Combo_Alloc( ComboHandle hComboCopyFrom );
+void Combo_Assign( ComboHandle hComboDst, ComboHandle hComboSrc );
+void Combo_Free( ComboHandle &rhComboFree );
+
+}; // namespace CfgProcessor
+
+
+#endif // #ifndef CFGPROCESSOR_H

utils/shadercompile/cmdsink.cpp

@@ -0,0 +1,114 @@
+//====== Copyright © 1996-2006, Valve Corporation, All rights reserved. =======//
+//
+// Purpose: Command sink interface implementation.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#include "cmdsink.h"
+
+
+namespace CmdSink
+{
+
+// ------ implementation of CResponseFiles --------------
+
+CResponseFiles::CResponseFiles( char const *szFileResult, char const *szFileListing ) :
+	m_fResult(NULL),
+	m_fListing(NULL),
+	m_lenResult(0),
+	m_dataResult(NULL),
+	m_dataListing(NULL)
+{
+	sprintf( m_szFileResult, szFileResult );
+	sprintf( m_szFileListing, szFileListing );
+}
+
+CResponseFiles::~CResponseFiles( void )
+{
+	if ( m_fResult )
+		fclose( m_fResult );
+
+	if ( m_fListing )
+		fclose( m_fListing );
+}
+
+bool CResponseFiles::Succeeded( void )
+{
+	OpenResultFile();
+	return ( m_fResult != NULL );
+}
+
+size_t CResponseFiles::GetResultBufferLen( void )
+{
+	ReadResultFile();
+	return m_lenResult;
+}
+
+const void * CResponseFiles::GetResultBuffer( void )
+{
+	ReadResultFile();
+	return m_dataResult;
+}
+
+const char * CResponseFiles::GetListing( void )
+{
+	ReadListingFile();
+	return ( ( m_dataListing && *m_dataListing ) ? m_dataListing : NULL );
+}
+
+void CResponseFiles::OpenResultFile( void )
+{
+	if ( !m_fResult )
+	{
+		m_fResult = fopen( m_szFileResult, "rb" );
+	}
+}
+
+void CResponseFiles::ReadResultFile( void )
+{
+	if ( !m_dataResult )
+	{
+		OpenResultFile();
+
+		if ( m_fResult )
+		{
+			fseek( m_fResult, 0, SEEK_END );
+			m_lenResult = (size_t) ftell( m_fResult );
+
+			if ( m_lenResult != size_t(-1) )
+			{
+				m_bufResult.EnsureCapacity( m_lenResult );
+				fseek( m_fResult, 0, SEEK_SET );
+				fread( m_bufResult.Base(), 1, m_lenResult, m_fResult );
+				m_dataResult = m_bufResult.Base();
+			}
+		}
+	}
+}
+
+void CResponseFiles::ReadListingFile( void )
+{
+	if ( !m_dataListing )
+	{
+		if ( !m_fListing )
+			m_fListing = fopen( m_szFileListing, "rb" );
+
+		if ( m_fListing )
+		{
+			fseek( m_fListing, 0, SEEK_END );
+			size_t len = (size_t) ftell( m_fListing );
+
+			if ( len != size_t(-1) )
+			{
+				m_bufListing.EnsureCapacity( len );
+				fseek( m_fListing, 0, SEEK_SET );
+				fread( m_bufListing.Base(), 1, len, m_fListing );
+				m_dataListing = (const char *) m_bufListing.Base();
+			}
+		}
+	}
+}
+
+}; // namespace CmdSink

utils/shadercompile/cmdsink.h

@@ -0,0 +1,117 @@
+//====== Copyright © 1996-2006, Valve Corporation, All rights reserved. =======//
+//
+// Purpose: Command sink interface implementation.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#ifndef CMDSINK_H
+#define CMDSINK_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include <stdio.h>
+#include <tier1/utlbuffer.h>
+
+
+namespace CmdSink
+{
+
+/*
+
+struct IResponse
+
+Interface to give back command execution results.
+
+*/
+struct IResponse
+{
+	virtual ~IResponse( void ) {}
+	virtual void Release( void ) { delete this; }
+
+	// Returns whether the command succeeded
+	virtual bool Succeeded( void ) = 0;
+
+	// If the command succeeded returns the result buffer length, otherwise zero
+	virtual size_t GetResultBufferLen( void ) = 0;
+	// If the command succeeded returns the result buffer base pointer, otherwise NULL
+	virtual const void * GetResultBuffer( void ) = 0;
+
+	// Returns a zero-terminated string of messages reported during command execution, or NULL if nothing was reported
+	virtual const char * GetListing( void ) = 0;
+};
+
+
+
+
+/*
+
+Response implementation when the result should appear in
+one file and the listing should appear in another file.
+
+*/
+class CResponseFiles : public IResponse
+{
+public:
+	explicit CResponseFiles( char const *szFileResult, char const *szFileListing );
+	~CResponseFiles( void );
+
+public:
+	// Returns whether the command succeeded
+	virtual bool Succeeded( void );
+
+	// If the command succeeded returns the result buffer length, otherwise zero
+	virtual size_t GetResultBufferLen( void );
+	// If the command succeeded returns the result buffer base pointer, otherwise NULL
+	virtual const void * GetResultBuffer( void );
+
+	// Returns a zero-terminated string of messages reported during command execution
+	virtual const char * GetListing( void );
+
+protected:
+	void OpenResultFile( void );		//!< Opens the result file if not open yet
+	void ReadResultFile( void );		//!< Reads the result buffer if not read yet
+	void ReadListingFile( void );		//!< Reads the listing buffer if not read yet
+
+protected:
+	char m_szFileResult[MAX_PATH];		//!< Name of the result file
+	char m_szFileListing[MAX_PATH];		//!< Name of the listing file
+
+	FILE *m_fResult;					//!< Result file (NULL if not open)
+	FILE *m_fListing;					//!< Listing file (NULL if not open)
+
+	CUtlBuffer m_bufResult;				//!< Buffer holding the result data
+	size_t m_lenResult;					//!< Result data length (0 if result not read yet)
+	const void *m_dataResult;			//!< Data buffer pointer (NULL if result not read yet)
+
+	CUtlBuffer m_bufListing;			//!< Buffer holding the listing
+	const char *m_dataListing;			//!< Listing buffer pointer (NULL if listing not read yet)
+};
+
+/*
+
+Response implementation when the result is a generic error.
+
+*/
+class CResponseError : public IResponse
+{
+public:
+	explicit CResponseError( void ) {}
+	~CResponseError( void ) {}
+
+public:
+	virtual bool Succeeded( void ) { return false; }
+
+	virtual size_t GetResultBufferLen( void ) { return 0; }
+	virtual const void * GetResultBuffer( void ) { return NULL; }
+
+	virtual const char * GetListing( void ) { return NULL; }
+};
+
+
+}; // namespace CmdSink
+
+
+#endif // #ifndef CMDSINK_H

utils/shadercompile/d3dxfxc.cpp

@@ -0,0 +1,278 @@
+//====== Copyright © 1996-2006, Valve Corporation, All rights reserved. =======//
+//
+// Purpose: D3DX command implementation.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#include "shadercompile.h"
+
+#include "d3dxfxc.h"
+#include "cmdsink.h"
+
+// Required to compile using D3DX* routines in the same process
+#include <d3dx9shader.h>
+#include "dx_proxy/dx_proxy.h"
+
+#include <tier0/icommandline.h>
+#include <tier1/strtools.h>
+
+#define D3DXSHADER_MICROCODE_BACKEND_OLD_DEPRECATED ( 1 << 25 )
+
+namespace InterceptFxc
+{
+
+	// The command that is intercepted by this namespace routines
+	enum ExecutableEnum
+	{
+		EXEC_FXC,
+		EXEC_SCE_CGC,
+		EXEC_COUNT
+	};
+	static const char * g_pExecutables[EXEC_COUNT] = 
+	{
+		"fxc.exe",
+		"sce-cgc.exe"
+	};
+
+	namespace Private		 
+	{
+		//
+		// Response implementation
+		//
+		class CResponse : public CmdSink::IResponse
+		{
+		public:
+			explicit CResponse( LPD3DXBUFFER pShader, LPD3DXBUFFER pListing, HRESULT hr );
+			~CResponse( void );
+
+		public:
+			virtual bool Succeeded( void ) { return m_pShader && (m_hr == D3D_OK); }
+			virtual size_t GetResultBufferLen( void ) { return ( Succeeded() ? m_pShader->GetBufferSize() : 0 ); }
+			virtual const void * GetResultBuffer( void ) { return ( Succeeded() ? m_pShader->GetBufferPointer() : NULL ); }
+			virtual const char * GetListing( void ) { return (const char *) ( m_pListing ? m_pListing->GetBufferPointer() : NULL ); }
+
+		protected:
+			LPD3DXBUFFER m_pShader;
+			LPD3DXBUFFER m_pListing;
+			HRESULT m_hr;
+		};
+
+		CResponse::CResponse( LPD3DXBUFFER pShader, LPD3DXBUFFER pListing, HRESULT hr ) :
+			m_pShader(pShader),
+			m_pListing(pListing),
+			m_hr(hr)
+		{
+			NULL;
+		}
+
+		CResponse::~CResponse( void )
+		{
+			if ( m_pShader )
+				m_pShader->Release();
+
+			if ( m_pListing )
+				m_pListing->Release();
+		}
+
+		//
+		// Perform a fast shader file compilation.
+		//		TODO: avoid writing "shader.o" and "output.txt" files to avoid extra filesystem access.
+		//
+		// @param pszFilename		the filename to compile (e.g. "debugdrawenvmapmask_vs20.fxc")
+		// @param pMacros			null-terminated array of macro-defines
+		// @param pszModel			shader model for compilation
+		//
+		void FastShaderCompile( ExecutableEnum nExecutable, const char *pszFilename, const D3DXMACRO *pMacros, const char *pszModel, CmdSink::IResponse **ppResponse )
+		{
+			LPD3DXBUFFER pShader = NULL; // NOTE: Must release the COM interface later
+			LPD3DXBUFFER pErrorMessages = NULL; // NOTE: Must release COM interface later
+			
+			HRESULT hresult = S_OK;
+
+			static DxProxyModule s_dxModule;
+			hresult = s_dxModule.D3DXCompileShaderFromFile( pszFilename, pMacros, NULL /* LPD3DXINCLUDE */,
+				"main",	pszModel, 0, &pShader, &pErrorMessages,
+				NULL /* LPD3DXCONSTANTTABLE *ppConstantTable */ );
+
+			if ( ppResponse )
+			{
+				*ppResponse = new CResponse( pShader, pErrorMessages, hresult );
+			}
+			else
+			{
+				if ( pShader )
+				{
+					pShader->Release();
+				}
+
+				if ( pErrorMessages )
+				{
+					pErrorMessages->Release();
+				}
+			}
+		}
+
+	}; // namespace Private
+
+	// return the next option (starting with -D or /D), or NULL
+	inline char * FindOptionD( char * pCommand )
+	{
+		char * p = pCommand;
+		while( V_isspace(*p) )
+			++p;
+		while( *p )
+		{
+			if( ( *p == '/' || *p == '-' ) && p[1] == 'D' )
+			{
+				return p + 2;
+			}
+			++p;
+		}
+		return NULL;
+	}
+	
+	
+
+	//
+	// Completely mimic the behaviour of "fxc.exe" in the specific cases related
+	// to shader compilations.
+	//
+	// @param pCommand       the command in form
+	//		"fxc.exe /DSHADERCOMBO=1 /DTOTALSHADERCOMBOS=4 /DCENTROIDMASK=0 /DNUMDYNAMICCOMBOS=4 /DFLAGS=0x0 /DNUM_BONES=1 /Dmain=main /Emain /Tvs_2_0 /DSHADER_MODEL_VS_2_0=1 /D_X360=1 /nologo /Foshader.o debugdrawenvmapmask_vs20.fxc>output.txt 2>&1"
+	//
+	void ExecuteCommand( ExecutableEnum nExecutable, const char *pCommand, CmdSink::IResponse **ppResponse )
+	{
+		// A duplicate portion of memory for modifications
+		void *bufEditableCommand = alloca( strlen( pCommand ) + 1 );
+		char *pEditableCommand = strcpy( (char *) bufEditableCommand, pCommand );
+
+		// Macros to be defined for D3DX
+		CUtlVector<D3DXMACRO> macros;
+
+		// Shader model (determined when parsing "/D" flags)
+		const char *pszShaderModel = NULL;
+
+		// Iterate over the command line and find all "/D...=..." settings
+		for ( char *pszFlag = pEditableCommand;
+			( pszFlag = FindOptionD( pszFlag ) ) != NULL;
+			/* advance inside */ )
+		{
+			// Name is immediately after "/D" or "-D", which is taken care of by FindOptionD
+			char *pszFlagName = pszFlag ;
+			
+			// Value will be determined later
+			char *pszValue = "";
+
+			if ( char *pchEq = strchr( pszFlag, '=' ) )
+			{
+				// Value is after '=' sign
+				*pchEq = 0;
+				pszValue = pchEq + 1;
+				pszFlag = pszValue;
+			}
+
+			if ( char *pchSpace = strchr( pszFlag, ' ' ) )
+			{
+				// Space is designating the end of the flag
+				*pchSpace = 0;
+				pszFlag = pchSpace + 1;
+			}
+			else
+			{
+				// Reached end of command line
+				pszFlag = "";
+			}
+
+			// Shader model extraction; see src\devtools\bin\fxc_prep.pl for details how this is generated;
+			// for PS3, it'll be sce_vp_rsx or sce_fp_rsx
+			if ( !strncmp(pszFlagName, "SHADER_MODEL_", 13) )
+			{
+				pszShaderModel = pszFlagName + 13;
+			}
+
+			// Add the macro definition to the macros array
+			int iMacroIdx = macros.AddToTail();
+			D3DXMACRO &m = macros[iMacroIdx];
+
+			// Fill the macro data
+			m.Name = pszFlagName;
+			m.Definition = pszValue;
+		}
+
+		// Add a NULL-terminator
+		{
+			D3DXMACRO nullTerminatorMacro = { NULL, NULL };
+			macros.AddToTail( nullTerminatorMacro );
+		}
+
+		// Convert shader model to lowercase
+		char chShaderModel[20] = {0};
+		if(pszShaderModel)
+		{
+			Q_strncpy( chShaderModel, pszShaderModel, sizeof(chShaderModel) - 1 );
+		}
+		Q_strlower( chShaderModel );
+
+		// Determine the file name (at the end of the command line before redirection)
+		char const *pszFilename = "";
+		if ( const char *pchCmdRedirect = strstr( pCommand, ">output.txt " ) )
+		{
+			size_t uCmdEndOffset = ( pchCmdRedirect - pCommand );
+
+			pEditableCommand[uCmdEndOffset] = 0;
+			pszFilename = &pEditableCommand[uCmdEndOffset];
+
+			while ( pszFilename > pEditableCommand &&
+				pszFilename[-1] &&
+				' ' != pszFilename[-1] )
+			{
+				-- pszFilename;
+			}
+		}
+
+		// Compile the stuff
+		Private::FastShaderCompile( nExecutable, pszFilename, macros.Base(), chShaderModel, ppResponse );
+	}
+
+	bool TryExecuteCommand( const char *pCommand, CmdSink::IResponse **ppResponse )
+	{
+		{
+			static bool s_bNoIntercept = ( CommandLine()->FindParm("-nointercept") != 0 );
+			if ( !s_bNoIntercept )
+			{
+				const char * pExecutable = pCommand;
+				while( *pExecutable && V_isspace(*pExecutable) )
+					++pExecutable;
+				const char *pExecutableEnd = pExecutable;
+				while( *pExecutableEnd && !V_isspace(*pExecutableEnd) )
+					++pExecutableEnd;
+
+				for( int i = 0;i < EXEC_COUNT; ++i )
+				{
+					if( pExecutable == strstr( pExecutable, g_pExecutables[i] ) )
+					{
+						
+						// Trap "???.exe" so that we did not spawn extra process 
+						//Msg( "[shadercompile] Intercepting call to %s\n", g_pExecutables[i] );
+						
+						const char *pParameters = pExecutableEnd;
+						while( *pParameters && V_isspace(*pParameters) )
+							++pParameters ;
+						
+						InterceptFxc::ExecuteCommand( ( ExecutableEnum )i, pParameters, ppResponse );
+						return true;
+					}
+				}
+			}
+		}
+
+		return false;
+	}
+
+}; // namespace InterceptFxc
+
+
+
+

utils/shadercompile/d3dxfxc.h

@@ -0,0 +1,24 @@
+//====== Copyright © 1996-2006, Valve Corporation, All rights reserved. =======//
+//
+// Purpose: D3DX command implementation.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#ifndef D3DXFXC_H
+#define D3DXFXC_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "cmdsink.h"
+
+namespace InterceptFxc
+{
+
+	bool TryExecuteCommand( const char *pCommand, CmdSink::IResponse **ppResponse );
+
+}; // namespace InterceptFxc
+
+#endif // #ifndef D3DXFXC_H

utils/shadercompile/shadercompile.h

@@ -0,0 +1,11 @@
+
+//====== Copyright © 1996-2006, Valve Corporation, All rights reserved. =======//
+//
+// Purpose: Module prototypes.
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+void DebugOut( const char *pMsg, ... );
+void DebugSafeWaitPoint( bool bForceWait = false );

utils/shadercompile/shadercompile_dll.vpc

@@ -0,0 +1,64 @@
+//-----------------------------------------------------------------------------
+//	SHADERCOMPILE_DLL.VPC
+//
+//	Project Script
+//-----------------------------------------------------------------------------
+
+$Macro SRCDIR		"..\.."
+$Macro OUTBINDIR	"$SRCDIR\..\game\bin"
+
+$Include "$SRCDIR\vpc_scripts\source_dll_win32_base.vpc"
+
+$Configuration
+{
+	$Compiler
+	{
+		$AdditionalIncludeDirectories		"$BASE,..\common;..\vmpi;$SRCDIR\dx9sdk\include"
+		$PreprocessorDefinitions			"$BASE;SHADERCOMPILE_EXPORTS;MPI"
+	}
+
+	$Linker
+	{
+		$AdditionalDependencies				"$BASE ws2_32.lib odbc32.lib odbccp32.lib"
+	}
+}
+
+$Project "Shadercompile_dll"
+{
+	$Folder	"Source Files"
+	{
+		$File	"..\common\cmdlib.cpp"
+		$File	"cmdsink.cpp"
+		$File	"d3dxfxc.cpp"
+		$File	"$SRCDIR\public\filesystem_helpers.cpp"
+		$File	"..\common\pacifier.cpp"
+		$File	"shadercompile.cpp"
+		$File	"shadercompile_ps3_helpers.cpp"
+		$File	"subprocess.cpp"
+		$File	"cfgprocessor.cpp"
+		$File	"..\common\threads.cpp"
+		$File	"..\common\vmpi_tools_shared.cpp"
+		$File	"..\common\tools_minidump.cpp"
+	}
+
+	$Folder	"Header Files"
+	{
+		$File	"cmdsink.h"
+		$File	"d3dxfxc.h"
+		$File	"$SRCDIR\public\ishadercompiledll.h"
+		$File	"shadercompile.h"
+		$File	"shadercompile_ps3_helpers.h"
+		$File	"utlnodehash.h"
+		$File	"cfgprocessor.h"
+		$File	"$SRCDIR\public\tier1\UtlStringMap.h"
+		$File	"$SRCDIR\common\dx_proxy\dx_proxy.h"
+		$File	"$SRCDIR\common\dx_proxy\scecgc_proxy.h"
+	}
+
+	$Folder	"Link Libraries"
+	{
+		$File	"$SRCDIR\lib\public\tier2.lib"
+		$File	"$SRCDIR\lib\public\vmpi.lib"
+		$File	"$SRCDIR\lib\common\vc8\lzma.lib"
+	}
+}

utils/shadercompile/shadercompile_dll.vpc.vpc_cache

@@ -0,0 +1,13 @@
+"vpc_cache"
+{
+	"CacheVersion"		"1"
+	"win32"
+	{
+		"CRCFile"		"shadercompile_dll.vcxproj.vpc_crc"
+		"OutputFiles"
+		{
+			"0"		"shadercompile_dll.vcxproj"
+			"1"		"shadercompile_dll.vcxproj.filters"
+		}
+	}
+}

utils/shadercompile/shadercompile_ps3_helpers.cpp

@@ -0,0 +1,455 @@
+//============ Copyright (c) Valve Corporation, All rights reserved. ============
+//
+// Functionality to handle collation of shader debugging metadata 
+// (i.e. shader PDBs) sent from shadercompile workers to the master.
+//
+//===============================================================================
+
+#include "shadercompile_ps3_helpers.h"
+
+#include <windows.h>
+#include "utlsymbollarge.h"
+
+// TOC file is a list of all files sent from worker to master and their locations
+// in the giant 1 GB pack files
+char g_PS3DebugTOCFilename[ MAX_PATH ];
+HANDLE g_hPS3DebugTOCFile = INVALID_HANDLE_VALUE;
+
+// Keep this in the range of a 32-bit integer, preferably a signed one for simplicity
+const size_t MAX_PS3_DEBUG_INFO_PACK_FILE_SIZE = 1024 * 1024 * 1024;
+
+// Max # of pack files to generate.  If we're exceeding this number, something is probably wrong.
+const int MAX_PS3_DEBUG_INFO_PACK_FILE_COUNT = 64;
+
+// Names of each of the pack files: ps3shaderdebug_packN for N = 0 to MAX_PS3_DEBUG_INFO_PACK_FILE_COUNT-1
+char g_PS3DebugInfoPackFilenames[ MAX_PS3_DEBUG_INFO_PACK_FILE_COUNT ][ MAX_PATH ];
+
+// Handle to the currently open pack file
+HANDLE g_hPS3DebugInfoCurrentPackFile = INVALID_HANDLE_VALUE;
+// Current size of the pack files
+size_t g_nPS3DebugInfoPackFileSizes[ MAX_PS3_DEBUG_INFO_PACK_FILE_COUNT ];
+// Index into the previous arrays for the current pack file we are operating on (valid during build-up phase)
+int g_nCurrentPS3DebugInfoFile;
+
+// Defined in shadercompile.cpp
+extern const char *g_pShaderPath;
+
+// The set of all filenames we have encountered so far
+CUtlSymbolTableLarge_CI g_PS3DebugInfoFileSet;
+
+int g_nDuplicatePS3DebugFileCount = 0;
+int g_nTotalPS3DebugFileCount = 0;
+
+//-----------------------------------------------------------------------------
+// An entry in the Table-Of-Contents file which is built during compilation.
+//
+// Each entry corresponds to one tiny PS3 debug metadata file generated
+// on a worker machine during Cg shader compile and sent back to the host.
+//-----------------------------------------------------------------------------
+struct PS3DebugFileTOCEntry_t
+{
+	// Length of destination filename, including NULL terminator (data follows immediately after this structure)
+	int32 m_nFilenameLength;
+	// Which of the debug info files this entry was stored in
+	int32 m_nFileIndex;
+	// Offset in ps3shaderdebug_pack%d.bin (where %d is m_nFileIndex)
+	int32 m_nFileOffset;
+	// Length of the debug file (usually on the order of kilobytes, so 32-bits is fine)
+	int32 m_nFileSize;
+
+	// Filename immediately follows:
+	// char m_Filename[m_nFilenameLength]
+};
+
+// Appends a single tiny (few KB) file received from the worker to the end fo the current giant pack file.
+// These will be expanded later after shader compilation is done.
+static void WritePS3DebugInfo( const char *pFullPath, byte *pFileData, DWORD nFileSize )
+{
+	if ( g_nPS3DebugInfoPackFileSizes[ g_nCurrentPS3DebugInfoFile ] + nFileSize > MAX_PS3_DEBUG_INFO_PACK_FILE_SIZE )
+	{
+		// These files should be on the order of a few kilobytes, but let's just make sure nothing insane happens here
+		Assert( nFileSize < MAX_PS3_DEBUG_INFO_PACK_FILE_SIZE ); 
+		CloseHandle( g_hPS3DebugInfoCurrentPackFile );
+		g_hPS3DebugInfoCurrentPackFile = INVALID_HANDLE_VALUE;
+		++ g_nCurrentPS3DebugInfoFile;
+	}
+	if ( g_hPS3DebugInfoCurrentPackFile == INVALID_HANDLE_VALUE )
+	{
+		char filename[ MAX_PATH ];
+		Q_snprintf( filename, MAX_PATH, "ps3shaderdebug_pack%02d.bin", g_nCurrentPS3DebugInfoFile );
+		Q_ComposeFileName( g_pShaderPath, filename, g_PS3DebugInfoPackFilenames[ g_nCurrentPS3DebugInfoFile ], MAX_PATH );
+
+		g_hPS3DebugInfoCurrentPackFile = CreateFile( g_PS3DebugInfoPackFilenames[ g_nCurrentPS3DebugInfoFile ], GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL );		
+	}
+
+	if ( g_hPS3DebugInfoCurrentPackFile == INVALID_HANDLE_VALUE )
+	{
+		Error( "Could not write to PS3 debug info file.  Make sure you have enough disk space, these things can be huuuge." );
+		return;
+	}
+
+	DWORD nBytesWritten = 0;
+	WriteFile( g_hPS3DebugInfoCurrentPackFile, pFileData, nFileSize, &nBytesWritten, NULL );
+	if ( nBytesWritten != nFileSize )
+	{
+		Error( "Error writing to PS3 debug info file.  Make sure you have enough disk space, these things can be huuuge." );
+		return;
+	}
+
+	PS3DebugFileTOCEntry_t tocEntry;
+	tocEntry.m_nFilenameLength = Q_strlen( pFullPath ) + 1;
+	tocEntry.m_nFileIndex = g_nCurrentPS3DebugInfoFile;
+	tocEntry.m_nFileOffset = g_nPS3DebugInfoPackFileSizes[ g_nCurrentPS3DebugInfoFile ];
+	tocEntry.m_nFileSize = nFileSize;
+
+	g_nPS3DebugInfoPackFileSizes[ g_nCurrentPS3DebugInfoFile ] += nBytesWritten;
+
+	WriteFile( g_hPS3DebugTOCFile, &tocEntry, sizeof( tocEntry ), &nBytesWritten, NULL );
+	if ( nBytesWritten != sizeof ( tocEntry ) )
+	{
+		Error( "Error writing to PS3 debug TOC file.  Make sure you have enough disk space, these things can be huuuge." );
+		return;
+	}
+
+	WriteFile( g_hPS3DebugTOCFile, pFullPath, tocEntry.m_nFilenameLength, &nBytesWritten, NULL );
+	if ( nBytesWritten != ( DWORD )tocEntry.m_nFilenameLength )
+	{
+		Error( "Error writing to PS3 debug TOC file.  Make sure you have enough disk space, these things can be huuuge." );
+		return;
+	}
+}
+
+bool PS3ShaderDebugInfoDispatch( MessageBuffer *pBuf, int nSource, int nPacketID )
+{
+	// Received packet from worker containing list of files generated by PS3 CG compiler
+	PS3ShaderDebugInfoPacket_t *pPacket = ( PS3ShaderDebugInfoPacket_t * )pBuf->data;
+	const char *pFilename = ( char * )pBuf->data + sizeof( PS3ShaderDebugInfoPacket_t );
+
+	++g_nTotalPS3DebugFileCount;
+
+	if ( g_PS3DebugInfoFileSet.Find( pFilename ) == UTL_INVAL_SYMBOL_LARGE )
+	{
+		g_PS3DebugInfoFileSet.AddString( pFilename );
+
+		// Re-create the file locally (beneath g_pShaderPath \cgc-capture), exactly as it was on the worker machine
+		// by writing it into a giant bin file which will later be decompressed
+		if ( Q_strnicmp( pFilename, "cgc-capture", 11 ) == 0 )
+		{
+			char fullPath[MAX_PATH];
+			Q_ComposeFileName( g_pShaderPath, pFilename, fullPath, MAX_PATH );
+			WritePS3DebugInfo( fullPath, ( byte * )pBuf->data + sizeof( PS3ShaderDebugInfoPacket_t ) + pPacket->m_nFileNameLength, pPacket->m_nFileDataLength );
+		}	
+	}
+	else
+	{
+		++ g_nDuplicatePS3DebugFileCount;
+	}
+
+	return true;
+}
+
+void InitializePS3ShaderDebugPackFiles()
+{
+	// Create files for debug information being returned from workers
+	Q_ComposeFileName( g_pShaderPath, "ps3shaderdebug_toc.bin", g_PS3DebugTOCFilename, MAX_PATH );
+	g_hPS3DebugTOCFile = CreateFile( g_PS3DebugTOCFilename, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL );
+}
+
+static void DisplayFileUnpackProgress( unsigned int nTotalSize, unsigned int nLast, unsigned int nCurrent )
+{
+	int nLastProgress = ( int )( 100.0f * ( double )nLast / ( double )nTotalSize );
+	int nCurrentProgress = ( int )( 100.0f * ( double )nCurrent / ( double )nTotalSize );
+	for ( int i = nLastProgress + 1; i <= nCurrentProgress; ++ i )
+	{
+		if ( i % 10 == 0 )
+		{
+			Msg( "%d", ( i / 10 ) );
+		}
+		else if ( i % 2 == 0 )
+		{
+			Msg( "." );
+		}
+	}
+}
+
+// Expand the giant ps3shaderdebug_toc.bin and ps3shaderdebug_packN.bin files into a directory tree of little files.
+// Writing out these files takes too long to do in-line with the shader compile (the master gets bogged down with file IO requests otherwise)
+void ExpandPS3DebugInfo()
+{
+	if ( g_hPS3DebugTOCFile != INVALID_HANDLE_VALUE )
+	{
+		Msg( "Unpacking giant shader debug info files into sub-directory tree.\n" );
+		Msg( "0" );
+
+		// Close the last files we were working on
+		CloseHandle( g_hPS3DebugTOCFile );
+		if ( g_hPS3DebugInfoCurrentPackFile != INVALID_HANDLE_VALUE )
+		{
+			CloseHandle( g_hPS3DebugInfoCurrentPackFile );
+		}
+
+		g_hPS3DebugTOCFile = CreateFile( g_PS3DebugTOCFilename, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL );
+		DWORD nTocFileSizeHigh;
+		DWORD nTocFileSize = GetFileSize( g_hPS3DebugTOCFile, &nTocFileSizeHigh );
+		if ( nTocFileSizeHigh != 0 )
+		{
+			Error( "PS3 debug info TOC File is greater than 4 GB.  This is probably not a good thing." );
+			return;
+		}
+
+		// A set of directories we have already created, so we know not to re-create them
+		CUtlSymbolTableLarge_CI createdPS3DebugInfoDirectories;
+
+		// Scratch space big enough to store any single sub-file, usually on the order of kilobytes
+		CUtlVector< byte > scratchSpace;
+
+		g_hPS3DebugInfoCurrentPackFile = INVALID_HANDLE_VALUE;
+
+		DWORD nCurrentTocEntryOffset = 0;
+		DWORD nCurrentDebugInfoOffset = 0;
+		int nCurrentDebugInfoFile = -1;
+		while ( nCurrentTocEntryOffset < nTocFileSize )
+		{
+			// There must be at least enough room for a TOC entry plus some string data afterwards
+			Assert( nCurrentTocEntryOffset + sizeof( PS3DebugFileTOCEntry_t ) < nTocFileSize );
+
+			DWORD nBytesRead = 0;
+			PS3DebugFileTOCEntry_t tocEntry;
+			ReadFile( g_hPS3DebugTOCFile, &tocEntry, sizeof( PS3DebugFileTOCEntry_t ), &nBytesRead, NULL );
+			Assert( nBytesRead == sizeof( PS3DebugFileTOCEntry_t ) );
+
+			char fileNameBuffer[ MAX_PATH ];
+			Assert( tocEntry.m_nFilenameLength < MAX_PATH );
+			ReadFile( g_hPS3DebugTOCFile, fileNameBuffer, MIN( tocEntry.m_nFilenameLength, MAX_PATH ), &nBytesRead, NULL );
+			Assert( nBytesRead == (DWORD)tocEntry.m_nFilenameLength );
+
+			// Create any necessary directories recursively
+			char dirToCreate[MAX_PATH];
+			Q_ExtractFilePath( fileNameBuffer, dirToCreate, MAX_PATH );
+			Q_StripTrailingSlash( dirToCreate );
+
+			if ( ( UtlSymLargeId_t )createdPS3DebugInfoDirectories.Find( dirToCreate ) == UTL_INVAL_SYMBOL_LARGE )
+			{
+				const char *pNextDir = strchr( fileNameBuffer, '\\' );
+				while ( pNextDir != NULL )
+				{
+					size_t nCharsToCopy = pNextDir - fileNameBuffer;
+					memcpy( dirToCreate, fileNameBuffer, nCharsToCopy );
+					dirToCreate[nCharsToCopy] = '\0';
+					if ( ( UtlSymLargeId_t )createdPS3DebugInfoDirectories.Find( dirToCreate ) == UTL_INVAL_SYMBOL_LARGE )
+					{
+						CreateDirectory( dirToCreate, NULL );
+						createdPS3DebugInfoDirectories.AddString( dirToCreate );
+					}
+					pNextDir = strchr( pNextDir + 1, '\\' );
+				}
+			}
+
+			// Read the data out of the corresponding debug info file
+			if ( nCurrentDebugInfoFile != tocEntry.m_nFileIndex )
+			{
+				if ( g_hPS3DebugInfoCurrentPackFile != INVALID_HANDLE_VALUE )
+				{
+					Assert( nCurrentDebugInfoOffset == g_nPS3DebugInfoPackFileSizes[ nCurrentDebugInfoFile ] );
+					CloseHandle( g_hPS3DebugInfoCurrentPackFile );
+					DeleteFile( g_PS3DebugInfoPackFilenames[ nCurrentDebugInfoFile ] );
+				}
+				nCurrentDebugInfoOffset = 0;
+				nCurrentDebugInfoFile = tocEntry.m_nFileIndex;
+				g_hPS3DebugInfoCurrentPackFile = CreateFile( g_PS3DebugInfoPackFilenames[ nCurrentDebugInfoFile ], GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL );
+			}
+
+			Assert( nCurrentDebugInfoOffset == (DWORD)tocEntry.m_nFileOffset );
+			scratchSpace.EnsureCount( tocEntry.m_nFileSize );
+			ReadFile( g_hPS3DebugInfoCurrentPackFile, scratchSpace.Base(), tocEntry.m_nFileSize, &nBytesRead, NULL );
+			Assert( nBytesRead == (DWORD)tocEntry.m_nFileSize );
+			nCurrentDebugInfoOffset += nBytesRead;
+
+			DWORD nBytesWritten = 0;
+			HANDLE hNewFile = CreateFile( fileNameBuffer, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL );
+			if ( hNewFile == INVALID_HANDLE_VALUE )
+			{
+				Error( "Unable to create PS3 shader debug info file: %s.  Ensure you have enoug disk space.\n", fileNameBuffer );
+			}
+			WriteFile( hNewFile, scratchSpace.Base(), tocEntry.m_nFileSize, &nBytesWritten, NULL );
+			Assert( nBytesWritten == (DWORD)tocEntry.m_nFileSize );
+			CloseHandle( hNewFile );
+
+			DWORD nEntrySize = sizeof( PS3DebugFileTOCEntry_t ) + tocEntry.m_nFilenameLength;
+			DisplayFileUnpackProgress( nTocFileSize, nCurrentTocEntryOffset, nCurrentTocEntryOffset + nEntrySize );
+			nCurrentTocEntryOffset += nEntrySize;
+		}
+
+		if ( g_hPS3DebugInfoCurrentPackFile != INVALID_HANDLE_VALUE )
+		{
+			CloseHandle( g_hPS3DebugInfoCurrentPackFile );
+			DeleteFile( g_PS3DebugInfoPackFilenames[ nCurrentDebugInfoFile ] );
+		}
+
+		CloseHandle( g_hPS3DebugTOCFile );
+		DeleteFile( g_PS3DebugTOCFilename );
+	}
+	Msg( "\nTotal shader debug files returned: %d, duplicates: %d\n", g_nTotalPS3DebugFileCount, g_nDuplicatePS3DebugFileCount );
+}
+
+
+static void SendFileContentsToMaster( const char *pFilename )
+{
+	HANDLE fileHandle = CreateFile( pFilename, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL );
+	if ( fileHandle != INVALID_HANDLE_VALUE )
+	{
+		DWORD fileSize = GetFileSize( fileHandle, NULL );
+		byte *pFileData = new byte[fileSize];
+		DWORD bytesRead;
+		ReadFile( fileHandle, pFileData, fileSize, &bytesRead, NULL );
+		if ( bytesRead == fileSize )
+		{
+			PS3ShaderDebugInfoPacket_t filePacket;
+			filePacket.m_PacketID = PS3_SHADER_DEBUG_INFO_PACKETID;
+			filePacket.m_nFileNameLength = Q_strlen( pFilename ) + 1;
+			filePacket.m_nFileDataLength = bytesRead;
+			CUtlBuffer myBuffer;
+			myBuffer.Put( &filePacket, sizeof( filePacket ) );
+			myBuffer.Put( pFilename, filePacket.m_nFileNameLength );
+			myBuffer.Put( pFileData, filePacket.m_nFileDataLength );
+			VMPI_SendData( myBuffer.Base(), sizeof( filePacket ) + filePacket.m_nFileNameLength + filePacket.m_nFileDataLength, VMPI_MASTER_ID );
+		}
+
+		delete[] pFileData;
+		CloseHandle( fileHandle );
+		unlink( pFilename );
+	}
+
+	fopen( pFilename, "r" );
+}
+
+template< typename TFunctor >
+static void ForEachFileRecursive( const char *pStartingPath, TFunctor callbackFunction )
+{
+	WIN32_FIND_DATA findFileData;
+	char searchPath[MAX_PATH];
+	Q_strncpy( searchPath, pStartingPath, MAX_PATH );
+	Q_ComposeFileName( pStartingPath, "*.*", searchPath, MAX_PATH );
+	HANDLE findHandle = FindFirstFile( searchPath, &findFileData );
+	bool bKeepSearching = ( findHandle != INVALID_HANDLE_VALUE );
+	while ( bKeepSearching )
+	{
+		if ( Q_stricmp( findFileData.cFileName, "." ) != 0 && Q_stricmp( findFileData.cFileName, ".." ) != 0 )
+		{
+			char fullFilePath[MAX_PATH];
+			Q_ComposeFileName( pStartingPath, findFileData.cFileName, fullFilePath, MAX_PATH );
+			printf( "Found file: %s\n\n", fullFilePath );
+			if ( ( findFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) != 0 )
+			{
+				ForEachFileRecursive( fullFilePath, callbackFunction );
+			}
+			else
+			{
+				callbackFunction( fullFilePath );
+			}
+		}
+
+		bKeepSearching = !!FindNextFile( findHandle, &findFileData );
+	}
+	FindClose( findHandle );
+}
+
+void SendSubDirectoryToMaster( const char *pStartingPath )
+{
+	ForEachFileRecursive( pStartingPath, SendFileContentsToMaster );
+}
+
+static bool SendShaderCompileLogContentsToMaster( const char *pFilename )
+{
+	FILE *pFile = fopen( pFilename, "r" );
+	if ( !pFile )
+		return false;
+	
+	const uint nPacketBufSize = 8192;
+	char packetBuf[nPacketBufSize];
+	
+	PS3ShaderCompileLogPacket_t &filePacket = *reinterpret_cast< PS3ShaderCompileLogPacket_t * >( &packetBuf );
+	filePacket.m_PacketID = PS3_SHADER_COMPILE_LOG_PACKETID;
+	filePacket.m_nPacketSize = 0;
+	uint nCurBufSize = sizeof( filePacket );
+	
+	while ( !feof( pFile ) )
+	{
+		const uint nMaxLineSize = 512;
+
+		char szLine[nMaxLineSize];
+		if ( !fgets( szLine, nMaxLineSize, pFile ) )
+			break;
+
+		int nCurLineSize = V_strlen( szLine );
+
+		V_memcpy( packetBuf + nCurBufSize, szLine, nCurLineSize );
+		nCurBufSize += nCurLineSize;
+		Assert( nCurBufSize <= nPacketBufSize );
+
+		if ( ( nPacketBufSize - nCurBufSize ) < nMaxLineSize )
+		{
+			filePacket.m_nPacketSize = nCurBufSize;
+
+			VMPI_SendData( &filePacket, nCurBufSize, VMPI_MASTER_ID );
+			nCurBufSize = sizeof( filePacket );
+		}
+	}
+
+	if ( nCurBufSize > sizeof( filePacket ) )
+	{
+		filePacket.m_nPacketSize = nCurBufSize;
+		VMPI_SendData( &filePacket, nCurBufSize, VMPI_MASTER_ID );
+	}
+
+	fclose( pFile );
+
+	return true;
+}
+
+void PS3SendShaderCompileLogContentsToMaster()
+{
+	char szLogFilename[MAX_PATH];
+	if ( GetEnvironmentVariableA( "PS3COMPILELOG", szLogFilename, sizeof( szLogFilename ) ) )
+	{
+		HANDLE hMutex = CreateMutex( NULL, FALSE, "PS3COMPILELOGMUTEX" );
+		if ( ( hMutex ) && ( WaitForSingleObject( hMutex, 10000 ) == WAIT_OBJECT_0 ) )
+		{
+			SendShaderCompileLogContentsToMaster( szLogFilename );
+
+			_unlink( szLogFilename );
+
+			ReleaseMutex( hMutex );
+		}
+	}
+}
+
+bool PS3ShaderCompileLogDispatch( MessageBuffer *pBuf, int nSource, int nPacketID )
+{
+	if ( pBuf->getLen() < sizeof( PS3ShaderCompileLogPacket_t ) )
+		return false;
+
+	PS3ShaderCompileLogPacket_t *pPacket = ( PS3ShaderCompileLogPacket_t * )pBuf->data;
+	const uint8 *pData = ( const uint8 * )pBuf->data + sizeof( PS3ShaderCompileLogPacket_t );
+	int nDataSize = pPacket->m_nPacketSize - sizeof( PS3ShaderCompileLogPacket_t );
+
+	if ( ( pPacket->m_nPacketSize >= sizeof( PS3ShaderCompileLogPacket_t ) ) && ( pBuf->getLen() >= pPacket->m_nPacketSize ) )
+	{
+		FILE *pFile = fopen( "ps3compilelog.txt", "ab" );
+		if ( pFile )
+		{
+			fwrite( pData, nDataSize, 1, pFile );
+			
+			fclose( pFile );
+		}
+	}
+
+	return true;
+}
+
+
+
+
+
+
+

utils/shadercompile/shadercompile_ps3_helpers.h

@@ -0,0 +1,75 @@
+//============ Copyright (c) Valve Corporation, All rights reserved. ============
+//
+// Functionality to handle collation of shader debugging metadata 
+// (i.e. shader PDBs) sent from shadercompile workers to the master.
+//
+//===============================================================================
+
+#pragma once
+
+#ifndef SHADERCOMPILE_PS3_HELPERS_H
+#define SHADERCOMPILE_PS3_HELPERS_H
+
+#include "vmpi.h"
+
+//-----------------------------------------------------------------------------
+// The ID of the VMPI packet sent from workers to the master for each
+// small PS3 shader debug metadata file generated by the Sony/Cg compiler.
+//-----------------------------------------------------------------------------
+#define PS3_SHADER_DEBUG_INFO_PACKETID 6
+#define PS3_SHADER_COMPILE_LOG_PACKETID 11
+
+//-----------------------------------------------------------------------------
+// Structure of the packet corresponding to PS3_SHADER_DEBUG_INFO_PACKETID
+//-----------------------------------------------------------------------------
+struct PS3ShaderDebugInfoPacket_t
+{
+	char m_PacketID; // This should be PS3_SHADER_DEBUG_INFO_PACKETID
+	int m_nFileNameLength;
+	int m_nFileDataLength;
+	// Followed by m_nFileNameLength + m_nFileDataLength bytes of data
+};
+
+//-----------------------------------------------------------------------------
+// Handler for VMPI packet: PS3_SHADER_DEBUG_INFO_PACKETID.
+// Called on the master when such a packet is received.
+//-----------------------------------------------------------------------------
+bool PS3ShaderDebugInfoDispatch( MessageBuffer *pBuf, int nSource, int nPacketID );
+
+//-----------------------------------------------------------------------------
+// Initializes the TOC and Pack files used to store the large amount
+// of shader metadata generated by the worker machines.
+// Called on the master.
+//-----------------------------------------------------------------------------
+void InitializePS3ShaderDebugPackFiles();
+
+//-----------------------------------------------------------------------------
+// Expand PS3 debug information that has been accumulated into giant pack
+// files on the master's machine.
+// These giant files expand into hundreds of thousands of tiny files
+// required for shader debugging.
+// Called on the master.
+//-----------------------------------------------------------------------------
+void ExpandPS3DebugInfo();
+
+//-----------------------------------------------------------------------------
+// Sends all files and non-empty directories to the VMPI master, one packet
+// per file (see PS3_SHADER_DEBUG_INFO_PACKETID). Called by workers.
+//-----------------------------------------------------------------------------
+void SendSubDirectoryToMaster( const char *pStartingPath );
+
+//-----------------------------------------------------------------------------
+// Shader compile statistics log file
+//-----------------------------------------------------------------------------
+struct PS3ShaderCompileLogPacket_t
+{
+	char m_PacketID; // This should be PS3_SHADER_COMPILE_LOG_PACKETID
+	int m_nPacketSize;
+	// Followed by m_nPacketSize bytes of data
+};
+
+void PS3SendShaderCompileLogContentsToMaster();
+
+bool PS3ShaderCompileLogDispatch( MessageBuffer *pBuf, int nSource, int nPacketID );
+
+#endif // SHADERCOMPILE_PS3_HELPERS_H

utils/shadercompile/subprocess.cpp

@@ -0,0 +1,314 @@
+//====== Copyright c 1996-2007, Valve Corporation, All rights reserved. =======//
+//
+// Purpose: 
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#include <windows.h>
+
+#include "cmdsink.h"
+
+#include "subprocess.h"
+
+#include "d3dxfxc.h"
+
+#include "tools_minidump.h"
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Base implementation of the shaderd kernel objects
+//
+//////////////////////////////////////////////////////////////////////////
+
+SubProcessKernelObjects::SubProcessKernelObjects( void ) :
+	m_hMemorySection( NULL ),
+	m_hMutex( NULL )
+{
+	ZeroMemory( m_hEvent, sizeof( m_hEvent ) );
+}
+
+SubProcessKernelObjects::~SubProcessKernelObjects( void )
+{
+	Close();
+}
+
+BOOL SubProcessKernelObjects::Create( char const *szBaseName )
+{
+	char chBufferName[0x100] = { 0 };
+	
+	sprintf( chBufferName, "%s_msec", szBaseName );
+	m_hMemorySection = CreateFileMapping( INVALID_HANDLE_VALUE, NULL,
+		PAGE_READWRITE, 0, 4 * 1024 * 1024, chBufferName ); // 4Mb for a piece
+	if ( NULL != m_hMemorySection )
+	{
+		if ( ERROR_ALREADY_EXISTS == GetLastError() )
+		{
+			CloseHandle( m_hMemorySection );
+			m_hMemorySection = NULL;
+
+			Assert( 0 && "CreateFileMapping - already exists!\n" );
+		}
+	}
+
+	sprintf( chBufferName, "%s_mtx", szBaseName );
+	m_hMutex = CreateMutex( NULL, FALSE, chBufferName );
+
+	for ( int k = 0; k < 2; ++ k )
+	{
+		sprintf( chBufferName, "%s_evt%d", szBaseName, k );
+		m_hEvent[k] = CreateEvent( NULL, FALSE, ( k ? TRUE /* = master */ : FALSE ), chBufferName );
+	}
+
+	return IsValid();
+}
+
+BOOL SubProcessKernelObjects::Open( char const *szBaseName )
+{
+	char chBufferName[0x100] = { 0 };
+
+	sprintf( chBufferName, "%s_msec", szBaseName );
+	m_hMemorySection = OpenFileMapping( FILE_MAP_ALL_ACCESS, FALSE, chBufferName );
+
+	sprintf( chBufferName, "%s_mtx", szBaseName );
+	m_hMutex = OpenMutex( MUTEX_ALL_ACCESS, FALSE, chBufferName );
+
+	for ( int k = 0; k < 2; ++ k )
+	{
+		sprintf( chBufferName, "%s_evt%d", szBaseName, k );
+		m_hEvent[k] = OpenEvent( EVENT_ALL_ACCESS, FALSE, chBufferName );
+	}
+
+	return IsValid();
+}
+
+BOOL SubProcessKernelObjects::IsValid( void ) const
+{
+	return m_hMemorySection && m_hMutex && m_hEvent;
+}
+
+void SubProcessKernelObjects::Close( void )
+{
+	if ( m_hMemorySection )
+		CloseHandle( m_hMemorySection );
+
+	if ( m_hMutex )
+		CloseHandle( m_hMutex );
+
+	for ( int k = 0; k < 2; ++ k )
+		if ( m_hEvent[k] )
+			CloseHandle( m_hEvent[k] );
+}
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Helper class to send data back and forth
+//
+//////////////////////////////////////////////////////////////////////////
+
+void * SubProcessKernelObjects_Memory::Lock( void )
+{
+	// Wait for our turn to act
+	for ( unsigned iWaitAttempt = 0; iWaitAttempt < 13u; ++ iWaitAttempt )
+	{
+		DWORD dwWait = ::WaitForSingleObject( m_pObjs->m_hEvent[ m_pObjs->m_dwCookie ], 10000 );
+		switch ( dwWait )
+		{
+		case WAIT_OBJECT_0:
+			{
+				m_pLockData = MapViewOfFile( m_pObjs->m_hMemorySection, FILE_MAP_ALL_ACCESS, 0, 0, 0 );
+				if ( !m_pLockData )
+				{
+					DWORD err = GetLastError();
+					Msg( "MapViewOfFile failed with error %d\n", err );
+				}
+
+				if ( m_pLockData && * ( const DWORD * ) m_pLockData != m_pObjs->m_dwCookie )
+				{
+					// Yes, this is our turn, set our cookie in that memory segment
+					* ( DWORD * ) m_pLockData = m_pObjs->m_dwCookie;
+					m_pMemory = ( ( byte * ) m_pLockData ) + 2 * sizeof( DWORD );
+					
+					return m_pMemory;
+				}
+				else
+				{
+					// We just acted, still waiting for result
+					UnmapViewOfFile( m_pLockData );
+					m_pLockData = NULL;
+					
+					SetEvent( m_pObjs->m_hEvent[ !m_pObjs->m_dwCookie ] );
+					Sleep( 1 );
+					
+					continue;
+				}
+			}
+			break;
+		
+		case WAIT_TIMEOUT:
+			{
+				char chMsg[0x100];
+				sprintf( chMsg, "th%08X> WAIT_TIMEOUT in Memory::Lock (attempt %d).\n", GetCurrentThreadId(), iWaitAttempt );
+				OutputDebugString( chMsg );
+			}
+			continue; // retry
+
+		default:
+			OutputDebugString( "WAIT failure in Memory::Lock\n" );
+			SetLastError( ERROR_BAD_UNIT );
+			return NULL;
+		}
+	}
+	
+	OutputDebugString( "Ran out of wait attempts in Memory::Lock\n" );
+	SetLastError( ERROR_NOT_READY );
+	return NULL;
+}
+
+BOOL SubProcessKernelObjects_Memory::Unlock( void )
+{
+	if ( m_pLockData )
+	{
+		// Assert that the memory hasn't been spoiled
+		Assert( m_pObjs->m_dwCookie == * ( const DWORD * ) m_pLockData );
+		
+		DWORD ret = UnmapViewOfFile( m_pLockData );
+		if ( ret == 0 )
+		{
+			DWORD err = GetLastError();
+			Msg( "UnmapViewOfFile failed with error %d\n", err );
+		}
+
+		m_pMemory = NULL;
+		m_pLockData = NULL;
+		
+		SetEvent( m_pObjs->m_hEvent[ !m_pObjs->m_dwCookie ] );
+		Sleep( 1 );
+
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+
+//////////////////////////////////////////////////////////////////////////
+//
+// Implementation of the command subprocess:
+//
+// MASTER      ---- command ------->     SUB
+//				string		-	zero terminated command string.
+//
+//
+// MASTER     <---- result --------      SUB
+//				dword		-	1 if succeeded, 0 if failed
+//				dword		-	result buffer length, 0 if failed
+//				<bytes>		-	result buffer data, none if result buffer length is 0
+//				string		-	zero-terminated listing string
+//
+//////////////////////////////////////////////////////////////////////////
+
+
+CSubProcessResponse::CSubProcessResponse( void const *pvMemory ) :
+	m_pvMemory( pvMemory )
+{
+	byte const *pBytes = ( byte const * ) pvMemory;
+	
+	m_dwResult = * ( DWORD const * ) pBytes;
+	pBytes += sizeof( DWORD );
+
+	m_dwResultBufferLength = * ( DWORD const * ) pBytes;
+	pBytes += sizeof( DWORD );
+
+	m_pvResultBuffer = pBytes;
+	pBytes += m_dwResultBufferLength;
+
+	m_szListing = ( char const * ) ( *pBytes ? pBytes : NULL );
+}
+
+
+void ShaderCompile_Subprocess_ExceptionHandler( unsigned long exceptionCode, void *pvExceptionInfo )
+{
+	// Subprocesses just silently die in our case, then this case will be detected by the worker process and an error code will be passed to the master
+	Assert( !"ShaderCompile_Subprocess_ExceptionHandler" );
+	::TerminateProcess( ::GetCurrentProcess(), exceptionCode );
+}
+
+
+int ShaderCompile_Subprocess_Main( char const *szSubProcessData )
+{
+	// Set our crash handler
+	SetupToolsMinidumpHandler( ShaderCompile_Subprocess_ExceptionHandler );
+
+	// Get our kernel objects
+	SubProcessKernelObjects_Open objs( szSubProcessData );
+
+	if ( !objs.IsValid() )
+		return -1;
+
+	// Enter the command pumping loop
+	SubProcessKernelObjects_Memory shrmem( &objs );
+	for (
+		void *pvMemory = NULL;
+		NULL != ( pvMemory = shrmem.Lock() );
+		shrmem.Unlock()
+		)
+	{
+		// The memory is actually a command
+		char const *szCommand = ( char const * ) pvMemory;
+		
+		if ( !stricmp( "keepalive", szCommand ) )
+		{
+			ZeroMemory( pvMemory, 4 * sizeof( DWORD ) );
+			continue;
+		}
+
+		if ( !stricmp( "quit", szCommand ) )
+		{
+			ZeroMemory( pvMemory, 4 * sizeof( DWORD ) );
+			return 0;
+		}
+
+		CmdSink::IResponse *pResponse = NULL;
+		if ( InterceptFxc::TryExecuteCommand( szCommand, &pResponse ) )
+		{
+			byte *pBytes = ( byte * ) pvMemory;
+			
+			// Result
+			DWORD dwSucceededResult = pResponse->Succeeded() ? 1 : 0;
+			* ( DWORD * ) pBytes = dwSucceededResult;
+			pBytes += sizeof( DWORD );
+
+			// Result buffer len
+			DWORD dwBufferLength = pResponse->GetResultBufferLen();
+			* ( DWORD * ) pBytes = dwBufferLength;
+			pBytes += sizeof( DWORD );
+
+			// Result buffer
+			const void *pvResultBuffer = pResponse->GetResultBuffer();
+			memcpy( pBytes, pvResultBuffer, dwBufferLength );
+			pBytes += dwBufferLength;
+
+			// Listing - copy string
+			const char *szListing = pResponse->GetListing();
+			if ( szListing )
+			{
+				while ( 0 != ( * ( pBytes ++ ) = * ( szListing ++ ) ) )
+				{
+					NULL;
+				}
+			}
+			else
+			{
+				* ( pBytes ++ ) = 0;
+			}
+		}
+		else
+		{
+			ZeroMemory( pvMemory, 4 * sizeof( DWORD ) );
+		}
+	}
+
+	return -2;
+}

utils/shadercompile/subprocess.h

@@ -0,0 +1,104 @@
+//====== Copyright c 1996-2007, Valve Corporation, All rights reserved. =======//
+//
+// Purpose: 
+//
+// $NoKeywords: $
+//
+//=============================================================================//
+
+#ifndef SUBPROCESS_H
+#define SUBPROCESS_H
+#ifdef _WIN32
+#pragma once
+#endif
+
+class SubProcessKernelObjects
+{
+	friend class SubProcessKernelObjects_Memory;
+
+public:
+	SubProcessKernelObjects( void );
+	~SubProcessKernelObjects( void );
+
+private:
+	SubProcessKernelObjects( SubProcessKernelObjects const & );
+	SubProcessKernelObjects & operator =( SubProcessKernelObjects const & );
+
+protected:
+	BOOL Create( char const *szBaseName );
+	BOOL Open( char const *szBaseName );
+
+public:
+	BOOL IsValid( void ) const;
+	void Close( void );
+
+protected:
+	HANDLE m_hMemorySection;
+	HANDLE m_hMutex;
+	HANDLE m_hEvent[2];
+	DWORD m_dwCookie;
+};
+
+class SubProcessKernelObjects_Create : public SubProcessKernelObjects
+{
+public:
+	SubProcessKernelObjects_Create( char const *szBaseName ) { Create( szBaseName ), m_dwCookie = 1; }
+};
+
+class SubProcessKernelObjects_Open : public SubProcessKernelObjects
+{
+public:
+	SubProcessKernelObjects_Open( char const *szBaseName ) { Open( szBaseName ), m_dwCookie = 0; }
+};
+
+class SubProcessKernelObjects_Memory
+{
+public:
+	SubProcessKernelObjects_Memory( SubProcessKernelObjects *p ) : m_pObjs( p ), m_pLockData( NULL ), m_pMemory( NULL ) {  }
+	~SubProcessKernelObjects_Memory() { Unlock(); }
+
+public:
+	void * Lock( void );
+	BOOL Unlock( void );
+
+public:
+	BOOL IsValid( void ) const { return m_pLockData != NULL; }
+	void * GetMemory( void ) const { return m_pMemory; }
+
+protected:
+	void *m_pMemory;
+
+private:
+	SubProcessKernelObjects *m_pObjs;
+	void *m_pLockData;
+};
+
+
+//
+// Response implementation
+//
+class CSubProcessResponse : public CmdSink::IResponse
+{
+public:
+	explicit CSubProcessResponse( void const *pvMemory );
+	~CSubProcessResponse( void ) { }
+
+public:
+	virtual bool Succeeded( void ) { return ( 1 == m_dwResult ); }
+	virtual size_t GetResultBufferLen( void ) { return ( Succeeded() ? m_dwResultBufferLength : 0 ); }
+	virtual const void * GetResultBuffer( void ) { return ( Succeeded() ? m_pvResultBuffer : NULL ); }
+	virtual const char * GetListing( void ) { return (const char *) ( ( m_szListing && * m_szListing ) ? m_szListing : NULL ); }
+
+protected:
+	void const *m_pvMemory;
+	DWORD m_dwResult;
+	DWORD m_dwResultBufferLength;
+	void const *m_pvResultBuffer;
+	char const *m_szListing;
+};
+
+
+int ShaderCompile_Subprocess_Main( char const *szSubProcessData );
+
+
+#endif // #ifndef SUBPROCESS_H

utils/shadercompile/utlnodehash.h

@@ -0,0 +1,93 @@
+//====== Copyright © 1996-2005, Valve Corporation, All rights reserved. =======//
+//
+// Purpose: hashed intrusive linked list.
+//
+// $NoKeywords: $
+//
+// Serialization/unserialization buffer
+//=============================================================================//
+
+#ifndef UTLNODEHASH_H
+#define UTLNODEHASH_H
+
+#ifdef _WIN32
+#pragma once
+#endif
+
+#include "tier1/utlmemory.h"
+#include "tier1/byteswap.h"
+#include "tier1/utlintrusivelist.h"
+
+#include <stdarg.h>
+
+// to use this class, your list node class must have a Key() function defined which returns an
+// integer type. May add this class to main utl tier when i'm happy w/ it.
+template<class T, int HASHSIZE = 7907, class K = int > class CUtlNodeHash
+{
+
+	int m_nNumNodes;
+
+public:
+
+	CUtlIntrusiveDList<T> m_HashChains[HASHSIZE];
+
+	CUtlNodeHash( void )
+	{
+		m_nNumNodes = 0;
+	}
+
+
+	T *FindByKey(K nMatchKey, int *pChainNumber = NULL)
+	{
+		unsigned int nChain=(unsigned int) nMatchKey ;
+		nChain %= HASHSIZE;
+		if ( pChainNumber )
+			*( pChainNumber ) = nChain;
+		for( T * pNode = m_HashChains[ nChain ].m_pHead; pNode; pNode = pNode->m_pNext )
+			if ( pNode->Key() == nMatchKey )
+				return pNode;
+		return NULL;
+	}
+
+	void Add( T * pNode )
+	{
+		unsigned int nChain=(unsigned int) pNode->Key();
+		nChain %= HASHSIZE;
+		m_HashChains[ nChain ].AddToHead( pNode );
+		m_nNumNodes++;
+	}
+
+
+	void Purge( void )
+	{
+		m_nNumNodes = 0;
+		// delete all nodes
+		for( int i=0; i < HASHSIZE; i++)
+			m_HashChains[i].Purge();
+	}
+
+	int Count( void ) const
+	{
+		return m_nNumNodes;
+	}
+
+	void DeleteByKey( K nMatchKey )
+	{
+		int nChain;
+		T *pSearch = FindByKey( nMatchKey, &nChain );
+		if ( pSearch )
+		{
+			m_HashChains[ nChain ].RemoveNode( pSearch );
+			m_nNumNodes--;
+		}
+	}
+
+	~CUtlNodeHash( void )
+	{
+		// delete all lists
+		Purge();
+	}
+};
+
+
+#endif