Subclass External Programs done for you

[moeur]

The above code is perhaps a little daunting so here is a simplified version for illustrative purposes.
We'll create a project that hooks posted messages and allows the VB user to change those messages before they are passed on to the original process.
First create the C++ dll. It will have three routines accessible to VB, so create your .def file

//HookDemo.def
LIBRARY MainHook
EXPORTS
	InstallFilterDLL	@1
	UnInstallFilterDLL	@2
	SetSharedData		@5

In your source file, do some basic steup

//HookDemo.cpp
#include <windows.h>
#include "WINUSER.H"    
/*---------------------------------------------
          Shared Variables
This data is shared between both prcesses.  The
VB App has access to these variables through the
function SetSharedData
---------------------------------------------*/
#pragma data_seg(".shared")
	bool ChangeMessage = false;
	int Shared_uMsg = 0;
	int Shared_wParam = 0;
	int Shared_lParam = 0;
	HWND hWndVB = 0; //handle to subclassed VB Window
	HWND hWndCtrl = 0;//handle to the window we want to monitor
#pragma data_seg()
#pragma comment(linker, "/SECTION:.shared,RWS")

//---------------------------------------------
// Global Variables, specific to each process
//---------------------------------------------
	HHOOK	hmsgHooks = 0;		// Hook handle for WH_GETMESSAGE
	HINSTANCE	hInstance;	// Global instance handle for DLL

//--------------------------------------------
//        DLL entry-point 
//--------------------------------------------
BOOL WINAPI DllMain(
    HINSTANCE hinstDLL,  // handle to DLL module
    DWORD fdwReason,     // reason for calling function
    LPVOID lpvReserved )  // reserved
{
    // Perform actions based on the reason for calling.
    switch( fdwReason ) 
    { 
        case DLL_PROCESS_ATTACH:
         // Initialize once for each new process.
         // Return FALSE to fail DLL load.
			hInstance = hinstDLL;//save dll handle for each process
            break;
	
        case DLL_THREAD_ATTACH:
         // Do thread-specific initialization.
            break;

        case DLL_THREAD_DETACH:
         // Do thread-specific cleanup.
            break;

        case DLL_PROCESS_DETACH:
         // Perform any necessary cleanup.
            break;
    }
    return TRUE;  // Successful DLL_PROCESS_ATTACH.
}

OK, Now let's create those three routines
First a routine for setting the hook

/*  set WH_GETMESSAGE hook
  Private Declare Function InstallFilterDLL Lib "C:\bin\HookDemo.dll" ( _
    ByVal dwThreadID As Long, _
	ByVal ExtrnHandle As Long, _
    ByVal VBHandle As Long _
) As Long
*/
__declspec(dllexport) int _stdcall InstallFilterDLL(DWORD dwThreadId, HWND ExtrnHandle, HWND VBhandle)
{
	if (hmsgHooks==0)
			hWndVB = VBhandle;\\save handle to subclassed VB Window
			hWndCtrl = ExtrnHandle;//Handle to external window
			hmsgHooks = SetWindowsHookEx(WH_GETMESSAGE,(HOOKPROC) GetMsgProc, (HINSTANCE) hInstance, dwThreadId);
		if (hmsgHooks==0) return GetLastError();
	return 0;
}

Next a routine for removing the hook

/*  Remove the WH_GETMESSAGE hook
  Private Declare Function UnInstallFilterDLL Lib "C:\bin\HookDemo.dll" () As Long
*/
 __declspec(dllexport) int UnInstallFilterDLL(void)
{
	LRESULT result;
	if (hmsgHooks != 0){
		result = UnhookWindowsHookEx(hmsgHooks);
		if (result == 0) return GetLastError();
		hmsgHooks = 0;
	}
	return 0;
}

And a routine so that VB can change the message data

/* SetSharedData Allows the VB App to alter the shared data
  Private Declare Sub SetSharedData Lib "C:\bin\HookDemo.dll" ( _
    ByVal uMsg As Long, _
    ByVal wParam As Long, _
    ByVal lParam As Long _
)
*/
__declspec(dllexport) void _stdcall SetSharedData(int uMsg, WPARAM wParam, LPARAM lParam)
{
	Shared_uMsg = uMsg;
	Shared_wParam = wParam;
	Shared_lParam = lParam;
	ChangeMessage = true;
}

Finally we have to create a routine to forward the messages to the VB App

/*---------------------------------------------------------------------------
				 Filter function for the WH_GETMESSAGE

  The GetMsgProc hook procedure can examine or modify the message. After the hook 
procedure returns control to the system, the GetMessage or PeekMessage function 
returns the message, along with any modifications, to the application that 
originally called it.
Allows changing the message, but not removal (use WM_NULL instead)
---------------------------------------------------------------------------*/

LRESULT CALLBACK GetMsgProc(int nCode, WPARAM wParam, LPARAM lParam)
{
	MSG *lpMsg;
	//return immediately for negative nCodes
	if (nCode >= 0){
		if (nCode==HC_ACTION){
			lpMsg = (MSG *) lParam;
			//see if this is the window we are monitoring
			if (lpMsg->hwnd == hWndCtrl){
				//forward the message to VB App
				ChangeMessage=false;
				SendMessage(hWndVB, lpMsg->message, lpMsg->wParam, lpMsg->lParam);
				if (ChangeMessage==true){//Did VB App make changes to the data?
					lpMsg->message = Shared_uMsg;
					lpMsg->wParam = Shared_wParam;
					lpMsg->lParam = Shared_lParam;
				}//end if change message
			}//end if correct hWnd
		}//end if nCode==Action
	}//end if nCode >=0			
	return(CallNextHookEx(hmsgHooks, nCode, wParam, lParam));
}

This is all you need, now here is how to use it from Visual Basic
You need to subclass something in order to retrieve the messages sent back from the dll.
In this example I have chosen to subclass a checkbox.
I've omitted the sublcassing code since this is explained in detail at various spots in this forum.
First, declare some API functions and our functions

Option Explicit
Private Declare Function FindWindow Lib "user32" Alias "FindWindowA" ( _
  ByVal lpClassName As String, _
  ByVal lpWindowName As String  _
) As Long
Private Declare Function GetWindowThreadProcessId Lib "user32" ( _
  ByVal hwnd As Long,  _
  lpdwProcessId As Long _
) As Long
Private Declare Function FindWindowEx Lib "user32" Alias "FindWindowExA" ( _
  ByVal hWnd1 As Long, _
  ByVal hWnd2 As Long, _
  ByVal lpsz1 As String, _
  ByVal lpsz2 As String _
) As Long
 
Private Const GWLP_WNDPROC = (-4)
Private Const WM_CHAR = &H102
 
'Make sure to change dll path to yours
Private Declare Sub SetSharedData Lib "C:\bin\HookDemo.dll" ( _
    ByVal uMsg As Long, _
    ByVal wParam As Long, _
    ByVal lParam As Long _
)
 
 Private Declare Function InstallFilterDLL Lib "C:\bin\HookDemo.dll" ( _
    ByVal dwThreadID As Long, _
    ByVal ExtrnHandle As Long, _
    ByVal VBHandle As Long _
) As Long
 
 Private Declare Function UnInstallFilterDLL Lib "C:\bin\HookDemo.dll" () As Long

This following routine sets the hook

Private Sub cmdSetHook_Click()
Dim hWndParent As Long
Dim hWndChild As Long
Dim ThreadID As Long
Dim ParentCaption As String
Dim ChildClass As String
Dim ChildCaption As String
 
  ParentCaption = "Untitled - Notepad"
  ChildClass = "Edit"
  ChildCaption = ""
 
  'subclass the checkbox
  Set CSubClsApp = New CSubclass
  CSubClsApp.hwnd = Check1.hwnd
  CSubClsApp.EnableSubclass
 
  'get handle to parent window
  hWndParent = FindWindow(vbNullString, ParentCaption)
  While hWndParent = 0
    If MsgBox("Open Notepad", vbOKCancel) = vbCancel Then Exit Sub
    hWndParent = FindWindow(vbNullString, ParentCaption)
  Wend
 
  'get handle to child window
  hWndChild = FindWindowEx(hWndParent, 0, ChildClass, ChildCaption)
 
  'get ThreadID
  ThreadID = GetWindowThreadProcessId(hWndChild, 0)
 
  'set the hook
  If InstallFilterDLL(ThreadID, hWndChild, Check1.hwnd) Then
    MsgBox "Cannot Install Hook"
    CSubClsApp.DisableSubclass
  End If
 
End Sub

Of Course we don't want our program to crash

Private Sub Form_QueryUnload(Cancel As Integer, UnloadMode As Integer)
  UnInstallFilterDLL
  CSubClsApp.DisableSubclass
End Sub

Finally, we need to write code to handle the hook messages

Public Function NewWndProc( _
  ByVal hwnd As Long, _
  ByVal uMsg As Long, _
  ByVal wParam As Long, _
  ByVal lParam As Long _
) As Long
'Do your message handling here
'If you want to change the data, do it like this
    If uMsg = WM_CHAR Then
        'change all keyboard input to 'X'
        wParam = Asc("X")
        'if you want to discard message, change uMsg to WM_NULL
        SetSharedData uMsg, wParam, lParam
    End If
        'Pass message to the default window procedure
        NewWndProc = CallWindowProc(CSubClsApp.OrigWndProc, hwnd, uMsg, wParam, lParam)
End Function

And that's it. This code will hook Notepad and change all keyboard input to 'X's
Next you might want to add a hook to trap sent messages.
This way you can respond to WM_DESTROY messages telling you the external app is shutting down.

Attached is the full code for the above demo.
Note: missing files added to attached file 2/16/05