I think this is the solution you're looking for. Let me tell you how it works...

  • Before running the program (I used Notepad for this example) using Shell(), it first checks to see if the program has already been run.
  • If it wasn't run, it runs the program for the first time.
  • If it was run before, then it tries to bring that window to the top.
  • If bringing it to the top fails, the user must have closed it, so it then it runs the program again.

Things to know:
  • When you start a program using the Shell() function, it returns that program's process ID.
  • I wrote a function that takes that process ID and compares it to the ProcID of all the top-level windows in order to get the Handle for that window. The handle is what you want to do most window manipulation, not the process ID.
  • After obtaining the handle, you can then use the BringWindowToTop API to bring that window to the top.
  • BringWindowToTop returns 0 if it fails. This means that the user closed the other program, so we just open it again.

To use this example, start a project and drop a command button on the form. Then paste the following code into the form's code window. After that, run the project and click the command button. Notepad will open. Now put your form on top of Notepad and click the command button again. Notepad will come to the top, but won't open again. Now, close Notepad and click the command button again. Notepad will reopen and come to the top.
Code:
Option Explicit

'****************** API Stuff ********************
'
' This usually goes in a module but for this
' example I put everything in the form code window
'
'*************************************************
Private Const GW_HWNDFIRST = 0
Private Const GW_HWNDNEXT = 2
Private Const GW_CHILD = 5

Private Declare Function GetWindow Lib "user32" _
    (ByVal hwnd As Long, ByVal wCmd As Long) As Long
  
Private Declare Function GetDesktopWindow Lib "user32" () As Long

Private Declare Function GetWindowThreadProcessId Lib "user32" _
    (ByVal hwnd As Long, lpdwProcessId As Long) As Long
    
Private Declare Function BringWindowToTop Lib "user32" _
    (ByVal hwnd As Long) As Long

'****************************************************
'         Normal General Declaration Stuff
'****************************************************

Private AppHandle As Long   'Holds the handle to the app we're running

'****************************************************
'             Form/Control Event Code
'****************************************************

Private Sub Command1_Click()
    Dim lRetVal As Long
    
    If AppHandle = 0 Then
        'Program has not been opened yet, so open it
        AppHandle = RunProg("notepad.exe")
    Else
        'Program has been opened, so try to bring it to top
        lRetVal = BringWindowToTop(AppHandle)
        
        'If it didn't come to top, then it's not open, so open it
        If lRetVal = 0 Then
            AppHandle = RunProg("notepad.exe")
        End If
    End If
End Sub

'****************************************************
'              USER Functions
'****************************************************

Private Function RunProg(sAppName As String) As Long
    'This function starts a program using Shell, and returns
    'that program's window handle
    
    Dim ProcessID As Long
   
    ProcessID = Shell(sAppName, vbNormalFocus)
    RunProg = GetHandleFromProcID(ProcessID)
End Function

Private Function GetHandleFromProcID(ProcIDInput As Long) As Long
    'This function gets a window's handle using it's process ID,
    'which is the value returned by the Shell function
    
    Dim DesktopHandle As Long
    Dim ChildHandle As Long
    Dim ChildProcID As Long
    
    DesktopHandle = GetDesktopWindow()
    ChildHandle = GetWindow(DesktopHandle, GW_CHILD)
    
    Do While ChildHandle <> 0
        GetWindowThreadProcessId ChildHandle, ChildProcID
        
        If ChildProcID = ProcIDInput Then
            GetHandleFromProcID = ChildHandle
            Exit Function
        End If
        
        ChildHandle = GetWindow(ChildHandle, GW_HWNDNEXT)
    Loop
End Function