This is a simple code snippet for two functions: MuteProcess and IsProcessMuted, going by process id.

Windows 7+, x86/x64 compatible.

Requirements (IDE only)
VB6: oleexp.tlb w/ addons (inc. w/ download) mIID.bas and mCoreAudio.bas
twinBASIC: Windows Development Library for twinBASIC (WinDevLib) (References->Available packages)

Code

Code:
    Private Function MuteProcess(ByVal pid As Long, ByVal bMute As Boolean, Optional pdwError As Long) As Boolean
        On Error GoTo e0
        Dim mDefRenderMM As IMMDevice
        Dim mDeviceEnum As MMDeviceEnumerator
        Dim pSessionMgr As IAudioSessionManager2
        Dim pEnum As IAudioSessionEnumerator
        Dim pControl As IAudioSessionControl
        Dim pControl2 As IAudioSessionControl2
        Dim pVol As ISimpleAudioVolume
        Set mDeviceEnum = New MMDeviceEnumerator
        mDeviceEnum.GetDefaultAudioEndpoint eRender, eConsole, mDefRenderMM

        If (mDefRenderMM Is Nothing) = False Then
            mDefRenderMM.Activate IID_IAudioSessionManager2, CLSCTX_INPROC_SERVER, CVar(0), pSessionMgr
        End If
        If (pSessionMgr Is Nothing) = False Then
            Set pEnum = pSessionMgr.GetSessionEnumerator()
            Dim nSessions As Long
            pEnum.GetCount nSessions
            If nSessions Then
                Dim i As Long
                For i = 0 To nSessions - 1
                    pEnum.GetSession i, pControl
                    Set pControl2 = pControl
                    Dim spid As Long
                    pControl2.GetProcessId spid
                    If spid = pid Then
                        Debug.Print "MuteProcess::Found audio session for pid " & pid
                        Set pVol = pControl2
                        Dim fMute As BOOL
                        fMute = IIf(bMute, 1, 0)
                        pVol.SetMute bMute, UUID_NULL
                        Exit Function
                    End If
                    Debug.Print "MuteProcess::Enumerated audio sessions but none found for pid " & pid
                    pdwError = 1
                Next
            Else
                Debug.Print "MuteProcess::No per-app audio sessions found."
                pdwError = 2
            End If
        Else
            Debug.Print "MuteProcess::Couldn't activate audio session manager."
            pdwError = 3
        End If
        Exit Function
    e0:
        Debug.Print "MuteProcess::An unexpected error occurred, 0x" & Hex$(Err.Number) & ": " & Err.Description
        pdwError = Err.Number
    End Function

    Private Function IsProcessMuted(ByVal pid As Long, pdwError As Long) As Boolean
        On Error GoTo e0
        Dim mDefRenderMM As IMMDevice
        Dim mDeviceEnum As MMDeviceEnumerator
        Dim pSessionMgr As IAudioSessionManager2
        Dim pEnum As IAudioSessionEnumerator
        Dim pControl As IAudioSessionControl
        Dim pControl2 As IAudioSessionControl2
        Dim pVol As ISimpleAudioVolume
        Set mDeviceEnum = New MMDeviceEnumerator
        mDeviceEnum.GetDefaultAudioEndpoint eRender, eConsole, mDefRenderMM

        If (mDefRenderMM Is Nothing) = False Then
            mDefRenderMM.Activate IID_IAudioSessionManager2, CLSCTX_INPROC_SERVER, CVar(0), pSessionMgr
        End If
        If (pSessionMgr Is Nothing) = False Then
            Set pEnum = pSessionMgr.GetSessionEnumerator()
            Dim nSessions As Long
            pEnum.GetCount nSessions
            If nSessions Then
                Dim i As Long
                For i = 0 To nSessions - 1
                    pEnum.GetSession i, pControl
                    Set pControl2 = pControl
                    Dim spid As Long
                    pControl2.GetProcessId spid
                    If spid = pid Then
                        Debug.Print "IsProcessMuted::Found audio session for pid " & pid
                        Set pVol = pControl2
                        Dim fMute As BOOL
                        pVol.GetMute fMute
                        IsProcessMuted = (fMute <> 0)
                        Exit Function
                    End If
                    Debug.Print "IsProcessMuted::Enumerated audio sessions but none found for pid " & pid
                    pdwError = 1
                Next
            Else
                Debug.Print "IsProcessMuted::No per-app audio sessions found."
                pdwError = 2
            End If
        Else
            Debug.Print "IsProcessMuted::Couldn't activate audio session manager."
            pdwError = 3
        End If
        Exit Function
    e0:
        Debug.Print "IsProcessMuted::An unexpected error occurred, 0x" & Hex$(Err.Number) & ": " & Err.Description
        pdwError = Err.Number
    End Function
The ISimpleAudioVolume interface also has GetMasterVolume/SetMasterVolume, and it would be trivial to adapt the code above to use those.