Option Explicit
Private Declare Function GetDesktopWindow Lib "user32" () As Long
Private Declare Function FindWindowEx Lib "user32" Alias "FindWindowExA" (ByVal hWnd1 As Long, ByVal hWnd2 As Long, ByVal lpsz1 As String, ByVal lpsz2 As String) As Long
Private Declare Function GetWindowText Lib "user32" Alias "GetWindowTextA" (ByVal hwnd As Long, ByVal lpString As String, ByVal cch As Long) As Long

Sub t_()    ' если известно расположение файлов

' запускаемый экземпляр
     Dim MyXL1 As Object
     Set MyXL1 = GetObject(, "Excel.Application")
     Debug.Print MyXL1.hwnd

     ' экземпляр какой-то книги
     Dim MyXL2 As Object
     Set MyXL2 = GetObject(ThisWorkbook.Path & "\Книга3.xlsx").Application
     Debug.Print MyXL2.hwnd
End Sub

Sub t__()   ' поиск дескрипторов всех экземпляров Excel. Возможный вектор поиска решения...
     Dim alH() As Long
     FindAllHwnd GetDesktopWindow, alH, "XLMAIN"

     Dim i As Integer
     For i = LBound(alH) To UBound(alH)
         Debug.Print alH(i)
         Debug.Print GetWindowText_(alH(i))
     Next i
End Sub

Sub FindAllHwnd(hWndParent As Long, _
                 ByRef alHwnd() As Long, _
                 Optional ByVal sClass As String = vbNullString, _
                 Optional ByVal sCaption As String = vbNullString)

     Dim hwnd As Long
     Dim n As Integer: n = 0
     Do
         hwnd = FindWindowEx(hWndParent, hwnd, sClass, sCaption)
         If hwnd <> 0 Then
             n = n + 1
             ReDim Preserve alHwnd(1 To n)
             alHwnd(n) = hwnd
         End If
     Loop Until hwnd = 0
End Sub

Private Function GetWindowText_(hwnd As Long)
     Dim MyStr As String: MyStr = String(100, Chr$(0))
     GetWindowText hwnd, MyStr, 100
     GetWindowText_ = Left$(MyStr, InStr(MyStr, Chr$(0)) - 1)
End Function

Option Explicit
Private Const OBJID_NATIVEOM = &HFFFFFFF0
Type GUID
     lD1 As Long
     iD2 As Integer
     iD3 As Integer
     abytD4(0 To 7) As Byte
End Type
Declare Function AccessibleObjectFromWindow Lib "oleacc" ( _
                    ByVal hWnd As Long, _
                    ByVal dwId As Long, _
                    riid As GUID, _
                    ppvObject As Object) As Long
'-------------------------------------------------------------------------------

Function AppFromHwnd(hWnd As Long) As Object
' GUID для IDispatch object {00020400-0000-0000-C000-000000000046}
     Dim id As GUID
     With id
         .lD1 = &H20400
         .iD2 = &H0: .iD3 = &H0
         .abytD4(0) = &HC0: .abytD4(1) = &H0
         .abytD4(2) = &H0: .abytD4(3) = &H0
         .abytD4(4) = &H0: .abytD4(5) = &H0
         .abytD4(6) = &H0: .abytD4(7) = &H46
     End With

     Dim obj As Object
     AccessibleObjectFromWindow hWnd, OBJID_NATIVEOM, id, obj
     Set AppFromHwnd = obj.Application
End Function