Re: Directory Tree - Generates a list of subdirectories.
To enumerate directories.
Use FindFirstFileEx with FindExInfoBasic and FindExSearchLimitToDirectories parameters, there is no other way to get decent performance without going to lower level API's.
Also little performance increase might be achieved using FIND_FIRST_EX_LARGE_FETCH flag, but this seems to be bit contradictory.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by Tech99
Even better perfomance is achieved using NtQueryDirectoryFile API.
** or in NT case, code a system driver which reads and parses MFT, then entire disk folders- and files names can be read in within few seconds - regardless of folder and file count.
Are there any VB6 samples of either of these approaches? I'd be very interested in these.
Basically it implements FindFirstFile/FindNextFile in VB6 using the NtQueryDirectoryFile API. Cool stuff.
Here's a cleaned up version without all the chinese
Code:
Option Explicit
Private Type LARGE_INTEGER
lowpart As Long
highpart As Long
End Type
Private Type UNICODE_STRING
uLength As Integer
uMaximumLength As Integer
pBuffer As Long
End Type
Private Type IO_STATUS_BLOCK
Status As Long
uInformation As Long
End Type
Private Type OBJECT_ATTRIBUTES
Length As Long
RootDirectory As Long
ObjectName As Long
Attributes As Long
SecurityDescriptor As Long
SecurityQualityOfService As Long
End Type
Private Type FILE_BOTH_DIRECTORY_INFORMATION
NextEntryOffset As Long
FileIndex As Long
CreationTime As LARGE_INTEGER
LastAccessTime As LARGE_INTEGER
LastWriteTime As LARGE_INTEGER
ChangeTime As LARGE_INTEGER
EndOfFile As LARGE_INTEGER
AllocationSize As LARGE_INTEGER
FileAttributes As Long
FileNameLength As Long
EaSize As Long
ShortNameLength As Byte
ShortName(23) As Byte
FileName As Byte
FileName1 As Integer '(519) As Byte
End Type
Private Const FileBothDirectoryInformation = 3
Private Const SYNCHRONIZE = &H100000
Private Const FILE_ANY_ACCESS = 0
Private Const FILE_LIST_DIRECTORY = 1
Private Const FILE_DIRECTORY_FILE = 1
Private Const FILE_SYNCHRONOUS_IO_NONALERT = &H20
Private Const FILE_OPEN_FOR_BACKUP_INTENT = &H4000
Private Const OBJ_CASE_INSENSITIVE = &H40
Private Declare Function NtQueryDirectoryFile Lib "ntdll.dll" (ByVal FileHandle As Long, _
ByVal hEvent As Long, _
ByVal ApcRoutine As Long, _
ByVal ApcContext As Long, _
ByRef IoStatusBlock As Any, _
FileInformation As Any, _
ByVal Length As Long, _
ByVal FileInformationClass As Long, _
ByVal ReturnSingleEntry As Long, _
FileName As Any, _
ByVal RestartScan As Long) As Long
Private Declare Function NtClose Lib "ntdll.dll" (ByVal ObjectHandle As Long) As Long
Private Declare Sub CopyMemory Lib "KERNEL32" Alias "RtlMoveMemory" (Destination As Any, Source As Any, ByVal Length As Long)
Private Declare Function NtOpenFile Lib "ntdll.dll" (FileHandle As Long, _
ByVal DesiredAccess As Long, _
ObjectAttributes As OBJECT_ATTRIBUTES, _
IoStatusBlock As IO_STATUS_BLOCK, _
ByVal ShareAccess As Long, _
ByVal OpenOptions As Long) As Long
Private Declare Sub RtlInitUnicodeString Lib "ntdll.dll" (DestinationString As Any, ByVal SourceString As Long)
Private Declare Sub ZeroMemory Lib "ntdll.dll" Alias "RtlZeroMemory" (dest As Any, ByVal numBytes As Long)
Private Function FindFirstFile(ByVal strDirectory As String, bytBuffer() As Byte) As Long
Dim strFolder As String
Dim obAttr As OBJECT_ATTRIBUTES
Dim objIoStatus As IO_STATUS_BLOCK
Dim ntStatus As Long
Dim hFind As Long
Dim strUnicode As UNICODE_STRING
strFolder = "\??\"
strFolder = strFolder & strDirectory
RtlInitUnicodeString strUnicode, StrPtr(strFolder)
obAttr.Length = LenB(obAttr)
obAttr.Attributes = OBJ_CASE_INSENSITIVE
obAttr.ObjectName = VarPtr(strUnicode)
obAttr.RootDirectory = 0
obAttr.SecurityDescriptor = 0
obAttr.SecurityQualityOfService = 0
ntStatus = NtOpenFile(hFind, _
FILE_LIST_DIRECTORY Or SYNCHRONIZE Or FILE_ANY_ACCESS, _
obAttr, _
objIoStatus, _
3, _
FILE_DIRECTORY_FILE Or FILE_SYNCHRONOUS_IO_NONALERT Or FILE_OPEN_FOR_BACKUP_INTENT)
If ntStatus = 0 And hFind <> -1 Then
ntStatus = NtQueryDirectoryFile(hFind, _
0, _
0, _
0, _
objIoStatus, _
bytBuffer(0), _
UBound(bytBuffer), _
FileBothDirectoryInformation, _
1, _
ByVal 0&, _
0)
If ntStatus = 0 Then
FindFirstFile = hFind
Else
NtClose hFind
End If
End If
End Function
Private Function FindNextFile(ByVal hFind As Long, bytBuffer() As Byte) As Boolean
Dim ntStatus As Long
Dim objIoStatus As IO_STATUS_BLOCK
ntStatus = NtQueryDirectoryFile(hFind, _
0, _
0, _
0, _
objIoStatus, _
bytBuffer(0), _
UBound(bytBuffer), _
FileBothDirectoryInformation, _
0, _
ByVal 0&, _
0)
If ntStatus = 0 Then
FindNextFile = True
Else
FindNextFile = False
End If
End Function
Private Sub cmdEnum_Click()
Dim pDir As FILE_BOTH_DIRECTORY_INFORMATION
Dim hFind As Long
Dim bytBuffer() As Byte
Dim bytName() As Byte
Dim strPath As String
Dim strFileName As String * 520
Dim dwFileNameOffset As Long
Dim dwDirOffset As Long
Me.lstFile.Clear
ReDim bytBuffer(LenB(pDir) + 260 * 2 - 3)
strPath = txtPath.Text
If Right(strPath, 1) <> "\" Then strPath = strPath & "\"
hFind = FindFirstFile(strPath, bytBuffer) '»ñÈ¡µÚÒ»¸öÎļþ/Ŀ¼¶ÔÏó
CopyMemory pDir, bytBuffer(0), LenB(pDir)
ReDim bytName(pDir.FileNameLength - 1)
dwFileNameOffset = VarPtr(bytBuffer(&H5E))
CopyMemory bytName(0), ByVal dwFileNameOffset, pDir.FileNameLength
strFileName = strPath & CStr(bytName)
Me.lstFile.AddItem strFileName
Erase bytBuffer
ReDim bytBuffer((LenB(pDir) + CLng(260 * 2 - 3)) * CLng(&H2000))
If FindNextFile(hFind, bytBuffer) Then
dwDirOffset = 0
Do While 1
ZeroMemory pDir, LenB(pDir)
CopyMemory pDir, ByVal VarPtr(bytBuffer(dwDirOffset)), LenB(pDir)
Erase bytName
ReDim bytName(pDir.FileNameLength - 1)
dwFileNameOffset = dwDirOffset + &H5E
dwFileNameOffset = VarPtr(bytBuffer(dwFileNameOffset))
CopyMemory bytName(0), ByVal dwFileNameOffset, pDir.FileNameLength
strFileName = strPath & CStr(bytName)
Me.lstFile.AddItem strFileName
If pDir.NextEntryOffset = 0 Then Exit Do
dwDirOffset = dwDirOffset + pDir.NextEntryOffset
Loop
End If
NtClose hFind
Me.lblMsg.Caption = "Count: " & Me.lstFile.ListCount
End Sub
I'm just not sure how to limit it to directories only yet (apart from checking the .FileAttributes and excluding those after the fact.. pDir.FileAttributes And FILE_ATTRIBUTE_DIRECTORY). It's also not recursive so you'd have to implement that no matter what, the API can't do that on its own.
Last edited by fafalone; Jul 28th, 2016 at 11:44 PM.
Basically it implements FindFirstFile/FindNextFile in VB6 using the NtQueryDirectoryFile API. Cool stuff.
Awesome - thanks for posting this! I'm going to do some timings to see how it compares with what I'm currently using: FindFirstFileExW with FindExInfoBasicInfoLevel and FIND_FIRST_EX_LARGE_FETCH.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by AAraya
Awesome - thanks for posting this! I'm going to do some timings to see how it compares with what I'm currently using: FindFirstFileExW with FindExInfoBasicInfoLevel and FIND_FIRST_EX_LARGE_FETCH.
Well initial performance results are very good! NTQueryDirectoryFile() is indeed quite a bit faster than Find First/NextFileEx. This is not surprising seeing that the Find APIs call NTQueryDirectoryFile from all of the documentation I've come across.
However, I am not getting correct results when enumerating a folder on my backup drive (a MyBook Live). The file and folder counts are off. The local drive works great however. At first I thought it was to the UNC path spec so I mapped a drive to it and used the drive letter, but this actually made the problem WORSE! Searched all over the net for an explanation for this but there's just not a lot of documentation on this undocumented function let alone much in VB for it.
If I can't find a solution to get this to work on all drives, I'm going to have to stick to the Find APIs.
Re: Directory Tree - Generates a list of subdirectories.
Another issue I'm having is with paths in UNC notation (\\server\share\folder\). I prepend the "\??\" required by NtOpenFile which results in a path of "\??\\\server\share\folder\" but when passed to NtOpenFile I get an NT Status code of STATUS_OBJECT_NAME_INVALID. No problems with paths in mapped drive notation. UNC paths must require a different treatment but good luck trying to find this information anywhere.
Re: Directory Tree - Generates a list of subdirectories.
It's just the count that is off? Or are there actually files/folders missing?
It adds a ".." and "." to the list that aren't files, so the count is off by 2 for that
I've seen a few network share examples for NtOpenFile... they all say it should be \\??\\UNC\\server\share and when I use that, the function returns a success, but then things later crash because it says there's a filename but .FileNameLength is 0, so it throws 'subscript out of range' when trying to ReDim the buffer. The path pointed to isn't empty.
Edit: So on another note, I've made it recursive. There were a few complications because it returns "." and ".." as folders, and nulls weren't being trimmed from the buffer.
Code:
Private Sub DoSearch(strPath As String)
'Debug.Print "DoSearch(" & strPath & ")"
Dim pDir As FILE_BOTH_DIRECTORY_INFORMATION
Dim hFind As Long
Dim bytBuffer() As Byte
Dim bytName() As Byte
Dim strFileName As String * 520
Dim dwFileNameOffset As Long
Dim dwDirOffset As Long
Dim sTrimmed As String
ReDim bytBuffer(LenB(pDir) + 260 * 2 - 3)
If Right(strPath, 1) <> "\" Then strPath = strPath & "\"
hFind = FindFirstFile(strPath, bytBuffer)
CopyMemory pDir, bytBuffer(0), LenB(pDir) '»ñÈ¡FILE_BOTH_DIRECTORY_INFORMATION½á¹¹£¬Ä¿µÄÊÇ»ñÈ¡FileNameLengthºÍNextEntryOffsetÊý¾Ý
ReDim bytName(pDir.FileNameLength - 1)
dwFileNameOffset = VarPtr(bytBuffer(&H5E))
CopyMemory bytName(0), ByVal dwFileNameOffset, pDir.FileNameLength
strFileName = strPath & CStr(bytName)
' Me.lstFile.AddItem strFileName
Erase bytBuffer
ReDim bytBuffer((LenB(pDir) + CLng(260 * 2 - 3)) * CLng(&H2000))
If FindNextFile(hFind, bytBuffer) Then
dwDirOffset = 0
Do While 1
ZeroMemory pDir, LenB(pDir) '°ÑFILE_BOTH_DIRECTORY_INFORMATIONÖÃ0
CopyMemory pDir, ByVal VarPtr(bytBuffer(dwDirOffset)), LenB(pDir) 'µÃµ½FILE_BOTH_DIRECTORY_INFORMATION½á¹¹
Erase bytName
ReDim bytName(pDir.FileNameLength - 1)
dwFileNameOffset = dwDirOffset + &H5E
dwFileNameOffset = VarPtr(bytBuffer(dwFileNameOffset))
CopyMemory bytName(0), ByVal dwFileNameOffset, pDir.FileNameLength
strFileName = strPath & CStr(bytName)
If pDir.FileAttributes And FILE_ATTRIBUTE_DIRECTORY Then
sTrimmed = Left$(strFileName, Len(strPath) + (pDir.FileNameLength / 2))
If (sTrimmed <> (strPath & ".")) And (sTrimmed <> (strPath & "..")) Then
Me.lstFile.AddItem sTrimmed
DoSearch sTrimmed
End If
End If
If pDir.NextEntryOffset = 0 Then Exit Do
dwDirOffset = dwDirOffset + pDir.NextEntryOffset
Loop
End If
NtClose hFind
End Sub
Edit2: I'm not sure if it makes a difference in terms of performance, but I also went on to change it to return FILE_DIRECTORY_INFORMATION instead of FILE_BOTH_DIR_INFORMATION, which doesn't retrieve the short name. Doing this also helps in understanding the code better; first the NtQuery API needs to be told to retrieve the other struct instead;
FileDirectoryInformation is used instead of FileBothDirectoryInformation, with the struct:
Code:
Private Type FILE_DIRECTORY_INFORMATION
NextEntryOffset As Long
FileIndex As Long
CreationTime As LARGE_INTEGER
LastAccessTime As LARGE_INTEGER
LastWriteTime As LARGE_INTEGER
ChangeTime As LARGE_INTEGER
EndOfFile As LARGE_INTEGER
AllocationSize As LARGE_INTEGER
FileAttributes As Long
FileNameLength As Long
FileName1 As Integer 'OFFSET=&H40 (64)
End Type
I've noted the byte offset of the filename here, because of this code: dwFileNameOffset = VarPtr(bytBuffer(&H40)) 'It was &H5E, which is the filename offset in FILE_BOTH_DIRECTORY_INFORMATION
[...]
dwFileNameOffset = dwDirOffset + &H40 'This also needs to be changed from &H5E[/tt]
The same technique can also be used to return any other supported value for the FileInformationClass parameter (see https://msdn.microsoft.com/en-us/lib...=vs.85%29.aspx). If you're not limiting to folders so don't need attributes, and don't need timestamps, you can even use FileNamesInformation, which returns only filenames.
The structures have to be padded to 4 bytes, so
Code:
Private Type FILE_ID_BOTH_DIR_INFORMATION
NextEntryOffset As Long
FileIndex As Long
CreationTime As LARGE_INTEGER
LastAccessTime As LARGE_INTEGER
LastWriteTime As LARGE_INTEGER
ChangeTime As LARGE_INTEGER
EndOfFile As LARGE_INTEGER
AllocationSize As LARGE_INTEGER
FileAttributes As Long
FileNameLength As Long '//64 bytes in standard members
EaSize As Long
ShortNameLength As Byte
ShortName(23) As Byte
FileId As LARGE_INTEGER
FId(2) As Byte 'Padding
FileName1 As Integer '//OFFSET=104 &H68
End Type
has a (2) padding of 3 bytes instead of just one like the other structure with short name.
The FileID isn't particularly useful in VB though. You can convert to Currency, but can't shift the last 4 digits to the left of the decimal, and you can't print it as Hex$() without overflowing
Last edited by fafalone; Jul 29th, 2016 at 08:03 PM.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by fafalone
It's just the count that is off? Or are there actually files/folders missing?
Files and folders are missing from the enumeration of the remote drive\folder! Weird right?
In terms of naming, I have come to learn that the Nt functions like NtOpenFile want names in the NT namespace convention whereas I'm working with files that use the Win32 namespace conventions. That's what prepending "\??\" is doing which works for a folder on mapped drive. I need the equivalent for a UNC path. There must be something more elegant than hardcoding it like this however. Surely there exists an API to convert a path from Win32 to Nt namespace? I'm currently looking but haven't found one yet.
Last edited by AAraya; Jul 29th, 2016 at 08:13 PM.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by AAraya
Surely there exists an API to convert a path from Win32 to Nt namespace? I'm currently looking but haven't found one yet.
Seek and you shall find! My research turned up that the way WinAPIs convert DOS paths to NT paths under the hood is via a call to the RtlDosPathNameToRelativeNtPathName() routine. So I'm on the path to a solution here. I'm not a C guy so coming up with VB6 declares for this is not something within my abilities. I'll see if someone else has already written an example for this in VB6.
...And NO. Not much code out there for this routine in general.
Is this something one of you Xtreme VB'ers are up to?
Last edited by AAraya; Jul 29th, 2016 at 08:15 PM.
Re: Directory Tree - Generates a list of subdirectories.
I don't think that function is going to help; its output is the "\??\UNC\\Server\share" that I already tried. It returned success but later gave 0-length names.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by fafalone
I don't think that function is going to help; its output is the "\??\UNC\\Server\share" that I already tried. It returned success but later gave 0-length names.
Dos Path
Full Path
NT Path
\\server\share\ABC\DEF
\\server\share\ABC\DEF
\??\UNC\server\share\ABC\DEF
Hmmm... were you able to write some VB6 code that uses this function and confirm the output? Or are you getting that from the documentation?
Appreciate your help with this. It's nice to know I'm not beating away at this on my own!
Re: Directory Tree - Generates a list of subdirectories.
If you want to play around with it, it goes like this:
Public Declare Function RtlDosPathNameToNtPathName_U Lib "ntdll" (ByVal DosFileName As Long, NtFileName As UNICODE_STRING, FilePart As Long, RelativeName As Any) As Boolean
in FindFirstFile
Code:
Dim strUnicode As UNICODE_STRING
Dim fp As Long, rn As Long
strFolder = "\??\"
strFolder = strFolder & strDirectory
RtlDosPathNameToNtPathName_U StrPtr(strFolder), strUnicode, fp, rn
Edit: That replaces the RtlInitUnicodeString call; don't call both.
The way it's written there results in the same conditions. It works as above for local directories/mapped drives, but not for \\server\share type paths, with or without \??\ prepended.
EDIT-Progress
Are you by chance testing this on shares hosted on the local machine?
I noticed that while it doesn't work for shares on my machine, I *could* use it to list from OTHER machines on my network, using \??\UNC\server\path; using both the original RtlInitUnicodeString way, and the RtlDosPathNameToNtPathName_U way.
And still, ntStatus is always S_OK on the local shares when specified in the exact same format. It seems to give the first folder of ".", then the problem seems to come in FindNextFile, FindNextFile err=0xC000000D STATUS_INVALID_PARAMETER
Edit 2: This may be a bug in this function. Just out of curiosity, I tried the same formatting with the normal CreateFile function. CreateFile(StrPtr("\??\UNC\server\share"), [...]) works for both other machines AND the local machine, which means there's nothing wrong with the format or paths being passed to NtOpenFile; which confines the problem to that second call to NtQuery..
Nothing I've done seems to help; I've limited the access rights, used NtCreateFile instead, etc, nothing is making that later error go away and there's nothing to adjust in that call.
--------------
PS- While we're working on this, I thought it would be helpful to make a function to get the name of NtStatus codes. There's like 1100 of them though so I'm attaching it. There's also an enum for the values.
Last edited by fafalone; Jul 30th, 2016 at 03:57 AM.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by fafalone
If you want to play around with it, it goes like this:
Public Declare Function RtlDosPathNameToNtPathName_U Lib "ntdll" (ByVal DosFileName As Long, NtFileName As UNICODE_STRING, FilePart As Long, RelativeName As Any) As Boolean
in FindFirstFile
Code:
Dim strUnicode As UNICODE_STRING
Dim fp As Long, rn As Long
strFolder = "\??\"
strFolder = strFolder & strDirectory
RtlDosPathNameToNtPathName_U StrPtr(strFolder), strUnicode, fp, rn
Edit: That replaces the RtlInitUnicodeString call; don't call both.
Thanks!
But I'm puzzled. I thought that the RtlDosPathNameToNtPathName() function took a DOS path name and converted it to an NT name. If so, why are you manually prepending it with "\??\"? I thought that this is what the function would do?
Re: Directory Tree - Generates a list of subdirectories.
One would think so, but for whatever reason that doesn't seem to be the case. Without adding the \??\ NtOpenFile returns 0xC000003B STATUS_OBJECT_PATH_SYNTAX_BAD; even for regular C:\path inputs.
In fact on further examination the function actually fails if passed a normal path and doesn't even seem to do anything. To check its output:
Code:
Dim sTrace As String
sTrace = String$(strUnicode.uLength / 2, 0)
CopyMemory ByVal StrPtr(sTrace), ByVal strUnicode.pBuffer, strUnicode.uLength
Debug.Print "pathtrace=" & sTrace
.uLength and .pBuffer both are zero if \??\ isn't there.
Side note; that function leaks memory. strUnicode needs to be freed at the end. Private Declare Sub RtlFreeUnicodeString Lib "ntdll.dll" (UnicodeString As UNICODE_STRING)
Last edited by fafalone; Jul 30th, 2016 at 09:49 AM.
Re: Directory Tree - Generates a list of subdirectories.
I've created my own routine to convert DOS to NT path names. It's pretty simple but maybe it'll help someone else searching for this in the future:
Code:
Public Function DosPathNameToNtPathName(pstrDOSPath As String) As String
'//http://googleprojectzero.blogspot.com/2016/02/the-definitive-guide-on-win32-to-nt.html
Dim strPrepend As String
Dim strNTPath As String
If LenB(pstrDOSPath) = 0 Then
strNTPath = ""
GoTo PROC_EXIT
End If
'make sure path has a trailing backslash
Dim strNameWithBackSlash As String
strNameWithBackSlash = AddSlash(pstrDOSPath)
If IsUNCPath(strNameWithBackSlash) Then
'...UNC path
'ex) "\\server\share\ABC\" = "\??\UNC\server\share\ABC\"
strPrepend = "\??\UNC\"
strNTPath = strPrepend & Right$(strNameWithBackSlash, Len(strNameWithBackSlash) - 2)
Else
'...mapped drive
'ex) "C:\Program Files\" = "\??\C:\Program Files\"
strPrepend = "\??\"
strNTPath = strPrepend & strNameWithBackSlash
End If
PROC_EXIT:
DosPathNameToNtPathName = strNTPath
End Function
Code:
Public Function IsUNCPath(pstrPath As String) As Boolean
Dim bolIsUNCPath As Boolean
If Len(pstrPath) > 1 Then
If IsSameString(Left$(pstrPath, 2), "\\", vbBinaryCompare) Then
bolIsUNCPath = True
Else
bolIsUNCPath = False
End If
Else
bolIsUNCPath = False
End If
IsUNCPath = bolIsUNCPath
End Function
Private Function AddSlash(pstrPath as string) as string
If Right$(pstrPath, 1) <> "\" then
AddSlash = pstrPath & "\"
Else
AddSlash = pstrPath
End If
End Function
Last edited by AAraya; Jul 30th, 2016 at 07:13 PM.
Re: Directory Tree - Generates a list of subdirectories.
Strangely, some changes I just made to my code had an impact on how many files I'm seeing on the remote share. I was seeing about 123 out of 290 files now I'm seeing 164. The only changes on my end were 1) Support for UNC path added (I was using a mapped drive to access the path before), 2) Switch from FILE_BOTH_DIR_INFORMATION to FILE_DIRECTORY_INFORMATION.
If I can't see all files on all drives then NTQueryDirectoryFile is useless to me, no matter how fast it is. What puzzles me is that the Find First/NextFileEx APIs find all of the files and these APIs call the lower level NT routine of NTQueryDirectoryFiles. So why would the higher level routines see all files and not the lower level ones? Is there something else that must be done when calling NTQueryDirectoryFiles, some missing steps or parameters? Or is there a chance that the VB6 code has a bug in it somewhere? But if this was the case why would it work flawlessly on a local hard drive? The NTStatus is reporting STATUS_SUCCESS as far as I can see. I'm not getting any errors.
Bummer, so so close to having this working...
I guess I'm going to try to slog through some code examples in other languages to see if there's something that is missing from the VB6 implementation we're playing around with here.
Re: Directory Tree - Generates a list of subdirectories.
I noticed a similar problem while using FILE_BOTH_DIR_INFORMATION... it didn't list folders, and it didn't list the last 4 files. But after switching to FILE_DIRECTORY_INFORMATION it showed everything in every remote folder I tried. Did you adjust the offsets in both places?
Just in case you're having a permissions issue, I'd change FILE_ANY_ACCESS to FILE_READ_ACCESS (&H1) so you're not requesting write access.
Try using the same changes I'm using; drop in the following to replace everything in frmMain in the sample project:
Code:
Option Explicit
Private Const FILE_ATTRIBUTE_DIRECTORY = &H10
Private Type LARGE_INTEGER
lowpart As Long
highpart As Long
End Type
Private Type UNICODE_STRING
uLength As Integer
uMaximumLength As Integer
pBuffer As Long
End Type
Private Type IO_STATUS_BLOCK
Status As Long
uInformation As Long
End Type
Private Type OBJECT_ATTRIBUTES
Length As Long
RootDirectory As Long
ObjectName As Long
Attributes As Long
SecurityDescriptor As Long
SecurityQualityOfService As Long
End Type
Private Type FILE_BOTH_DIRECTORY_INFORMATION
NextEntryOffset As Long
FileIndex As Long
CreationTime As LARGE_INTEGER '8 bytes
LastAccessTime As LARGE_INTEGER
LastWriteTime As LARGE_INTEGER
ChangeTime As LARGE_INTEGER
EndOfFile As LARGE_INTEGER
AllocationSize As LARGE_INTEGER
FileAttributes As Long
FileNameLength As Long '//64
EaSize As Long
ShortNameLength As Byte
ShortName(23) As Byte
FileName As Byte
FileName1 As Integer '(519) As Byte; OFFSET=&H5E (94 bytes, this is the 95th)
End Type
Private Type FILE_DIRECTORY_INFORMATION
NextEntryOffset As Long
FileIndex As Long
CreationTime As LARGE_INTEGER
LastAccessTime As LARGE_INTEGER
LastWriteTime As LARGE_INTEGER
ChangeTime As LARGE_INTEGER
EndOfFile As LARGE_INTEGER
AllocationSize As LARGE_INTEGER
FileAttributes As Long
FileNameLength As Long
FileName1 As Integer 'OFFSET=&H40 (64)
End Type
Private Type FILE_FULL_DIR_INFORMATION
NextEntryOffset As Long
FileIndex As Long
CreationTime As LARGE_INTEGER
LastAccessTime As LARGE_INTEGER
LastWriteTime As LARGE_INTEGER
ChangeTime As LARGE_INTEGER
EndOfFile As LARGE_INTEGER
AllocationSize As LARGE_INTEGER
FileAttributes As Long
FileNameLength As Long '//64 bytes in standard members
EaSize As Long
FileName1 As Integer 'OFFSET=&H48
End Type
Private Type FILE_ID_BOTH_DIR_INFORMATION
NextEntryOffset As Long
FileIndex As Long
CreationTime As LARGE_INTEGER
LastAccessTime As LARGE_INTEGER
LastWriteTime As LARGE_INTEGER
ChangeTime As LARGE_INTEGER
EndOfFile As LARGE_INTEGER
AllocationSize As LARGE_INTEGER
FileAttributes As Long
FileNameLength As Long '//64 bytes in standard members
EaSize As Long
ShortNameLength As Byte
ShortName(23) As Byte
FileId As LARGE_INTEGER
FId(2) As Byte 'Padding
FileName1 As Integer '//OFFSET=104 &H68
End Type
Private Type FILE_ID_FULL_DIR_INFORMATION
NextEntryOffset As Long
FileIndex As Long
CreationTime As LARGE_INTEGER
LastAccessTime As LARGE_INTEGER
LastWriteTime As LARGE_INTEGER
ChangeTime As LARGE_INTEGER
EndOfFile As LARGE_INTEGER
AllocationSize As LARGE_INTEGER
FileAttributes As Long
FileNameLength As Long '//64 bytes in standard members
EaSize As Long
FileId As LARGE_INTEGER
FileName1 As Integer '//OFFSET=76 &H4C
End Type
Private Type FILE_NAMES_INFORMATION
NextEntryOffset As Long
FileIndex As Long
FileNameLength As Long
FileName1 As Integer '//OFFSET=12 &HC
End Type
'Only the uncommented values can be used with NtQueryDirectoryFile
Private Enum FILE_INFORMATION_CLASS
FileDirectoryInformation = 1
FileFullDirectoryInformation = 2
FileBothDirectoryInformation = 3
FileNamesInformation = 12
FileObjectIdInformation = 29
FileReparsePointInformation = 33
FileIdBothDirectoryInformation = 37
FileIdFullDirectoryInformation = 38
End Enum
Private Const SYNCHRONIZE = &H100000
Private Const FILE_ANY_ACCESS = 0
Private Const FILE_READ_ACCESS = 1
Private Const FILE_LIST_DIRECTORY = 1
Private Const FILE_DIRECTORY_FILE = 1
Private Const FILE_SYNCHRONOUS_IO_NONALERT = &H20
Private Const FILE_OPEN_FOR_BACKUP_INTENT = &H4000
Private Const OBJ_CASE_INSENSITIVE = &H40
Private Const FILE_OPEN = 1
Private Const GENERIC_WRITE As Long = &H40000000
Private Const FILE_SHARE_READ As Long = &H1&
Private Const OPEN_ALWAYS As Long = 4&
Private Const OPEN_EXISTING As Long = 3&
Private Const CREATE_ALWAYS As Long = 2&
Private Const FILE_ATTRIBUTE_ARCHIVE As Long = &H20&
Private Const GENERIC_READ As Long = &H80000000
Private Const FILE_END As Long = 2&
Private Const GENERIC_ALL As Long = &H10000000
Private Const FILE_ATTRIBUTE_NORMAL As Long = &H80&
Private Declare Function RtlDosPathNameToNtPathName_U Lib "ntdll" (ByVal DosFileName As Long, NtFileName As UNICODE_STRING, FilePart As Long, RelativeName As Any) As Boolean
Private Declare Function CreateFile Lib "kernel32" Alias "CreateFileW" ( _
ByVal lpFileName As Long, _
ByVal dwDesiredAccess As Long, _
ByVal dwShareMode As Long, _
ByVal lpSecurityAttributes As Long, _
ByVal dwCreationDisposition As Long, _
ByVal dwFlagsAndAttributes As Long, _
ByVal hTemplateFile As Long) As Long
Private Declare Function CloseHandle Lib "kernel32" ( _
ByVal hObject As Long) As Long
Private Declare Function NtQueryDirectoryFile Lib "ntdll.dll" (ByVal FileHandle As Long, _
ByVal hEvent As Long, _
ByVal ApcRoutine As Long, _
ByVal ApcContext As Long, _
ByRef IoStatusBlock As Any, _
FileInformation As Any, _
ByVal Length As Long, _
ByVal FileInformationClass As FILE_INFORMATION_CLASS, _
ByVal ReturnSingleEntry As Long, _
FileName As Any, _
ByVal RestartScan As Long) As Long
Private Declare Function NtClose Lib "ntdll.dll" (ByVal ObjectHandle As Long) As Long
Private Declare Sub CopyMemory Lib "kernel32" Alias "RtlMoveMemory" (Destination As Any, Source As Any, ByVal Length As Long)
Private Declare Function NtOpenFile Lib "ntdll.dll" (FileHandle As Long, _
ByVal DesiredAccess As Long, _
ObjectAttributes As OBJECT_ATTRIBUTES, _
IoStatusBlock As IO_STATUS_BLOCK, _
ByVal ShareAccess As Long, _
ByVal OpenOptions As Long) As Long
Private Declare Function NtCreateFile Lib "ntdll.dll" (FileHandle As Long, _
ByVal DesiredAccess As Long, _
ObjectAttributes As OBJECT_ATTRIBUTES, _
IoStatusBlock As IO_STATUS_BLOCK, _
AllocationSize As Any, _
ByVal FileAttributes As Long, _
ByVal ShareAccess As Long, _
ByVal CreateDisposition As Long, _
ByVal CreateOptions As Long, _
ByVal EaBuffer As Any, _
ByVal EaLength As Long) As Long
Private Declare Sub RtlInitUnicodeString Lib "ntdll.dll" (DestinationString As Any, ByVal SourceString As Long)
Private Declare Sub RtlFreeUnicodeString Lib "ntdll.dll" (UnicodeString As UNICODE_STRING)
Private Declare Sub ZeroMemory Lib "ntdll.dll" Alias "RtlZeroMemory" (dest As Any, ByVal numBytes As Long)
Private Function FindFirstFile(ByVal strDirectory As String, bytBuffer() As Byte) As Long
Dim strFolder As String
Dim obAttr As OBJECT_ATTRIBUTES
Dim objIoStatus As IO_STATUS_BLOCK
Dim NTSTATUS As Long
Dim hFind As Long
Dim strUnicode As UNICODE_STRING
Dim fp As Long, rn As Long
strFolder = "\??\"
strFolder = strFolder & strDirectory
RtlDosPathNameToNtPathName_U StrPtr(strFolder), strUnicode, fp, rn
' RtlInitUnicodeString strUnicode, StrPtr(strFolder)
obAttr.Length = LenB(obAttr)
obAttr.Attributes = OBJ_CASE_INSENSITIVE
obAttr.ObjectName = VarPtr(strUnicode)
obAttr.RootDirectory = 0
obAttr.SecurityDescriptor = 0
obAttr.SecurityQualityOfService = 0
' NTSTATUS = NtCreateFile(hFind, FILE_LIST_DIRECTORY Or SYNCHRONIZE Or FILE_READ_ACCESS, _
' obAttr, objIoStatus, ByVal 0&, 0&, FILE_SHARE_READ, FILE_OPEN, _
' FILE_DIRECTORY_FILE Or FILE_SYNCHRONOUS_IO_NONALERT, ByVal 0&, 0&)
NTSTATUS = NtOpenFile(hFind, _
FILE_LIST_DIRECTORY Or SYNCHRONIZE Or FILE_READ_ACCESS, _
obAttr, _
objIoStatus, _
3, _
FILE_DIRECTORY_FILE Or FILE_SYNCHRONOUS_IO_NONALERT) ' Or FILE_OPEN_FOR_BACKUP_INTENT)
Debug.Print "hFind=" & hFind & ",ntStatus=0x" & Hex$(NTSTATUS) & " " & GetNTStatusStr(NTSTATUS)
Debug.Print "iostatus=0x" & Hex$(objIoStatus.Status) & ",iobytes=" & objIoStatus.uInformation
If NTSTATUS = 0 And hFind <> -1 Then
NTSTATUS = NtQueryDirectoryFile(hFind, _
0, _
0, _
0, _
objIoStatus, _
bytBuffer(0), _
UBound(bytBuffer) + 1, _
FileDirectoryInformation, _
1, _
ByVal 0&, _
0)
If NTSTATUS = 0 Then
FindFirstFile = hFind
Else
Debug.Print "Error2, ntStatus=0x" & Hex$(NTSTATUS) & " " & GetNTStatusStr(NTSTATUS)
NtClose hFind
End If
Else
Debug.Print "Error, ntStatus=0x" & Hex$(NTSTATUS) & " " & GetNTStatusStr(NTSTATUS)
End If
RtlFreeUnicodeString strUnicode
End Function
Private Function FindNextFile(ByVal hFind As Long, bytBuffer() As Byte) As Boolean
Dim NTSTATUS As Long
Dim objIoStatus As IO_STATUS_BLOCK
NTSTATUS = NtQueryDirectoryFile(hFind, _
0, _
0, _
0, _
objIoStatus, _
bytBuffer(0), _
UBound(bytBuffer) + 1, _
FileDirectoryInformation, _
0, _
ByVal 0&, _
0)
If NTSTATUS = 0 Then
FindNextFile = True
Else
Debug.Print "FindNextFile hFind=" & hFind & ",err=0x" & Hex$(NTSTATUS) & " " & GetNTStatusStr(NTSTATUS)
Debug.Print "iostatus=0x" & Hex$(objIoStatus.Status) & ",iobytes=" & objIoStatus.uInformation
FindNextFile = False
End If
End Function
Private Sub cmdEnum_Click()
Dim strPath As String
strPath = txtPath.Text
Me.lstFile.Clear
DoSearch strPath
Me.lblMsg.Caption = "Count=" & Me.lstFile.ListCount
End Sub
Private Sub DoSearch(strPath As String)
Debug.Print "DoSearch(" & strPath & ")"
Dim pDir As FILE_DIRECTORY_INFORMATION
Dim hFind As Long
Dim bytBuffer() As Byte
Dim bytName() As Byte
Dim strFileName As String * 520
Dim dwFileNameOffset As Long
Dim dwDirOffset As Long
Dim sTrimmed As String
Dim i As Long
ReDim bytBuffer(LenB(pDir) + 260 * 2 - 3)
If Right(strPath, 1) <> "\" Then strPath = strPath & "\"
hFind = FindFirstFile(strPath, bytBuffer)
If hFind = -1 Then
Debug.Print "hFind=-1, exiting"
Exit Sub
End If
CopyMemory pDir, bytBuffer(0), LenB(pDir)
Debug.Print "initlen=" & pDir.FileNameLength
ReDim bytName(pDir.FileNameLength - 1)
dwFileNameOffset = VarPtr(bytBuffer(&H40))
CopyMemory bytName(0), ByVal dwFileNameOffset, pDir.FileNameLength
strFileName = strPath & CStr(bytName)
Erase bytBuffer
ReDim bytBuffer((LenB(pDir) + CLng(260 * 2 - 3)) * CLng(&H2000))
If FindNextFile(hFind, bytBuffer) Then
dwDirOffset = 0
Do While 1
ZeroMemory pDir, LenB(pDir)
CopyMemory pDir, ByVal VarPtr(bytBuffer(dwDirOffset)), LenB(pDir)
Erase bytName
ReDim bytName(pDir.FileNameLength - 1)
dwFileNameOffset = dwDirOffset + &H40
dwFileNameOffset = VarPtr(bytBuffer(dwFileNameOffset))
CopyMemory bytName(0), ByVal dwFileNameOffset, pDir.FileNameLength
strFileName = strPath & CStr(bytName)
sTrimmed = Left$(strFileName, Len(strPath) + (pDir.FileNameLength / 2))
If (sTrimmed <> (strPath & ".")) And (sTrimmed <> (strPath & "..")) Then
i = i + 1
Me.lstFile.AddItem CStr(i) & ": " & sTrimmed
Debug.Print "file=" & sTrimmed
If pDir.FileAttributes And FILE_ATTRIBUTE_DIRECTORY Then
DoSearch sTrimmed
End If
End If
If pDir.NextEntryOffset = 0 Then Exit Do
dwDirOffset = dwDirOffset + pDir.NextEntryOffset
Loop
End If
NtClose hFind
End Sub
That will add all files and folders, recursively, to the list. It will also number things so you can track down what's not showing up if you still have that problem.
One other thing to consider... is the performance gain from this function so huge it's a significant factor in the latency of listing the contents of a remote system? Something to measure.
Last edited by fafalone; Jul 30th, 2016 at 11:48 PM.
Re: Directory Tree - Generates a list of subdirectories.
Well sadly using your code produced the same results on my remote share. I don't have this problem with the Find First/Next APIs so I'm going to stick with them. I'm very intrigued by the performance gain but obviously can't use NTQueryDirectoryFile if it's missing files. The lack of documentation and examples is also an issue.
Oh well... can't say it's been for naught however. I've learned a bunch working on this. I'm going to take a look at the other NT functions and see if there's anything else in there which I can make use of.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by fafalone
PS- While we're working on this, I thought it would be helpful to make a function to get the name of NtStatus codes. There's like 1100 of them though so I'm attaching it. There's also an enum for the values.
That was useful, thanks! I've created another very useful routine for working with NTStatus codes. This one converts an NTStatus code into a more familiar Win32 error code equivalent:
Code:
Private Type OVERLAPPED
Internal As Long
InternalHigh As Long
Offset As Long
OffsetHigh As Long
hEvent As Long
End Type
Private Declare Sub SetLastError Lib "kernel32" (ByVal dwErrCode As Long)
Private Declare Function GetOverlappedResult Lib "kernel32" (ByVal hFile As Long, lpOverlapped As OVERLAPPED, lpNumberOfBytesTransferred As Long, ByVal bWait As Long) As Long
Public Function ConvertNtStatusToWin32Error(NTStatus As Long) As Long
Dim oldError As Long
Dim result As Long
Dim br As Long
Dim o As OVERLAPPED
With o
.Internal = NTStatus
.InternalHigh = 0
.Offset = 0
.OffsetHigh = 0
.hEvent = 0
End With
oldError = Err.LastDllError() 'don't use GetLastError in VB6. It's not reliable.
Call GetOverlappedResult(0&, o, br, 0&)
result = Err.LastDllError()
SetLastError (oldError)
ConvertNtStatusToWin32Error = result
End Function
Re: Directory Tree - Generates a list of subdirectories.
<clip>
Just in case you're having a permissions issue, I'd change FILE_ANY_ACCESS to FILE_READ_ACCESS (&H1) so you're not requesting write access.
</clip>
I would use FILE_DIRECTORY_FILE option and FILE_LIST_DIRECTORY access.
Re: Directory Tree - Generates a list of subdirectories.
Yes Tech9 get with the program
Code:
NTSTATUS = NtOpenFile(hFind, _
FILE_LIST_DIRECTORY Or SYNCHRONIZE Or FILE_READ_ACCESS, _
obAttr, _
objIoStatus, _
3, _
FILE_DIRECTORY_FILE Or FILE_SYNCHRONOUS_IO_NONALERT)
-----
Edit:
I believe I found the problem with the missing files. It won't list more than 144 files in a particular directory. Haven't figured out how to solve it yet; increasing the buffer size didn't help. This limit only applies on my remote share.
Last edited by fafalone; Aug 1st, 2016 at 02:41 PM.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by fafalone
I believe I found the problem with the missing files. It won't list more than 144 files in a particular directory. Haven't figured out how to solve it yet; increasing the buffer size didn't help. This limit only applies on my remote share.
Relieved that someone else is seeing the same problem I am! The number 144 that you are experiencing is not the same for me - I see more than that on my remote share but it's not too far off from that. I don't recall the exact number but I'm at around 169. I'm hopeful that you will find the problem. My optimism is rekindled!
Re: Directory Tree - Generates a list of subdirectories.
Yeah and the number varies based on which information I request... 144 is what I get with FILE_DIRECTORY_INFORMATION, and with FILE_FULL_DIR_INFORMATION I get 139.
Edit: So FILE_NAMES_INFORMATION seems to be returning all the files. The only problem is that this structure doesn't include FileAttributes, so to search recursively you'd have to make a separate call to PathIsDirectory.
Code:
Private Type FILE_NAMES_INFORMATION
NextEntryOffset As Long
FileIndex As Long
FileNameLength As Long
FileName1 As Integer '//OFFSET= &HC
End Type
FileNamesInformation = 12
Last edited by fafalone; Aug 1st, 2016 at 08:31 PM.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by fafalone
So FILE_NAMES_INFORMATION seems to be returning all the files. The only problem is that this structure doesn't include FileAttributes, so to search recursively you'd have to make a separate call to PathIsDirectory.
Rather than calling PathIsDirectory you can stay with the NT API to determine if a path is a directory. Here's a routine I just created. It should be faster than the Win32 equivalent.
Code:
Private Declare Function NtQueryAttributesFile Lib "ntdll.dll" (ObjectAttributes As OBJECT_ATTRIBUTES, FileBasicInfo As FILE_BASIC_INFORMATION) As Long
Public Function NTIsPathDirectory(NTPath As String) As Boolean
'//Determines if a path is a DIRECTORY
'Note 1: NTPath must be an NT path not a DOS Path (i.e. "\??\C:\" not "C:\")
'Note 2: NtQueryAttributesFile() uses as a result FILE_BASIC_INFORMATION, but it fills only FileAttributes field.
Dim obAttr As OBJECT_ATTRIBUTES
Dim fbi As FILE_BASIC_INFORMATION
Dim usUnicode As UNICODE_STRING
Dim lngNTStatus As Long
'initialize Unicode String structure
RtlInitUnicodeString usUnicode, StrPtr(NTPath)
'populate Object Attributes struct
With obAttr
.Length = LenB(obAttr)
.Attributes = OBJ_CASE_INSENSITIVE
.ObjectName = VarPtr(usUnicode)
.RootDirectory = 0
.SecurityDescriptor = 0
.SecurityQualityOfService = 0
End With
lngNTStatus = NtQueryAttributesFile(obAttr, fbi)
If lngNTStatus = STATUS_SUCCESS Then
NTIsPathDirectory = (fbi.FileAttributes And FILE_ATTRIBUTE_DIRECTORY)
Else
Debug.Print "Path: " & NTPath; ", NTStatus: " & CStr(lngNTStatus)
End If
End Function
This solution would be fine if someone simply needs the list of all files/folders. I need more than that however, I need all of the file details (size, dates, attributes). I could obtain this with a follow-up call to NtQueryFullAttributesFile (or NtQueryInformationFile) but I think that the additional call to this function for each file found would negate any performance gain I obtained from using NTQueryDirectoryFile.
This service operates on a directory file specified by the FileHandle
parameter. The service returns information about files in the directory
specified by the file handle. The ReturnSingleEntry parameter specifies
that only a single entry should be returned rather than filling the buffer.
The actual number of files whose information is returned, is the smallest
of the following:
o One entry, if the ReturnSingleEntry parameter is TRUE.
o The number of files whose information fits into the specified
buffer.
o The number of files that exist.
o One entry if the optional FileName parameter is specified.
Fafalone's code consider all of these things I believe. Though I do have questions about the bytBuffer() size. Why is this being used as the buffer size for FindNextFile?
Code:
(LenB(pDir) + CLng(260 * 2 - 3)) * CLng(&H2000)
Breaking this down into its components:
1) byte length of the FILE_DIRECTORY_INFORMATION structure
Code:
LenB(pDir)
2) MAX_PATH * 2 - 3
Code:
CLng(260 * 2 - 3))
3) 8192
Code:
CLng(&H2000)
My questions are:
2) Why is 260 being used? That's a Win32 number to represent MAX_PATH. But this limitation doesn't apply in kernel mode. The actual length limit for full paths in the kernel is 32767. Does this implementation not allow for long file names? Why do we subtract 3 from MAX_PATH * 2?
3) What is the purpose of multiplying all of this by 8192?
Thanks for helping me out with this. And maybe talking this out will yield some insight which leads to a solution?
Though if the buffer size was the issue here I would expect to see a STATUS_BUFFER_TOO_SMALL error but this never occurs.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by AAraya
Is this the section to which you are referring?
...
Though if the buffer size was the issue here I would expect to see a STATUS_BUFFER_TOO_SMALL error but this never occurs.
I was referring this, under NTQueryDirectoryFile and Routine Description in there...
The actual number of files whose information is returned, is the smallest
of the following:
o One entry, if the ReturnSingleEntry parameter is TRUE.
o The number of files whose information fits into the specified buffer.
o The number of files that exist.
o One entry if the optional FileName parameter is specified.
I would not assume that STATUS_BUFFER_TOO_SMALL would be a return value, routine just fills available buffer and that's it.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by AAraya
Why is 260 being used? That's a Win32 number to represent MAX_PATH. But this limitation doesn't apply in kernel mode. The actual length limit for full paths in the kernel is 32767. Does this implementation not allow for long file names? Why do we subtract 3 from MAX_PATH * 2?
260 comes from the fact that the 'originally' in NTFS file system, maximum path length was 1+2+256+1 ie. [drive][:\][path][null], nowadays maximum path length is that mentioned 32K.
I wonder (now i am assuming) that when/if buffer is insufficient in size, the return value would be STATUS_BUFFER_OVERFLOW?
Re: Directory Tree - Generates a list of subdirectories.
The 8192 shouldn't matter unless there's more than 8192 file names (that's why it's multiplied by the length of the name). I tried increasing it; it makes no difference.
I'm having a temporary memory lapse; the full path can be 32,767, but the buffer here doesn't store the full path, only the file name. Can that be longer than MAX_PATH?
Re: Directory Tree - Generates a list of subdirectories.
The problem of missing files is not restricted to just remote shares. Performing an NT enumeration (no recursion into subfolders) on my Win System32 folder found 505 objects in the root but the count should be about 3900! Not even close! See if you can reproduce this result on your computers.
EDIT: Actually this was probably my doing when I was monkeying with buffer sizes... What stinks is that NTQueryDirectoryFile simply returns the number of files whose information fits into the specified buffer size with not notice passed back to me that this happened. How am I to know that my buffer was too small and that there are more files?
I check the NTStatus return codes and the IoStatus object return values but the return values never notify me of a problem. My understanding was that this was supposed to notify me if the buffer was too small and then I simply resize the buffer and try again. ???
Re: Directory Tree - Generates a list of subdirectories.
Here's another thread which uses NTQueryDirectoryFile. It's a little different in that it's getting Pipes not files but lots of the code is the same. Might be something useful in here for anyone who stumbles on this in the future...
Re: Directory Tree - Generates a list of subdirectories.
Well the system folder is tricky. Have you tried Wow64DisableWow64FsRedirection? I've searched paths with 5000+ files without the count even being off by 1.
Also we should probably start doing time comparisons to see if this is all even worth here. GetTickCount is terribly inadequate; I like the high resolution timing class from VBSpeed, which uses QueryPerformanceCounter.
Last edited by fafalone; Aug 2nd, 2016 at 09:40 PM.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by fafalone
Well the system folder is tricky. Have you tried Wow64DisableWow64FsRedirection? I've searched paths with 5000+ files without the count even being off by 1.
Also we should probably start doing time comparisons to see if this is all even worth here. GetTickCount is terribly inadequate; I like the high resolution timing class from VBSpeed, which uses QueryPerformanceCounter.
Yes, I turn off FS redirection on 64 bit OS. Good thought but that's not the issue.
I thought the problem I was having was due to the buffer size but I've been able to reproduce the issue with a pretty massive buffer so that's not the issue.
My plan was definitely to do timings but only once I got it working properly. It doesn't matter to me how fast it is if I can't get all the files. Also, remember I'm not just getting the list of file names like you are, I'm getting all of the file details via FILE_DIRECTORY_INFORMATION. For now I'm going to step away from this and focus on other areas of my project. Do appreciate all of your help with this. Hope that the dialog and code in here helps someone else with this.
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by AAraya
Rather than calling PathIsDirectory you can stay with the NT API to determine if a path is a directory. Here's a routine I just created. It should be faster than the Win32 equivalent.
btw... does not work with Samba (unix/linux) shares, fbi.FileAttributes value is -357103104 (0xEAB70A00).
Instead of that the NtQueryFullAttributesFile might do...
Re: Directory Tree - Generates a list of subdirectories.
Originally Posted by Tech99
btw... does not work with Samba (unix/linux) shares, fbi.FileAttributes value is -357103104 (0xEAB70A00).
Instead of that the NtQueryFullAttributesFile might do...
Good to know. I play only in the Windows world so that doesn't really impact me but it may bite others. I don't have access to those types of shares so perhaps you can try out the NtQueryFullAttributesFile suggestion?
Re: Directory Tree - Generates a list of subdirectories.
Well don't worry, I'm not a big fan of giving up on bizarre problems so I'll keep working on this. Just one more question, the file discrepancy on local disks, has that happened anywhere outside of C:\Windows and does it happen with the other strangely structured folders (Program Files, Program Files (x86), and Users)?
Re: Directory Tree - Generates a list of subdirectories.
Tested that code from post 43, it 'seems to work' on local folders*, but not in shares**.
* Folders under \windows directory not necessarily report correct count, fex. sysWOW64 reports correct count, but system32 do not.
** Does not enumerate files/folders from shares correctly, fex. W28K share with over 45 folders and over 50K files, return only 1 as file count ie. enumerates only X:\sharename\. <- X:\sharename\and one dot.
** Samba share with 37 folders and 29K files, enumerates 119 files (both dot folders ie. '.' and '..' and 117 files).
Reply With Quote
Originally Posted by Tech99
