PreRead: how to read a different file during Paging I/O

I want to correct language here - you are having trouble not with my code, but with some code in the driver you are working on. My code worked in a driver I developed in the past. As I do not have idea what you are doing in your driver, I will try to speculate:

  • Do not mess files being opened as pagefile and regular file. Keep data in files that can be opened only as pagefiles.
  • Add some synchronisation. Try to serialise IO to a file. There are two options - exclusive access and read-write ERESOURCE. Try exclusive access, if it helps, try read-write ERESOURCE.

My apologies, I'm sure it's something I'm doing wrong in my code, and not the fault of your code.

Can you send a full BSOD dump and show an example of your code?

With pleasure. Here's the full BSOD dump:

https://www.mediafire.com/file/bdisv3ockal68jj/MEMORY.zip/file

Here's how I open the backup file as a paging file:

#define OpenAsPagingFile

NTSTATUS OpenBackupFile(_In_ PWSTR FilePath, _Out_ PHANDLE FileHandle, _Out_ PFILE_OBJECT *FileObject, _Out_ PDEVICE_OBJECT *DeviceObject, _Out_ PSIZE_T FileSize)
// opens a file that we will later read from in the PreRead callback, returns HANDLE and FileObject and file size
{
  PAGED_CODE();

  NTSTATUS result;

  // initialize the return values
  *FileHandle   = NULL;
  *FileObject   = NULL;
  *DeviceObject = NULL;
  *FileSize     = 0;

  IO_STATUS_BLOCK   ioStatus = {0};
  OBJECT_ATTRIBUTES objAttr  = {0};
  UNICODE_STRING    filePath = {0};

  RtlInitUnicodeString(&filePath, FilePath);
  InitializeObjectAttributes(&objAttr, &filePath, OBJ_CASE_INSENSITIVE | OBJ_KERNEL_HANDLE, NULL, NULL);

  // we use IoCreateFile to open the file
  // ideally we would like to open it as a paging file, because that avoids potential deadlock problems
  // sadly, in multi-thread situations, this results in an NTFS_FILE_SYSTEM BSOD
  #ifdef OpenAsPagingFile
    if (NT_SUCCESS(result = IoCreateFile(FileHandle, GENERIC_READ, &objAttr, &ioStatus, NULL, FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ, FILE_OPEN, FILE_SYNCHRONOUS_IO_NONALERT | FILE_NON_DIRECTORY_FILE | FILE_RANDOM_ACCESS | FILE_NO_COMPRESSION | FILE_NO_INTERMEDIATE_BUFFERING, NULL, 0, 0, NULL, IO_OPEN_PAGING_FILE | IO_NO_PARAMETER_CHECKING)))
  #else
    if (NT_SUCCESS(result = IoCreateFile(FileHandle, GENERIC_READ, &objAttr, &ioStatus, NULL, FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ, FILE_OPEN, FILE_SYNCHRONOUS_IO_NONALERT | FILE_NON_DIRECTORY_FILE | FILE_RANDOM_ACCESS | FILE_NO_COMPRESSION | FILE_NO_INTERMEDIATE_BUFFERING, NULL, 0, 0, NULL, IO_NO_PARAMETER_CHECKING)))
  #endif
  {
    // query the file size
    FILE_STANDARD_INFORMATION stdInfo = {0};
    if ((NT_SUCCESS(result = ZwQueryInformationFile(*FileHandle, &ioStatus, &stdInfo, sizeof(stdInfo), FileStandardInformation))) && (stdInfo.EndOfFile.QuadPart))
    {
      *FileSize = stdInfo.EndOfFile.QuadPart;

      // we also need the FileObject of the file, so we can later call FltReadFile etc
      result = ObReferenceObjectByHandle(*FileHandle, FILE_READ_DATA, NULL, KernelMode, (PVOID*) FileObject, NULL);
      if (NT_SUCCESS(result))
        *DeviceObject = IoGetRelatedDeviceObject(*FileObject);
    }
    if (!NT_SUCCESS(result))
      ZwClose(*FileHandle);
  }
  return result;
}

Here's how I allocate the NonPagedPool read buffer for reading 16 KB blocks from the backup file:

#define BUF_SIZE 16384
#define BUF_ALIGN_ADD (BUF_SIZE - 1)
#define BUF_ALIGN_MASK (~BUF_ALIGN_ADD)

// allocate 32 KB
context->ReadBuf = ExAllocatePoolWithTag(NonPagedPool, BUF_SIZE * 2, BUF_TAG);

// calculated 16 KB aligned address within the 32 KB allocation
context->ReadBufAligned = (PVOID) ((((LONG_PTR) context->ReadBuf) + BUF_ALIGN_ADD) & BUF_ALIGN_MASK);

And finally, here's the PreReadCallback:

(Sorry, this code is a bit complicated because I'm (potentially) doing multiple reads from the backup file for each original read request. I think I have to do that, because the original read requests don't necessarily have to be aligned, but page file reads have to be.)

#define DoIrpDirectly

FLT_PREOP_CALLBACK_STATUS PreReadCallback(_Inout_ PFLT_CALLBACK_DATA Data, _In_ PCFLT_RELATED_OBJECTS FltObjects, _Flt_CompletionContext_Outptr_ PVOID *CompletionContext)
// replace the file content for our special interest files
{
  PAGED_CODE();
  UNREFERENCED_PARAMETER(CompletionContext);

  FLT_PREOP_CALLBACK_STATUS result = FLT_PREOP_SUCCESS_NO_CALLBACK;

  // the dispatch level is supposed to be PASSIVE or APC
  if (KeGetCurrentIrql() > APC_LEVEL)
    return (FLT_IS_FASTIO_OPERATION(Data)) ? FLT_PREOP_DISALLOW_FASTIO : FLT_PREOP_SUCCESS_NO_CALLBACK;

  // check if this is a special interest file
  PSTREAM_CONTEXT backupContext = NULL;
  if ((NT_SUCCESS(FltGetStreamHandleContext(FltObjects->Instance, FltObjects->FileObject, &backupContext))) && (backupContext))
  {
    // it is, now we fulfill the read request ourselves

    if (!Data->Iopb->Parameters.Read.MdlAddress)
      // we need a system buffer to write to, accessing a user buffer directly seems dangerous
      FltLockUserBuffer(Data);

    if (Data->Iopb->Parameters.Read.MdlAddress)
    {
      // figure out which file pointer to read from
      LARGE_INTEGER byteOffset = Data->Iopb->Parameters.Read.ByteOffset;
      if ((byteOffset.QuadPart == -1LL) || ((byteOffset.HighPart == -1) && (byteOffset.LowPart == FILE_USE_FILE_POINTER_POSITION)))
        byteOffset.QuadPart = FltObjects->FileObject->CurrentByteOffset.QuadPart;

      if (byteOffset.QuadPart >= (LONGLONG) GlobalData.BackupFileSize)
        // special case: we don't support reading with the file pointer set to the end-of-file (or even beyond that)
        Data->IoStatus.Status = STATUS_END_OF_FILE;
      else
      {
        // figure out how much to read
        ULONG bytesRequested = Data->Iopb->Parameters.Read.Length;
        ULONG bytesAvailable = (ULONG) (GlobalData.BackupFileSize - byteOffset.QuadPart);
        ULONG bytesToProvide = (bytesRequested <= bytesAvailable) ? bytesRequested : bytesAvailable;

        if (bytesToProvide == 0)
        {
          // special case: nothing to read
          Data->IoStatus.Status = STATUS_SUCCESS;
          Data->IoStatus.Information = 0;
        }
        else
        {
          // we have actual work to do
          PUCHAR buf = MmGetSystemAddressForMdlSafe(Data->Iopb->Parameters.Read.MdlAddress, NormalPagePriority | MdlMappingNoExecute);
          if (!buf)
            // ooops, maybe the OS is out of memory?
            Data->IoStatus.Status = STATUS_INSUFFICIENT_RESOURCES;
          else
            try
            {
              // The read request might be at odd offsets with odd length.
              // But we want to (potentially) read from a paging file, which means our read requests needs to be aligned.
              // We read aligned 16 KB sections from the backup file, and use those to fulfill the original read request.

              // first calculate the beginning and end of the 16 KB read requests we have to perform
              ULONG firstReadOffset = ( (ULONG) byteOffset.QuadPart                                  ) & BUF_ALIGN_MASK;
              ULONG lastReadOffset  = (((ULONG) byteOffset.QuadPart) + bytesToProvide + BUF_ALIGN_ADD) & BUF_ALIGN_MASK;

              // how many 16 KB sections is this?
              ULONG numberOfReads = (lastReadOffset - firstReadOffset) / BUF_SIZE;

              // how many of the leading bytes of the first 16 KB section do we have to throw away (due to mis-alignment)?
              ULONG skipFirstBytes = (ULONG) (byteOffset.QuadPart - firstReadOffset);

              // keep track of how many bytes we still need to fill into the original buffer, and what the (moving) filePos in the backup file is
              ULONG bytesLeftToProvide = bytesToProvide;
              LONG_PTR dstBuf = (LONG_PTR) buf;
              NTSTATUS status = 0;
              LARGE_INTEGER filePos = {0};
              filePos.QuadPart = firstReadOffset;

              // if we have opened the backup file as a paging file, then we need to serialize access to it
              #ifdef OpenAsPagingFile
                GlobalLock();
                try
                {
              #endif

              // now loop through the 16 KB sections we read from the backup file
              for (ULONG i1 = 0; i1 < numberOfReads; i1++)
              {
                ULONG bytesRead = 0;
                status = STATUS_INSUFFICIENT_RESOURCES;

                // we have an aligned NonPagedPool buffer allocated for the backup file, we need to create an MDL for it
                PMDL mdl = IoAllocateMdl(backupContext->ReadBufAligned, BUF_SIZE, FALSE, FALSE, NULL);
                if (mdl)
                {
                  // mark the MDL as NonPagedPool (which it is)
                  MmBuildMdlForNonPagedPool(mdl);

                  #ifdef DoIrpDirectly

                    // We create an IRP ourselves, to have full control over everything.
                    // Doing IO on a file opened as a paging file is a bit more challenging.
                    PIRP irp = IoAllocateIrp(GlobalData.BackupFileDeviceObject->StackSize, FALSE);
                    if (irp)
                    {
                      PIO_STACK_LOCATION irpSp = IoGetNextIrpStackLocation(irp);
                      KEVENT event;
                      IO_STATUS_BLOCK ioStatus = {0};
                      RtlZeroMemory(&ioStatus, sizeof(ioStatus));
                      KeInitializeEvent(&event, SynchronizationEvent, FALSE);
                      irp->MdlAddress = mdl;
                      irp->Flags = IRP_PAGING_IO | IRP_NOCACHE | IRP_SYNCHRONOUS_PAGING_IO;
                      irp->RequestorMode = KernelMode;
                      irp->UserIosb = &ioStatus;
                      irp->UserEvent = &event;
                      irp->UserBuffer = backupContext->ReadBufAligned;
                      irp->Tail.Overlay.OriginalFileObject = GlobalData.BackupFileObject;
                      irp->Tail.Overlay.Thread = PsGetCurrentThread();
                      irpSp->MajorFunction = IRP_MJ_READ;
                      irpSp->Parameters.Read.Length = BUF_SIZE;     // always exactly 16 KB (16 * 1024)
                      irpSp->Parameters.Read.ByteOffset = filePos;  // moving file pos, always aligned to 16 KB
                      irpSp->FileObject = GlobalData.BackupFileObject;
                      IoSetCompletionRoutine(irp, ReadFileIrpCompletion, &event, TRUE, TRUE, TRUE);
                      status = IoCallDriver(GlobalData.BackupFileDeviceObject, irp);
                      if (status == STATUS_PENDING)
                      {
                        KeWaitForSingleObject(&event, Executive, KernelMode, FALSE, NULL);
                        status = ioStatus.Status;
                      }
                      bytesRead = (ULONG) ioStatus.Information;
                    }
                    // we set the MDL to "NULL" in the completion routine, so we need to free it ourselves
                    IoFreeMdl(mdl);

                  #else

                    // we use FltPerformSynchronousIo, for the sake of simpler code, it basically produces the exact same results
                    PFLT_CALLBACK_DATA callbackData = NULL;
                    if ((NT_SUCCESS(status = FltAllocateCallbackData(GlobalData.FilterInstance, GlobalData.BackupFileObject, &callbackData))) && (callbackData))
                    {
                      callbackData->Iopb->MajorFunction = IRP_MJ_READ;
                      callbackData->Iopb->MinorFunction = IRP_MN_NORMAL;
                      callbackData->Iopb->Parameters.Read.Length     = BUF_SIZE;
                      callbackData->Iopb->Parameters.Read.Key        = 0;
                      callbackData->Iopb->Parameters.Read.ByteOffset = filePos;
                      callbackData->Iopb->Parameters.Read.ReadBuffer = backupContext->ReadBufAligned;
                      callbackData->Iopb->Parameters.Read.MdlAddress = mdl;
                      callbackData->Iopb->IrpFlags = IRP_READ_OPERATION | IRP_SYNCHRONOUS_API | IRP_NOCACHE | IRP_PAGING_IO | IRP_SYNCHRONOUS_PAGING_IO;
                      FltPerformSynchronousIo(callbackData);
                      status = callbackData->IoStatus.Status;
                      bytesRead = (ULONG) callbackData->IoStatus.Information;
                      FltFreeCallbackData(callbackData);
                    }
                    // we do *not* free the MDL in this branch because FltPerformSynchronousIo does that for us already

                  #endif
                }

                if ((!NT_SUCCESS(status)) || ((bytesRead < BUF_SIZE) && (i1 < numberOfReads - 1)))
                  break;

                // the 16 KB read was successfull, now figure out how many bytes of it need to be copied
                ULONG usefulBytes = min(bytesRead - skipFirstBytes, bytesLeftToProvide);
                memcpy((PVOID) dstBuf, (PVOID) (((LONG_PTR) (backupContext->ReadBufAligned)) + skipFirstBytes), usefulBytes);

                // now update all the tracking variables
                dstBuf += usefulBytes;
                bytesLeftToProvide -= usefulBytes;
                skipFirstBytes = 0;
                filePos.QuadPart += BUF_SIZE;
              }

              // unlock the global lock (only when opening the backup file as a paging file)
              #ifdef OpenAsPagingFile
                }
                finally
                {
                  GlobalUnlock();
                }
              #endif

              // fill the IoStatus of the original read quest
              if ((NT_SUCCESS(status)) && (bytesLeftToProvide == 0))
              {
                Data->IoStatus.Status = status;
                Data->IoStatus.Information = bytesToProvide;
              }
              else
                Data->IoStatus.Status = STATUS_INSUFFICIENT_RESOURCES;

              #ifdef _DEBUG
                DbgPrintEx(DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL, "%s, Paging I/O: %s, Synchronous: %s, IRP Flags: %x, Read Offset: %d, Length: %d, bytesToProvide: %d, FileObject.CurrentByteOffset: %d", (KeGetCurrentIrql() == PASSIVE_LEVEL) ? "PASSIVE_LEVEL" : ((KeGetCurrentIrql() == APC_LEVEL) ? "APC_LEVEL" : "DISPATCH_LEVEL+"), (Data->Iopb->IrpFlags & (IRP_PAGING_IO | IRP_SYNCHRONOUS_PAGING_IO)) ? "+" : "-", (FltObjects->FileObject->Flags & FO_SYNCHRONOUS_IO) ? "+" : "-", Data->Iopb->IrpFlags, (ULONG) byteOffset.QuadPart, bytesRequested, bytesToProvide, (ULONG) FltObjects->FileObject->CurrentByteOffset.QuadPart);
              #endif

              // since we fulfill this read request ourselves, we also need to update the file pointer in the TargetFileObject ourselves
              if ((NT_SUCCESS(Data->IoStatus.Status)) && (!(Data->Iopb->IrpFlags & IRP_PAGING_IO)) && (Data->Iopb->TargetFileObject->Flags & FO_SYNCHRONOUS_IO))
                Data->Iopb->TargetFileObject->CurrentByteOffset.QuadPart = byteOffset.QuadPart + Data->IoStatus.Information;

            }
            except (EXCEPTION_EXECUTE_HANDLER)
            {
              // for some reason, the read request crashed, maybe the buffer isn't accessible?
              Data->IoStatus.Status = STATUS_INVALID_USER_BUFFER;
            }
        }
      }

      // if something went wrong, we set IoStatus.Information to zero
      if (!NT_SUCCESS(Data->IoStatus.Status))
        Data->IoStatus.Information = 0;

      // since we (usually) update TargetFileObject->CurrentByteOffset, we need to mark the Callback Data as "dirty"
      FltSetCallbackDataDirty(Data);
      result = FLT_PREOP_COMPLETE;

    }
    FltReleaseContext(backupContext);
  }

  return result;
}

P.S: Forgot the completion routine:

NTSTATUS ReadFileIrpCompletion(IN PDEVICE_OBJECT DeviceObject, IN PIRP Irp, IN PVOID Context)
// this is the completion routine we register for our IRP backup file read requests
{
  UNREFERENCED_PARAMETER(DeviceObject);
  UNREFERENCED_PARAMETER(Context);

  // we set the MDL to NULL, to prevent OS from freeing the MDL
  // the OS might free the MDL otherwise, or maybe not, we're not sure
  // this approach makes sure the MDL is not freed, so we can free it ourselves and avoid a leak
  Irp->MdlAddress = NULL;

  // not sure I understand this, but Microsoft documentation seems to say it's needed
  if (Irp->PendingReturned)
    IoMarkIrpPending(Irp);

  return STATUS_SUCCESS;
}

And maybe I should also show the GlobalLock:

// FltAcquireResourceExclusive disables normal kernel APC delivery, so no need to call KeEnterCriticalRegion
#define   GlobalLock() FltAcquireResourceExclusive(GlobalData.Lock);
#define GlobalUnlock() FltReleaseResource         (GlobalData.Lock);

GlobalData.Lock = ExAllocatePoolZero(NonPagedPool, sizeof(ERESOURCE), LOCK_TAG);
ExInitializeResourceLite(GlobalData.Lock);

Thank you very much for looking into this!

Looks like something bad happened in ntfs!NtfsLookupNtfsMcbEntryWithSyncFlag(). There is a part of decompiled ntfs!NtfsPagingFileIo():

if ( !(unsigned __int8)NtfsLookupNtfsMcbEntryWithSyncFlag(
                             (int)Scb + 384,
                             v25,
                             (int)v82,
                             (int)v79,
                             0i64,
                             0i64,
                             0i64,
                             0i64) )
    {
      if ( WPP_GLOBAL_Control != (PDEVICE_OBJECT)&WPP_GLOBAL_Control
        && (HIDWORD(WPP_GLOBAL_Control->Timer) & 1) != 0
        && BYTE1(WPP_GLOBAL_Control->Timer) )
      {
        WPP_SF_(WPP_GLOBAL_Control->AttachedDevice, 12i64, &WPP_42c5de6baa65390274c31d200daade38_Traceguids);
      }
      KeBugCheckEx(0x24u, 0xB0000C0BC7ui64, v25, *((_QWORD *)Scb + 3), *((_QWORD *)Scb + 4));  // "C7" /"C8" is just a code line number
    }

In older (leaked) versions of NTFS there was a call to ntfs!NtfsRaiseStatus():

    //
    //  Paging files reads/ writes should always be correct.  If we didn't
    //  find the allocation, something bad has happened.
    //

    if (!NtfsLookupNtfsMcbEntry( &Scb->Mcb,
                                 ThisVcn,
                                 &ThisLcn,
                                 &ThisClusterCount,
                                 NULL,
                                 NULL,
                                 NULL,
                                 NULL )) {

        NtfsRaiseStatus( IrpContext, STATUS_FILE_CORRUPT_ERROR, NULL, Scb->Fcb );
    }

Are you sure the region you are reading exists in the target file?
P.S. I'll try to look at Scb->Mcb in dump and compile & run your code later.

1 Like

Oh my. Thank you very much for the hint!!

I've found the problem - the backup file I used (which was simply a copy of a DLL file) had a non-aligned overall file size. And if I try to make a 16 KB read on the last part of the file (which is less than 16 KB), I get the BSOD. Simply increasing the length of the backup file to be aligned to 16 KB completely fixes the problem.

Well... it'll be difficult due to some political circumstances :slight_smile:
I'm glad you found a solution to your problem.

[this problem has been rectified... no need for this post by the mods]

Oh, I'm sorry, I was not aware of that. My bad. I thought I was just being nice.

I've edited/deleted those comments now.

All good now. Thank you for your fast and courteous cooperation.

1 Like