I linked to that thread in the original post. The primary topic of the thread is that PNP drivers will not unload if the last open handle is on a CDO. This is a known and accepted issue of the driver that I work on. I am more concerned with the documentation. Is the statement “However, the driver must not delete the control device objects until after the framework has deleted the other device objects.” correct? If this is actually a requirement of WDF, I would at least consider moving the deletes of the CDOs from EvtDeviceReleaseHardware to EvtCleanupCallback, although deleting in EvtDeviceReleaseHardware apparently worked fine for many years. The only related information in the linked thread is that Doron_Holan states “The correct pattern is to delete in the last fdo’s cleanup”.
Anyhow, the problem I have does not appear to be related with the time when the CDOs are deleted. Instead WDF has (in my opinion) a bug that it blocks forever if the last reference on a device is released by a WdfRequestComplete in the EvtDeviceFileCreate callback of a CDO. The result is not just that the driver does not unload, but that a process, or in the case I have even another kernel service, hangs forever in the EvtFileCreate. Consequently, the system does not shutdown anymore and with Driver Verifier enabled, bluescreens. In that case I get the following callstack:
nt!DbgBreakPoint
Wdf01000!_FX_DRIVER_GLOBALS::WaitForSignal+0xc6 [minkernel\wdf\framework\shared\object\globals.cpp @ 1972]
Wdf01000!FxIoQueue::Dispose+0xdb [minkernel\wdf\framework\shared\irphandlers\io\fxioqueue.cpp @ 552]
Wdf01000!FxObject::DisposeChildrenWorker+0x1e68e [minkernel\wdf\framework\shared\object\fxobjectstatemachine.cpp @ 1212]
Wdf01000!FxObject::PerformDisposingDisposeChildrenLocked+0x35 [minkernel\wdf\framework\shared\object\fxobjectstatemachine.cpp @ 846]
Wdf01000!FxObject::PerformEarlyDisposeWorkerAndUnlock+0x6a [minkernel\wdf\framework\shared\object\fxobjectstatemachine.cpp @ 926]
Wdf01000!FxObject::PerformEarlyDispose+0x8d [minkernel\wdf\framework\shared\object\fxobjectstatemachine.cpp @ 592]
Wdf01000!FxObject::DisposeChildrenWorker+0x1e629 [minkernel\wdf\framework\shared\object\fxobjectstatemachine.cpp @ 1191]
Wdf01000!FxObject::DeleteObject+0x26d [minkernel\wdf\framework\shared\object\fxobjectstatemachine.cpp @ 124]
Wdf01000!FxPkgGeneral::DecrementOpenHandleCount+0x23 [minkernel\wdf\framework\shared\irphandlers\general\fxpkggeneral.cpp @ 1983]
Wdf01000!FxRequest::CompleteInternal+0x1e855 [minkernel\wdf\framework\shared\core\fxrequest.cpp @ 862]
Wdf01000!imp_WdfRequestComplete+0x8b [minkernel\wdf\framework\shared\core\fxrequestapi.cpp @ 436]
cdoBug!WdfRequestComplete+0x45 [C:\Program Files (x86)\Windows Kits\10\Include\wdf\kmdf\1.15\wdfrequest.h @ 996]
cdoBug!cdoBugEvtFileCreate+0x22 [cdoBug.c @ 14]
Wdf01000!FxPkgGeneral::OnCreate+0xa68 [minkernel\wdf\framework\shared\irphandlers\general\fxpkggeneral.cpp @ 1302]
Wdf01000!FxPkgGeneral::Dispatch+0x44c [minkernel\wdf\framework\shared\irphandlers\general\fxpkggeneral.cpp @ 785]
Wdf01000!FxDevice::DispatchWithLock+0x112 [minkernel\wdf\framework\shared\core\fxdevice.cpp @ 1430]
VerifierExt!xdv_IRP_MJ_CREATE_wrapper+0x99
Relevant member variables of FxIoQueue are:
m_Deleted = 0, m_Disposing = 1, m_Dispatching = 0, m_DriverIoCount = 1, m_Queue.m_RequestCount = 0
To my understanding the bug is that FxIoQueue::DispatchEvents is called with m_DriverIoCount = 1, resulting in a totalIoCount=1 that prevents the FxIoQueue::DispatchEvents from setting the m_FinishDisposing event.
For reference, the driver written to trigger the issue is:
typedef struct {
PKTHREAD thread;
volatile BOOLEAN threadShallRun;
} DEVICE_CTX;
WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(DEVICE_CTX, GetDeviceCtx);
void cdoBugEvtFileCreate(WDFDEVICE d, WDFREQUEST r, WDFFILEOBJECT fo) {
WdfRequestComplete(r, STATUS_UNSUCCESSFUL);
}
void cdoBugThreadRoutine(void *ctxVoid) {
DECLARE_CONST_UNICODE_STRING(ntDevName, L"\Device\cdoBug");
DEVICE_CTX *ctx = ctxVoid;
while (ctx->threadShallRun) {
PWDFDEVICE_INIT devInit;
devInit = WdfControlDeviceInitAllocate( WdfGetDriver(), &SDDL_DEVOBJ_SYS_ALL_ADM_RWX_WORLD_RWX_RES_RWX);
if (!devInit) {
return;
}
NTSTATUS status = WdfDeviceInitAssignName(devInit, &ntDevName);
if (!NT_SUCCESS(status)) {
WdfDeviceInitFree(devInit);
return;
}
WDF_FILEOBJECT_CONFIG foc;
WDF_FILEOBJECT_CONFIG_INIT(&foc, cdoBugEvtFileCreate, NULL, NULL);
WdfDeviceInitSetFileObjectConfig(devInit, &foc, WDF_NO_OBJECT_ATTRIBUTES);
WDFDEVICE cdo;
status = WdfDeviceCreate(&devInit, WDF_NO_OBJECT_ATTRIBUTES, &cdo);
if (!NT_SUCCESS(status)) {
WdfDeviceInitFree(devInit);
return;
}
WdfControlFinishInitializing(cdo);
LARGE_INTEGER li;
li.QuadPart = WDF_REL_TIMEOUT_IN_MS(10);
KeDelayExecutionThread(Executive, FALSE, &li);
WdfObjectDelete(cdo);
}
}
NTSTATUS cdoBugEvtD0Entry(WDFDEVICE device, WDF_POWER_DEVICE_STATE pds) {
DEVICE_CTX *ctx = GetDeviceCtx(device);
ctx->threadShallRun = TRUE;
HANDLE hThread = 0;
OBJECT_ATTRIBUTES objAttr;
InitializeObjectAttributes(&objAttr, NULL, OBJ_KERNEL_HANDLE, NULL, NULL);
NTSTATUS status = PsCreateSystemThread(&hThread, THREAD_ALL_ACCESS, &objAttr, 0, NULL, cdoBugThreadRoutine, ctx);
if (NT_SUCCESS(status)) {
status = ObReferenceObjectByHandle(hThread, SYNCHRONIZE, NULL, KernelMode, &ctx->thread, NULL);
ZwClose(hThread);
}
return status;
}
NTSTATUS cdoBugEvtD0Exit(WDFDEVICE device, WDF_POWER_DEVICE_STATE pds) {
DEVICE_CTX *ctx = GetDeviceCtx(device);
ctx->threadShallRun = FALSE;
KeWaitForSingleObject(ctx->thread, Executive, KernelMode, FALSE, NULL);
ctx->thread = NULL;
return STATUS_SUCCESS;
}
NTSTATUS cdoBugEvtDeviceAdd(WDFDRIVER d, PWDFDEVICE_INIT deviceInit) {
WDF_OBJECT_ATTRIBUTES attributes;
WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attributes, DEVICE_CTX);
WDF_PNPPOWER_EVENT_CALLBACKS callbacks;
WDF_PNPPOWER_EVENT_CALLBACKS_INIT(&callbacks);
callbacks.EvtDeviceD0Entry = cdoBugEvtD0Entry;
callbacks.EvtDeviceD0Exit = cdoBugEvtD0Exit;
WdfDeviceInitSetPnpPowerEventCallbacks(deviceInit, &callbacks);
WDFDEVICE device;
return WdfDeviceCreate(&deviceInit, &attributes, &device);
}
NTSTATUS DriverEntry(PDRIVER_OBJECT driverObject,
PUNICODE_STRING registryPath) {
WDF_DRIVER_CONFIG config;
WDF_DRIVER_CONFIG_INIT(&config, cdoBugEvtDeviceAdd);
return WdfDriverCreate(driverObject, registryPath, WDF_NO_OBJECT_ATTRIBUTES, &config, WDF_NO_HANDLE);
}
and a single userspace process calls CreateFile(L"\\\\.\\globalroot\\Device\\cdoBug", GENERIC_READ, 0, NULL, OPEN_EXISTING, 0, NULL); in an infinite loop.