Fixing an IDA Pro 7.5+ startup crash on Wine

Date: 2022-09-03－ #IDA Pro, #Wine, #Python

Starting with IDA Pro release 7.5, I was observing crashes when running the Windows version in Wine. This is due to IDA relying on LdrUnloadDll refusing to unload python3X.dll when there are still some references to it.

wine crash

The following patch (against Wine 7.16, but probably self contained enough to be applicable to many versions) remedies this by updating LdrUnloadDll such that it never unloads anything named python.dll at all. Not a very clean approach, but does the trick.

diff --git a/dlls/ntdll/loader.c b/dlls/ntdll/loader.c
index 043bce67ea9..a3ffc0047c4 100644
--- a/dlls/ntdll/loader.c
+++ b/dlls/ntdll/loader.c
@@ -2984,6 +2984,7 @@ done:
     return status;
 }
 
+static HMODULE python_base;
 
 /***********************************************************************
  *	search_dll_file
@@ -3196,6 +3197,26 @@ NTSTATUS WINAPI DECLSPEC_HOTPATCH LdrLoadDll(LPCWSTR path_name, DWORD flags,
     NTSTATUS nts;
     WCHAR *dllname = append_dll_ext( libname->Buffer );
 
+    const char python_dll_name[] = "python.dll";
+    unsigned int j = 0;
+    int is_python_dll = 0;
+    for (unsigned int i = 0; i < libname->Length / 2; i++) {
+        /* The actual library is named pythonXYZ.dll with XYZ numeric. We are
+         * somewhat less strict and allow numerals anywhere inside the string.
+         */
+        if (libname->Buffer[i] >= '0' && libname->Buffer[i] <= '9') {
+            continue;
+        }
+        if (libname->Buffer[i] == python_dll_name[j]) {
+            j++;
+        } else {
+            j = 0;
+        }
+        if (j == sizeof(python_dll_name) - 1 /* sorry, no stdlib */) {
+            is_python_dll = 1;
+        }
+    }
+
     RtlEnterCriticalSection( &loader_section );
 
     nts = load_dll( path_name, dllname ? dllname : libname->Buffer, flags, &wm, FALSE );
@@ -3211,6 +3232,10 @@ NTSTATUS WINAPI DECLSPEC_HOTPATCH LdrLoadDll(LPCWSTR path_name, DWORD flags,
     }
     *hModule = (wm) ? wm->ldr.DllBase : NULL;
 
+    if (is_python_dll) {
+        python_base = wm->ldr.DllBase;
+    }
+
     RtlLeaveCriticalSection( &loader_section );
     RtlFreeHeap( GetProcessHeap(), 0, dllname );
     return nts;
@@ -3842,11 +3867,16 @@ static NTSTATUS MODULE_DecRefCount( LDR_DDAG_NODE *node, void *context )
  *
  *
  */
+
 NTSTATUS WINAPI LdrUnloadDll( HMODULE hModule )
 {
     WINE_MODREF *wm;
     NTSTATUS retv = STATUS_SUCCESS;
 
+    if (hModule == python_base) {
+        return retv;
+    }
+
     if (process_detaching) return retv;
 
     TRACE("(%p)\n", hModule);