PInvoke x64 error with .Net 4.0

Question

I was instructed to get some x64-based C # code that calls a native x64 dll called Detagger, which is used to convert HTML to text, while preserving the basic HTML structure.

This code worked for many years when working with the x86 target platform for C # code and x86 dll assembly, but it failed to configure the x64 platform target platform and using x64 assembly for dll. In fact, x64 works fine if a C # application is built with .Net framework 3.5 or lower. It crashes when building from 4.0 or higher.

The corresponding DLL has the following header:

#ifdef WIN32
    #ifdef USE_DLL
    #ifdef DLL_EXPORTS
        #define DLL_DECLARE __declspec(dllexport) long __stdcall
    #else
        #define DLL_DECLARE __declspec(dllimport) long __stdcall
    #endif
    #else
    #define DLL_DECLARE long
    #endif
#else
    #define DLL_DECLARE long
#endif

...

DLL_DECLARE CONVERTER_Allocate ();  // returns non-zero Handle if succeeds

...

DLL_DECLARE CONVERTER_ResetPolicies (long Handle);

API CONVERT_Allocate(), "" (, , ), "" . , .

CONVERTER_ResetPolicies(), , ( "" ). API : ( ).

++, stdcall, dll ( 4 x86, x64). x64 , fastcall, stdcall, .Net 3.5 , .

PInvoke, dll:

// DLL_DECLARE CONVERTER_Allocate();
[DllImport(_dll, EntryPoint = "CONVERTER_Allocate")]
public static extern IntPtr Allocate();

// DLL_DECLARE CONVERTER_ResetPolicies(long Handle);
[DllImport(_dll, EntryPoint = "CONVERTER_ResetPolicies")]
public static extern APIResult ResetPolicies(IntPtr handle);

#:

IntPtr handle = DetaggerAPI.Allocate();
var result = DetaggerAPI.ResetPolicies();

CONVERTER_ResetPolicies(). :

#:   handle = 0x00000000e82d0080

DLL:

:

RAX = 000000018001B490 RBX = 0000000FCC66EB68 RCX = 00000000E82D0080
RDX = 0000000FCC66EC80 RSI = 0000000FCF8B44A8 RDI = 0000000FCC66E980 
R8  = 00001EB6102A86D4 R9  = 0000000FE84C4001 R10 = 00007FF9497961F0
R11 = 0000000000000000 R12 = 0000000000000000 R13 = 0000000FCC66EAF0
R14 = 0000000FCC66EB68 R15 = 0000000000000004 RIP = 000000018001B490 
RSP = 0000000FCC66E848 RBP = 0000000FCC66E850 EFL = 00000246 

CS = 0033 DS = 0000 ES = 0000 SS = 002B FS = 0000 GS = 0000 

OV = 0 UP = 0 EI = 1 PL = 0 ZR = 1 AC = 0 PE = 1 CY = 0 

, handle RCX (e82d0080).

( , ):

000000018001B490  sub         rsp,28h                   ; subtract 40 from stack pointer, sets up stack frame
000000018001B494  call        000000018001B090  

    000000018001B090  push        rbx  
    000000018001B092  sub         rsp,20h               ; subtract 32 from stack pointer, sets up stack frame
    000000018001B096  test        ecx,ecx               ; check if ecx is 0
    000000018001B098  movsxd      rbx,ecx               ; move value in ecx (the handle passed in) to rbx and sign-extend it to qword
                                                        ; rbx changes from 0000000FCC66EB68 to FFFFFFFFE82D0080
    000000018001B09B  je          000000018001B0C6      ; if ecx is 0, probably jump to a function that returns an error
->  000000018001B09D  cmp         dword ptr [rbx],4D2h  ; compare value pointed to by rbx (as a dword) to 042d (1234),
                                                        ; but rbx points to FFFFFFFFE82D0080, which is probably an invalid memory location,
                                                        ; so !!this is the line that crashes !!
    000000018001B0A3  jne         000000018001B0C6      ; jump if not equal

    000000018001B0A5  mov         ecx,dword ptr [1801122C0h]  
    000000018001B0AB  mov         dword ptr [rbx+2F0B0h],ecx  
    000000018001B0B1  lea         rcx,[rbx+2F0B8h]  
    000000018001B0B8  call        00000001800A7C40  
    000000018001B0BD  mov         rax,rbx  
    000000018001B0C0  add         rsp,20h  
    000000018001B0C4  pop         rbx  
    000000018001B0C5  ret  

000000018001B499  test        rax,rax  
000000018001B49C  jne         000000018001B4BC  
000000018001B49E  cmp         dword ptr [1801122C0h],eax  
000000018001B4A4  je          000000018001B4B2  
000000018001B4A6  lea         rcx,[1800D7B70h]  
000000018001B4AD  call        000000018001B290  
000000018001B4B2  mov         eax,2                     ; if we got here, return 2 in eax, meaning APIResult.Invalid.  Note that this is 32bits.
000000018001B4B7  add         rsp,28h                   ; clean up stack frame
000000018001B4BB  ret                                   ; return

, , "handle" RCX,

movsxd  rbx,ecx

RBX, , , , - .

cmp dword ptr [rbx],4D2h

RBX, .

https://msdn.microsoft.com/en-us/library/ee941656(v=vs.100).aspx#core, " Invoke" , 3.5 SP1 4.0:

, . .

, - stdcall (fastcall ), , , .

, :

• , .NET 3.5 , .
• ++/cli dll PInvoke.

- , - , .

Answer 1

, x64 DLL , . , 64- 32- (long).

:

• e82d0080
• DLL 64-
• DLL 64- .

, - :

DLL_DECLARE CONVERTER_ResetPolicies (long Handle) {
    int* ptr = (int*)Handle;
    if (*ptr == 0x4D2h) 
         ...
}

, Handle > 0x7FFFFFFF - movsxd rbx,ecx.

, Handle 0x7FFFFFFF. , .Net 3.5, 4.0 , , . , Handle 3.5.

, , Windows 7 8, 4 Windows 8. , , , .

Answer 2

, . , , long Windows 32-, .

, , ++ long. , , Linux, long - 64- Linux ( , , ).

intptr_t ( linux Windows <cstdint>/<stdint.h>), [] . , , long intptr_t, , , .

. , intptr_t, , , typedef void*, . , void* - , ( ).

Answer 3

The PInvoke signatures provided by the provider do not look right: long is 4 bytes in x64 mode, but IntPtr has 8 bytes in x64 mode , I suggest changing them to UInt32.

// DLL_DECLARE CONVERTER_Allocate();
[DllImport(_dll, EntryPoint = "CONVERTER_Allocate")]
public static extern UInt32 Allocate();

// DLL_DECLARE CONVERTER_ResetPolicies(long Handle);
[DllImport(_dll, EntryPoint = "CONVERTER_ResetPolicies")]
public static extern APIResult ResetPolicies(UInt32 handle);

This probably shouldn't work under .NET 3.5 either, and it just works out of luck. In addition, I have no idea what APIResult is, so I have not considered this part.