Is it possible to use 64-bit and 32-bit commands in the same executable file on 64-bit Linux?

Question

The 64-bit processor (amd64) supports 32-bit Intel instructions in compatibility mode. In addition, a 64-bit Linux installation allows you to run ELFs containing 32-bit commands if the ELF header says that it is a 32-bit executable.

I was wondering if it is possible to put some assembly instructions inside an ELF that switches the CPU compatibility mode to 32 bits in the middle of the program (and then later back)? If these assembly instructions are not allowed by the kernel, is it possible somehow we can make the kernel switch the already running process to 32-bit?

This is mainly a question of curiosity, since I do not see any precedents in it.

Answer 1

Switching between long mode and compatibility mode is done by changing CS. User mode code cannot modify the descriptor table, but it can perform a distant transition or distant call to a code segment that is already present in the descriptor table. I think that on Linux (for example) the required compatibility mode handle is present.

Here is sample code for Linux (Ubuntu). Build with

$ gcc -no-pie switch_mode.c switch_cs.s

switch_mode.c:

#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>

extern bool switch_cs(int cs, bool (*f)());
extern bool check_mode();

int main(int argc, char **argv)
{
    int cs = 0x23;
    if (argc > 1)
        cs = strtoull(argv[1], 0, 16);
    printf("switch to CS=%02x\n", cs);

    bool r = switch_cs(cs, check_mode);

    if (r)
        printf("cs=%02x: 64-bit mode\n", cs);
    else
        printf("cs=%02x: 32-bit mode\n", cs);

    return 0;
}

switch_cs.s:

        .intel_syntax noprefix
        .code64
        .text
        .globl switch_cs
switch_cs:
        push    rbx
        push    rbp
        mov     rbp, rsp
        sub     rsp, 0x18

        mov     rbx, rsp
        movq    [rbx], offset .L1
        mov     [rbx+4], edi

        // Before the lcall, switch to a stack below 4GB.
        // This assumes that the data segment is below 4GB.
        mov     rsp, offset stack+0xf0
        lcall   [rbx]

        // restore rsp to the original stack
        leave
        pop     rbx
        ret

        .code32
.L1:
        call    esi
        lret


        .code64
        .globl check_mode
// returns false for 32-bit mode; true for 64-bit mode
check_mode:
        xor     eax, eax
        // In 32-bit mode, this instruction is executed as
        // inc eax; test eax, eax
        test    rax, rax
        setz    al
        ret

        .data
        .align  16
stack:  .space 0x100