What is the maximum file mapping size on a 64-bit machine

Question

I am new to 64-bit architecture. Could you tell me what MAX file size is supported by file matching on a 64-bit Linux machine. I want to open more than 20 GB of files by matching files, is it available?

I am writing sample code. But it causes a bus error when I get the pointer value at the GBSIZE offset:

unsigned char* pCur = pBegin + GBSIZE;
//pBegin is the pointer returned by mmap
printf("%c",*pCur); 

BTW, printf("%c",*pBegin );works great. and the size of my address: 38 bits physical, 48 bits virtual

Here is the complete code:

#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>

//#define FILEPATH "smallfile"
#define FILEPATH "bigfile"
#define GBSIZE (1024L*1024L*1024L)
#define TBSIZE (1024L*GBSIZE)
#define NUMSIZE  (20L * GBSIZE)
//#define NUMSIZE  (10)
#define FILESIZE (NUMINTS * sizeof(int))

int main(int argc, char *argv[])
{
    int i;
    int fd;
    unsigned char *pBegin;

    fd = open(FILEPATH, O_RDONLY);
        if (fd == -1) {
        perror("Error opening file for reading");
        exit(EXIT_FAILURE);
    }

    pBegin = mmap(0, NUMSIZE, PROT_READ, MAP_SHARED, fd, 0);
    if (pBegin == MAP_FAILED) {
        close(fd);
        perror("Error mmapping the file");
        exit(EXIT_FAILURE);
    }

    /** ERROR happens here!!! **/
    unsigned char* pCur = pBegin + GBSIZE;
    printf("%c",*pCur);

    if (munmap(pBegin, NUMSIZE) == -1) {
        perror("Error un-mmapping the file");
    }
    close(fd);
    return 0;
}

Answer 1

64- , , 64 . , , /proc/cpuinfo ( 48 ).

grep "address sizes" /proc/cpuinfo

, , 47 Linux.

, , 20- . 47 128 .

Answer 2

mmap(2) man:

   void *mmap(void *addr, size_t length, int prot, int flags,
              int fd, off_t offset);

length - size_t, 64- 64 . , 20- .

Answer 3

64- 20 .

Answer 4

(This answer was originally edited in the OP question)

You requested a 20 GB card for a 50 MB file.

As the mmap man page is described , it mmapis executed when you request too much length, however it will give SIGBUSor SIGSEGVwhen you are actually trying to read outside the main file.

Answer 5

Agree with MarkR, you are looking for an invalid address.

// A bug in these lines.
unsigned char* pCur = pBegin + GBSIZE;      
printf("%c",*pCur);  

unsigned char* pEnd = pBegin + NUMSIZE;  
unsigned char* pLast = pEnd - 1;
unsigned char* pCur = pLast;

I changed your code to use HUGE TLB flags as follows.

#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/mman.h>

#define MAP_HUGETLB     0x40000         /*  create a huge page mapping */
#define MAP_HUGE_SHIFT  26
#define MAP_HUGE_1GB    (30 << MAP_HUGE_SHIFT)

#define KSIZE 1024L
#define MSIZE (1024L*1024L)
#define GSIZE (1024L*1024L*1024L)
#define TSIZE (1024L*GSIZE)

#define INIT_MEM 0
// Fail on my MacBook Pro (Retina, 13-inch, Early 2015)
// Darwin Kernel Version 16.5.0:x86_64
// #define NUMSIZE  (16L * TSIZE)

// mmap ok; init: got killed; signal 9
// #define NUMSIZE  (8L * TSIZE)

// Got killed signal 9
// #define NUMSIZE  (1L * TSIZE)

// OK
// #define NUMSIZE  (200L * GSIZE)

// OK
#define NUMSIZE  (20L * GSIZE)
typedef unsigned long long ETYPE;

#define MEMSIZE (NUMSIZE*sizeof(ETYPE))
#define PGSIZE (16*KSIZE)

void init(ETYPE* ptr) {
        *ptr = (ETYPE)ptr;
}

int verify(ETYPE* ptr) {
        if (*ptr != (ETYPE)ptr) {
                fprintf(stderr, "ERROR: 0x%016llx != %p.\n", *ptr, ptr);
                return -1;
        }
        else {
                fprintf(stdout, "OK: 0x%016llx = %p.\n", *ptr, ptr);
        }
        return 0;
}
int main(int argc, char *argv[])
{
    int i;
    int fd;
    ETYPE *pBegin;

    int flags = MAP_SHARED | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_1GB;
    printf("mmap memory size:%lu GB\n", MEMSIZE/GSIZE);
    pBegin = (ETYPE*) mmap(0, MEMSIZE, PROT_READ | PROT_WRITE, flags, -1, 0);
    if (pBegin == MAP_FAILED) {
        perror("Error mmapping the file");
        exit(EXIT_FAILURE);
    }

    ETYPE* pEnd = pBegin + NUMSIZE;
    ETYPE* pCur = pBegin;

#if INIT_MEM
    while (pCur < pEnd) {
            init(pCur);
            // ++pCur;  //slow if init all addresses.
            pCur += (PGSIZE/sizeof(ETYPE)); 
    }
#endif

    init(&pBegin[0]);
    init(&pBegin[NUMSIZE-1]);

    verify(&pBegin[0]);
    verify(&pBegin[NUMSIZE-1]);

    if (munmap(pBegin, MEMSIZE) == -1) {
        perror("Error un-mmapping the file");
    }
    return 0;
}