I've been debugging a segfault issue with the elfreader and I would like to point out something that I have noticed.
ELF.h declares an inline function, getElfArchType (MemoryBuffer *Object). When this function is called from ObjectFile::createELFObjectFile the pointer to Object was getting corrupted. The only thing that distinguished this pointer was that it was declared as a unique_ptr. The flow from when it was created to getElfArchType is pretty direct.
The segfault only happens only at optimization level -O0.
The segfault doesn't happen if getElfArchType is declared as, "static inline"
FWIW I'm compiling on x86_64/gcc-4.7.1.
This is the assembly, (widen your email window!)
Broken: Call side
ObjectFile *ObjectFile::createELFObjectFile(MemoryBuffer *Object) {
0x00000000004cc864 <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE>: push %rbp
0x00000000004cc865 <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+1>: mov %rsp,%rbp
0x00000000004cc868 <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+4>: push %rbx
0x00000000004cc869 <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+5>: sub $0x38,%rsp
0x00000000004cc86d <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+9>: mov %rdi,-0x38(%rbp)
std::pair<unsigned char, unsigned char> Ident = getElfArchType(Object);
0x00000000004cc871 <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+13>: mov -0x38(%rbp),%rax
0x00000000004cc875 <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+17>: mov %rax,%rdi
0x00000000004cc878 <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+20>: callq 0x4a51c5 <_ZN4llvm6object14getElfArchTypeEPNS_12MemoryBufferE>
Callee side: The address of *Object is at -0x18(%rbp) not -0x20(%rbp):
0x00000000004a51c5 <_ZN4llvm6object14getElfArchTypeEPNS_12MemoryBufferE>: push %rbp
0x00000000004a51c6 <_ZN4llvm6object14getElfArchTypeEPNS_12MemoryBufferE+1>: mov %rsp,%rbp
0x00000000004a51c9 <_ZN4llvm6object14getElfArchTypeEPNS_12MemoryBufferE+4>: sub $0x20,%rsp
0x00000000004a51cd <_ZN4llvm6object14getElfArchTypeEPNS_12MemoryBufferE+8>: mov %rdi,-0x18(%rbp)
0x00000000004a51d1 <_ZN4llvm6object14getElfArchTypeEPNS_12MemoryBufferE+12>: mov %rsi,-0x20(%rbp)
0x00000000004a51d5 <_ZN4llvm6object14getElfArchTypeEPNS_12MemoryBufferE+16>: mov -0x20(%rbp),%rax
0x00000000004a51d9 <_ZN4llvm6object14getElfArchTypeEPNS_12MemoryBufferE+20>: mov %rax,%rdi
0x00000000004a51dc <_ZN4llvm6object14getElfArchTypeEPNS_12MemoryBufferE+23>: callq 0x4a4a74 <_ZNK4llvm12MemoryBuffer13getBufferSizeEv>
Working version call side, getElfArchType is "static inline"
ObjectFile *ObjectFile::createELFObjectFile(MemoryBuffer *Object) {
0x00000000004cc8da <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE>: push %rbp
0x00000000004cc8db <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+1>: mov %rsp,%rbp
0x00000000004cc8de <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+4>: push %rbx
0x00000000004cc8df <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+5>: sub $0x38,%rsp
0x00000000004cc8e3 <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+9>: mov %rdi,-0x38(%rbp)
std::pair<unsigned char, unsigned char> Ident = getElfArchType(Object);
0x00000000004cc8e7 <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+13>: mov -0x38(%rbp),%rax
0x00000000004cc8eb <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+17>: mov %rax,%rdi
0x00000000004cc8ee <_ZN4llvm6object10ObjectFile19createELFObjectFileEPNS_12MemoryBufferE+20>: callq 0x4cc868 <getElfArchType>
Callee side gets the correct address:
0x00000000004cc868 <getElfArchType>: push %rbp
0x00000000004cc869 <getElfArchType+1>: mov %rsp,%rbp
0x00000000004cc86c <getElfArchType+4>: push %rbx
0x00000000004cc86d <getElfArchType+5>: sub $0x18,%rsp
0x00000000004cc871 <getElfArchType+9>: mov %rdi,-0x18(%rbp)
0x00000000004cc875 <getElfArchType+13>: mov -0x18(%rbp),%rax
0x00000000004cc879 <getElfArchType+17>: mov %rax,%rdi
0x00000000004cc87c <getElfArchType+20>: callq 0x4a4b06 <_ZNK4llvm12MemoryBuffer13getBufferSizeEv>