[Ecls-list] Problem with foreign data pointers from a C library on 64 bits

[Philippe Hanrigou]

Hi,

I am having some problems using ffi to relay pointers between 
ECL and a custom C library. It looks as if the pointers end up
being truncated to 32 bits for a reason that completely escapes 
me... My C is rusty and I am more than green in Common LISP,
so my hope is that somebody here might be kind enough to help
me out ;-)

I can reproduce the problem with this minimal setup:

- A C file simulating native code:

// == foo-native.c ===================================

#include <stdlib.h>
#include <ecl/ecl.h>

void *create_pointer() {
    void *magic;
    printf("\n>>> [create pointer] enter\n");

    magic = malloc(sizeof *magic);
    if (NULL == magic) {
        FEerror("Can't allocate memory", 2);
        return NULL;
    }

    printf("<<< [create pointer] got %p \n", magic);

    return magic;
}

char *use_pointer(void *p) {
    printf("\n>>> [use pointer] got %p\n", p);
    return "OK";
}

// ===================================================

- I compile it to the libfoo.a library with

# ===================================================

cflags="`ecl-config --cflags` -I.  -I${HOME}/Projects/ecl/deps/gc-7.2alpha6/install/include -Wall -g"
gcc ${cflags} -o foo-native.o -c foo-native.c
ar -rcs libfoo.a foo-native*.o

# ===================================================

- I then try to drive the C library from ECL with

;; == foo.lisp ==========================================

(defpackage :foo
    (:use :common-lisp :ffi))
(in-package :foo)

(proclaim '(OPTIMIZE (SAFETY 3) (DEBUG 3) (SPEED 0)))

(ffi:load-foreign-library "foo" :system-library t)

(defun new-p ()
  (c-inline () () :pointer-void "@(return)=create_pointer();" :side-effects t))

(defun use-p (p)
  (c-inline (p) (:pointer-void) :cstring "@(return)=use_pointer(#0);" :side-effects t))

(use-p (new-p))

;; ===================================================

- Define an ASDF file

;; === foo.asd =========================================
(proclaim '(optimize (safety 3) (debug 3) (speed 2)))
(require 'asdf)
(asdf:defsystem :foo
  :name "Foo"
  :components ((:file "foo")))
;; ===================================================

- And try to run the whole thing with :

#!/usr/bin/env ecl -shell
;; ===================================================
(require 'asdf)
(push  #p"./" asdf:*central-registry*)
(asdf:oos 'asdf:load-op 'foo)
(require 'foo)
;; ===================================================

But then I get:

;; ===================================================
;;; Loading #P"/opt/local/lib/ecl-11.1.1/asdf.fas"
;;; Loading #P"/opt/local/lib/ecl-11.1.1/cmp.fas"
;;; Loading "/Users/ph7/Projects/ECL-Bugs/foo.asd"
;;;
;;; Compiling /Users/ph7/Projects/ECL-Bugs/foo.lisp.
;;; OPTIMIZE levels: Safety=3, Space=0, Speed=2, Debug=3
;;;
;;; Compiling (DEFUN NEW-P ...).
;;; Compiling (DEFUN USE-P ...).
;;; End of Pass 1.
;;; Emitting code for NEW-P.
;;; Emitting code for USE-P.
;;; Note:
;;;   Invoking external command:
;;;   gcc -I. -I/opt/local/include/ -I/Users/ph7/Projects/ecl/deps/gc-7.2alpha6/install/include -g -O2 -fPIC -fno-common -g -O2 -fPIC -fno-common -D_THREAD_SAFE -Ddarwin -O2 -w -c /Users/ph7/.cache/common-lisp/ecl-11.1.1-cfcfee52-macosx-x86/Users/ph7/Projects/ECL-Bugs/ASDF-TMP-foo.c -o /Users/ph7/.cache/common-lisp/ecl-11.1.1-cfcfee52-macosx-x86/Users/ph7/Projects/ECL-Bugs/ASDF-TMP-foo.o 
;;; Finished compiling /Users/ph7/Projects/ECL-Bugs/foo.lisp.
;;;
;;; Note:
;;;   Invoking external command:
;;;   gcc -I. -I/opt/local/include/ -I/Users/ph7/Projects/ecl/deps/gc-7.2alpha6/install/include -g -O2 -fPIC -fno-common -g -O2 -fPIC -fno-common -D_THREAD_SAFE -Ddarwin -O2 -w -c /Users/ph7/.Trash/eclinitplVKkA.c -o /Users/ph7/.Trash/eclinitplVKkA.o 
;;; Note:
;;;   Invoking external command:
;;;   gcc -o /Users/ph7/.cache/common-lisp/ecl-11.1.1-cfcfee52-macosx-x86/Users/ph7/Projects/ECL-Bugs/foo.fas -L/opt/local/lib/ /Users/ph7/.Trash/eclinitplVKkA.o /Users/ph7/.cache/common-lisp/ecl-11.1.1-cfcfee52-macosx-x86/Users/ph7/Projects/ECL-Bugs/foo.o -bundle -L/Users/ph7/Projects/ecl/deps/gc-7.2alpha6/install/lib -L/Users/ph7/Projects/ecl/deps/gc-7.2alpha6/install/lib -lecl -lgc -lpthread -lm -lfoo 
;;; Loading "/Users/ph7/.cache/common-lisp/ecl-11.1.1-cfcfee52-macosx-x86/Users/ph7/Projects/ECL-Bugs/foo.fas"

>>> [create pointer] enter
<<< [create pointer] got 0x7fe519c00d40 
>> [ecl_make_foreign_data] enter got 0x19c00d40
>> [ecl_make_foreign_data] output 0x104054d80 pointing to 0x19c00d40
>> [ecl_foreign_data_pointer_safe] object 0x104054d80 pointing to 0x19c00d40
>> [ecl_foreign_data_pointer_safe] return 0x19c00d40
>>> [use pointer] got 0x19c00d40

Condition of type: SEGMENTATION-VIOLATION
Detected access to an invalid or protected memory address.
;; ===================================================


Notice how only the last 32 bits of the original pointer 
(0x7fe5 19c0 0d40) are preserved when we try to use the
pointer (0x19c0 0d40).

Since I thought ecl_make_foreign_data / ecl_foreign_data_pointer_safe,
might have something to do with it, I recompiled ECL with the following
modifications:

diff --git a/src/c/ffi.d b/src/c/ffi.d
index a43dcf4..b6a334d 100644
--- a/src/c/ffi.d
+++ b/src/c/ffi.d
@@ -192,10 +192,13 @@ static ffi_type *ecl_type_to_libffi_type[] = {
 cl_object
 ecl_make_foreign_data(cl_object tag, cl_index size, void *data)
 {
+  printf("\n>> [ecl_make_foreign_data] enter got %p", data);
        cl_object output = ecl_alloc_object(t_foreign);
        output->foreign.tag = tag == Cnil ? @':void' : tag;
        output->foreign.size = size;
        output->foreign.data = (char*)data;
+
+  printf("\n>> [ecl_make_foreign_data] output %p pointing to %p", output, output->foreign.data);
        return output;
 }
 
@@ -212,10 +215,12 @@ ecl_allocate_foreign_data(cl_object tag, cl_index size)
 void *
 ecl_foreign_data_pointer_safe(cl_object f)
 {
+  printf("\n>> [ecl_foreign_data_pointer_safe] object %p pointing to %p", f, f->foreign.data);
        if (ecl_unlikely(type_of(f) != t_foreign)) {
                 FEwrong_type_only_arg(@[si::foreign-data-pointer], f,
                                       @[si::foreign-data]);
         }
+  printf("\n>> [ecl_foreign_data_pointer_safe] return %p", f->foreign.data);
        return f->foreign.data;
 }

But based on the previous output it seems that when we get to 
ecl_make_foreign_data the pointer is already truncated.

>>> [create pointer] enter
<<< [create pointer] got 0x7fe519c00d40 
>> [ecl_make_foreign_data] enter got 0x19c00d40

I can reproduce the problem on OS X and FreeBSD 8.2 (both 
64 bits). I also tried to compile the C file with "-arch x86_64" flag, 
but the truncation still happens:

>>> [create pointer] enter
<<< [create pointer] got 0x7ff6a9e00120 
>> [ecl_make_foreign_data] enter got 0xffffffffa9e00120
>> [ecl_make_foreign_data] output 0x107019d80 pointing to 0xffffffffa9e00120
>> [ecl_foreign_data_pointer_safe] object 0x107019d80 pointing to 0xffffffffa9e00120
>> [ecl_foreign_data_pointer_safe] return 0xffffffffa9e00120
>>> [use pointer] got 0xffffffffa9e00120


Any idea why? 

Thanks in advance,
- Philippe