--- gforth/libcc.fs	2006/12/28 20:53:04	1.5
+++ gforth/libcc.fs	2009/10/03 21:33:48	1.59
@@ -1,12 +1,12 @@
 \ libcc.fs	foreign function interface implemented using a C compiler
 
-\ Copyright (C) 2006 Free Software Foundation, Inc.
+\ Copyright (C) 2006,2007,2008 Free Software Foundation, Inc.
 
 \ This file is part of Gforth.
 
 \ Gforth is free software; you can redistribute it and/or
 \ modify it under the terms of the GNU General Public License
-\ as published by the Free Software Foundation; either version 2
+\ as published by the Free Software Foundation, either version 3
 \ of the License, or (at your option) any later version.
 
 \ This program is distributed in the hope that it will be useful,
@@ -15,15 +15,13 @@
 \ GNU General Public License for more details.
 
 \ You should have received a copy of the GNU General Public License
-\ along with this program; if not, write to the Free Software
-\ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
+\ along with this program. If not, see http://www.gnu.org/licenses/.
 
 
 \ What this implementation does is this: if it sees a declaration like
 
-\ \ something that tells it to include <unistd.h>
 \ \ something that tells it that the current library is libc
-
+\ \c #include <unistd.h>
 \ c-function dlseek lseek n d n -- d
 
 \ it genererates C code similar to the following:
@@ -46,20 +44,124 @@
 \ the function pointer of gforth_c_lseek_ndn_d on the stack and
 \ calls CALL-C.
 
+\ ToDo:
+
+\ Batching, caching and lazy evaluation:
+
+\ Batching:
+
+\ New words are deferred, and the corresponding C functions are
+\ collected in one file, until the first word is EXECUTEd; then the
+\ file is compiled and linked into the system, and the word is
+\ resolved.
+
+\ Caching:
+
+\ Instead of compiling all this stuff anew for every execution, we
+\ keep the files around and have an index file containing the function
+\ names and their corresponding .so files.  If the needed wrapper name
+\ is already present, it is just linked instead of generating the
+\ wrapper again.  This is all done by loading the index file(s?),
+\ which define words for the wrappers in a separate wordlist.
+
+\ The files are built in .../lib/gforth/$VERSION/libcc/ or
+\ ~/.gforth/libcc/$HOST/.
+
 \ other things to do:
 
 \ c-variable forth-name c-name
 \ c-constant forth-name c-name
 
+\ Todo: conversion between function pointers and xts (both directions)
+
+\ taking an xt and turning it into a function pointer:
+
+\ e.g., assume we have the xt of + and want to create a C function int
+\ gforth_callback_plus(int, int), and then pass the pointer to that
+\ function:
+
+\ There should be Forth code like this:
+\   ] + 0 (bye)
+\ Assume that the start of this code is START
+        
+\ Now, there should be a C function:
+
+\ int gforth_callback_plus(int p1, int p2)
+\ {
+\   Cell   *sp = gforth_SP;
+\   Float  *fp = gforth_FP;
+\   Float  *fp = gforth_FP;
+\   Address lp = gforth_LP;
+\   sp -= 2;
+\   sp[0] = p1;
+\   sp[1] = p2;
+\   gforth_engine(START, sp, rp, fp, lp);
+\   sp += 1;
+\   gforth_RP = rp;
+\   gforth_SP = sp;
+\   gforth_FP = fp;
+\   gforth_LP = lp;
+\   return sp[0];
+\ }
+
+\ and the pointer to that function is the C function pointer for the XT of +.
+
+\ Future problems:
+\   how to combine the Forth code generation with inlining
+\   START is not a constant across executions (when caching the C files)
+\      Solution: make START a variable, and store into it on startup with dlsym
+
+\ Syntax:
+\  callback <rettype> <params> <paramtypes> -- <rettype>
+
 
 \ data structures
 
-\ c-function word body:
-\  cell function pointer
-\  char return type index
-\  char parameter count n
-\  char*n parameters (type indices)
-\  counted string: c-name
+\ For every c-function, we have three words: two anonymous words
+\ created by c-function-ft (first time) and c-function-rt (run-time),
+\ and a named deferred word.  The deferred word first points to the
+\ first-time word, then to the run-time word; the run-time word calls
+\ the c function.
+
+[ifundef] parse-name
+    ' parse-word alias parse-name
+[then]
+[ifundef] defer!
+: defer! ( xt xt-deferred -- ) \ gforth  defer-store
+\G Changes the @code{defer}red word @var{xt-deferred} to execute @var{xt}.
+    >body [ has? rom [IF] ] @ [ [THEN] ] ! ;
+[then]
+
+\ : delete-file 2drop 0 ;
+
+require struct.fs
+require mkdir.fs
+
+\ c-function-ft word body:
+struct
+    cell% field cff-cfr \ xt of c-function-rt word
+    cell% field cff-deferred \ xt of c-function deferred word
+    cell% field cff-lha \ address of the lib-handle for the lib that
+                        \ contains the wrapper function of the word
+    char% field cff-rtype  \ return type
+    char% field cff-np     \ number of parameters
+    1 0   field cff-ptypes \ #npar parameter types
+    \  counted string: c-name
+end-struct cff%
+
+variable c-source-file-id \ contains the source file id of the current batch
+0 c-source-file-id !
+variable lib-handle-addr \ points to the library handle of the current batch.
+                         \ the library handle is 0 if the current
+                         \ batch is not yet compiled.
+  here 0 , lib-handle-addr ! \ just make sure LIB-HANDLE always works
+2variable lib-filename   \ filename without extension
+2variable lib-modulename \ basename of the file without extension
+2variable libcc-named-dir-v \ directory for named libcc wrapper libraries
+0 value libcc-path       \ pointer to path of library directories
+
+defer replace-rpath ( c-addr1 u1 -- c-addr2 u2 )
+' noop is replace-rpath
 
 : .nb ( n -- )
     0 .r ;
@@ -67,13 +169,103 @@
 : const+ ( n1 "name" -- n2 )
     dup constant 1+ ;
 
-\ dlerror
+: front-string { c-addr1 u1 c-addr2 u2 -- c-addr3 u3 }
+    \ insert string c-addr2 u2 in buffer c-addr1 u1; c-addr3 u3 is the
+    \ remainder of the buffer.
+    assert( u1 u2 u>= )
+    c-addr2 c-addr1 u2 move
+    c-addr1 u1 u2 /string ;
+
+: front-char { c-addr1 u1 c -- c-addr3 u2 }
+    \ insert c in buffer c-addr1 u1; c-addr3 u3 is the remainder of
+    \ the buffer.
+    assert( u1 0 u> )
+    c c-addr1 c!
+    c-addr1 u1 1 /string ;
+
+: s+ { addr1 u1 addr2 u2 -- addr u }
+    u1 u2 + allocate throw { addr }
+    addr1 addr u1 move
+    addr2 addr u1 + u2 move
+    addr u1 u2 +
+;
+
+: append { addr1 u1 addr2 u2 -- addr u }
+    addr1 u1 u2 + dup { u } resize throw { addr }
+    addr2 addr u1 + u2 move
+    addr u ;
+
+\ linked list stuff (should go elsewhere)
+
+struct
+    cell% field list-next
+    1 0   field list-payload
+end-struct list%
+
+: list-insert { node list -- }
+    list list-next @ node list-next !
+    node list list-next ! ;
+
+: list-append { node endlistp -- }
+    \ insert node at place pointed to by endlistp
+    node endlistp @ list-insert
+    node list-next endlistp ! ;
+
+: list-map ( ... list xt -- ... )
+    \ xt ( ... node -- ... )
+    { xt } begin { node }
+	node while
+	    node xt execute
+	    node list-next @
+    repeat ;
+
+2variable c-libs \ library names in a string (without "lib")
+
+: add-lib ( c-addr u -- ) \ gforth
+\G Add library lib@i{string} to the list of libraries, where
+    \G @i{string} is represented by @i{c-addr u}.
+    c-libs 2@ d0= IF  0 allocate throw 0 c-libs 2!  THEN
+    c-libs 2@ s"  -l" append 2swap append c-libs 2! ;
+
+: add-libpath ( c-addr u -- ) \ gforth
+\G Add path @i{string} to the list of library search pathes, where
+    \G @i{string} is represented by @i{c-addr u}.
+    c-libs 2@ d0= IF  0 allocate throw 0 c-libs 2!  THEN
+    c-libs 2@ s"  -L" append 2swap append c-libs 2! ;
+
+\ C prefix lines
+
+\ linked list of longcstrings: [ link | count-cell | characters ]
+
+list%
+    cell% field c-prefix-count
+    1 0   field c-prefix-chars
+end-struct c-prefix%
+
+variable c-prefix-lines 0 c-prefix-lines !
+variable c-prefix-lines-end c-prefix-lines c-prefix-lines-end !
+
+: print-c-prefix-line ( node -- )
+    dup c-prefix-chars swap c-prefix-count @ type cr ;
+
+: print-c-prefix-lines ( -- )
+    c-prefix-lines @ ['] print-c-prefix-line list-map ;
+
+: save-c-prefix-line ( c-addr u -- )
+    c-source-file-id @ ?dup-if
+	>r 2dup r> write-line throw
+    then
+    align here 0 , c-prefix-lines-end list-append ( c-addr u )
+    longstring, ;
+
+: \c ( "rest-of-line" -- ) \ gforth backslash-c
+    \G One line of C declarations for the C interface
+    -1 parse save-c-prefix-line ;
 
-\ require lib.fs
+s" #include <gforth/" version-string s+ s" /libcc.h>" append ( c-addr u )
+  2dup save-c-prefix-line drop free throw
 
-\ library libc libc.so.6
-\ libc sleep int (int) sleep
-\ libc dlerror (ptr) dlerror
+\ Types (for parsing)
 
 wordlist constant libcc-types
 
@@ -199,7 +391,7 @@ create gen-par-types
     ." ," gen-par-n ." ," gen-par-n ." )" ;
 
 : gen-wrapped-r ( pars c-name fp-change1 sp-change1 -- fp-change sp-change )
-    2dup gen-par-r 2>r ." =" gen-wrapped-void 2r> ;
+    2dup gen-par-r 2>r ." =" gen-wrapped-call 2r> ;
 
 : gen-wrapped-func ( pars c-name fp-change1 sp-change1 -- fp-change sp-change )
     gen-wrapped-a ;
@@ -215,15 +407,27 @@ create gen-wrapped-types
 : gen-wrapped-stmt ( pars c-name fp-change1 sp-change1 ret -- fp-change sp-change )
     cells gen-wrapped-types + @ execute ;
 
-: gen-wrapper-function ( addr -- )
+: wrapper-function-name ( addr -- c-addr u )
     \ addr points to the return type index of a c-function descriptor
-    c@+ { ret } count 2dup { d: pars } chars + count { d: c-name }
-    .\" #include \"engine/libcc.h\"\n"
-    ." void gforth_c_" c-name type ." _"
+    count { r-type } count { d: pars }
+    pars + count { d: c-name }
+    s" gforth_c_" { d: prefix }
+    prefix nip c-name nip + pars nip + 3 + { u }
+    u allocate throw { c-addr }
+    c-addr u
+    prefix front-string c-name front-string '_ front-char
     pars bounds u+do
-	i c@ type-letter emit
+	i c@ type-letter front-char
     loop
-    ." _" ret type-letter emit .\" (void)\n"
+    '_ front-char r-type type-letter front-char assert( dup 0= )
+    2drop c-addr u ;
+
+: gen-wrapper-function ( addr -- )
+    \ addr points to the return type index of a c-function descriptor
+    dup { descriptor }
+    count { ret } count 2dup { d: pars } chars + count { d: c-name }
+    ." void " lib-modulename 2@ type ." _LTX_" descriptor wrapper-function-name 2dup type drop free throw
+    .\" (GFORTH_ARGS)\n"
     .\" {\n  Cell MAYBE_UNUSED *sp = gforth_SP;\n  Float MAYBE_UNUSED *fp = gforth_FP;\n  "
     pars c-name 2over count-stacks ret gen-wrapped-stmt .\" ;\n"
     ?dup-if
@@ -234,67 +438,218 @@ create gen-wrapped-types
     endif
     .\" }\n" ;
 
-: compile-wrapper-function ( -- )
-    s" gcc -fPIC -shared -Wl,-soname,xxx.so.1 -Wl,-export_dynamic -o xxx.so.1 -O xxx.c" system
-    $? abort" compiler generated error" ;
+: scan-back { c-addr u1 c -- c-addr u2 }
+    \ the last occurence of c in c-addr u1 is at u2-1; if it does not
+    \ occur, u2=0.
+    c-addr 1- c-addr u1 + 1- u-do
+	i c@ c = if
+	    c-addr i over - 1+ unloop exit endif
+    1 -loop
+    c-addr 0 ;
+
+: dirname ( c-addr1 u1 -- c-addr2 u2 )
+    \ directory name of the file name c-addr1 u1, including the final "/".
+    '/ scan-back ;
+
+: basename ( c-addr1 u1 -- c-addr2 u2 )
+    \ file name without directory component
+    2dup dirname nip /string ;
+
+: gen-filename ( x -- c-addr u )
+    \ generates a file basename for lib-handle-addr X
+    0 <<# ['] #s $10 base-execute #> 
+    s" gforth_c_" 2swap s+ #>> ;
+
+: libcc-named-dir ( -- c-addr u )
+    libcc-named-dir-v 2@ ;
+
+: libcc-tmp-dir ( -- c-addr u )
+    s" ~/.gforth/libcc-tmp/" ;
+
+: prepend-dirname ( c-addr1 u1 c-addr2 u2 -- c-addr3 u3 )
+    2over s+ 2swap drop free throw ;
+
+: open-wrappers ( -- addr|0 )
+    lib-filename 2@ s" .la" s+
+    2dup libcc-named-dir string-prefix? if ( c-addr u )
+	\ see if we can open it in the path
+	libcc-named-dir nip /string
+	libcc-path open-path-file if
+	    0 exit endif
+	( wfile-id c-addr2 u2 ) rot close-file throw save-mem ( c-addr2 u2 )
+    endif
+    \ 2dup cr type
+    2dup open-lib >r
+    drop free throw r> ;
+
+: c-library-name-setup ( c-addr u -- )
+    assert( c-source-file-id @ 0= )
+    { d: filename }
+    here 0 , lib-handle-addr ! filename lib-filename 2!
+    filename basename lib-modulename 2! ;
+   
+: c-library-name-create ( -- )
+    lib-filename 2@ s" .c" s+ 2dup w/o create-file throw
+    dup c-source-file-id !
+    ['] print-c-prefix-lines swap outfile-execute
+    drop free throw ;
+
+: c-named-library-name ( c-addr u -- )
+    \ set up filenames for a (possibly new) library; c-addr u is the
+    \ basename of the library
+    libcc-named-dir prepend-dirname c-library-name-setup
+    open-wrappers dup if
+	lib-handle-addr @ !
+    else
+        libcc-named-dir $1ff mkdir-parents drop
+	drop c-library-name-create
+    endif ;
+
+: c-tmp-library-name ( c-addr u -- )
+    \ set up filenames for a new library; c-addr u is the basename of
+    \ the library
+    libcc-tmp-dir 2dup $1ff mkdir-parents drop
+    prepend-dirname c-library-name-setup c-library-name-create ;
+
+: lib-handle ( -- addr )
+    lib-handle-addr @ @ ;
+
+: init-c-source-file ( -- )
+    lib-handle 0= if
+	c-source-file-id @ 0= if
+	    here gen-filename c-tmp-library-name
+	endif
+    endif ;
+
+: c-source-file ( -- file-id )
+    c-source-file-id @ assert( dup ) ;
+
+: notype-execute ( ... xt -- ... )
+    what's type { oldtype } try
+	['] 2drop is type execute 0
+    restore
+	oldtype is type
+    endtry
+    throw ;
+
+: c-source-file-execute ( ... xt -- ... )
+    \ direct the output of xt to c-source-file, or nothing
+    lib-handle if
+	notype-execute
+    else
+	c-source-file outfile-execute
+    endif ;
+
+: .lib-error ( -- )
+    [ifdef] lib-error
+        ['] cr stderr outfile-execute
+        lib-error ['] type stderr outfile-execute
+    [then] ;
+
+DEFER compile-wrapper-function ( -- )
+: compile-wrapper-function1 ( -- )
+    lib-handle 0= if
+	c-source-file close-file throw
+	0 c-source-file-id !
+	[ libtool-command s"  --silent --mode=compile " s+
+	  libtool-cc append s"  -I '" append
+	  s" includedir" getenv append  s" '" append ] sliteral
+	s"  -O -c " s+ lib-filename 2@ append s" .c -o " append
+	lib-filename 2@ append s" .lo" append ( c-addr u )
+	\    2dup type cr
+	2dup system drop free throw $? abort" libtool compile failed"
+	[ libtool-command s"  --silent --mode=link " s+
+	  libtool-cc append libtool-flags append s"  -module -rpath " s+ ] sliteral
+	lib-filename 2@ dirname replace-rpath s+ s"  " append
+	lib-filename 2@ append s" .lo -o " append
+	lib-filename 2@ append s" .la" append ( c-addr u )
+	c-libs 2@ append
+	\    2dup type cr
+	2dup system drop free throw $? abort" libtool link failed"
+	open-wrappers dup 0= if
+	    .lib-error true abort" open-lib failed"
+	endif
+	( lib-handle ) lib-handle-addr @ !
+    endif
+    lib-filename 2@ drop free throw 0 0 lib-filename 2! ;
+' compile-wrapper-function1 IS compile-wrapper-function
 \    s" ar rcs xxx.a xxx.o" system
 \    $? abort" ar generated error" ;
 
-: link-wrapper-function ( -- )
-    s" /home/anton/gforth/xxx.so.1" open-lib ( lib-handle )
-    s" gforth_c_strlen_a_n" rot lib-sym dup 0= -32 and throw ;
+: link-wrapper-function { cff -- sym }
+    cff cff-rtype wrapper-function-name { d: wrapper-name }
+    wrapper-name cff cff-lha @ @ assert( dup ) lib-sym dup 0= if
+        .lib-error -&32 throw
+    endif
+    wrapper-name drop free throw ;
 
-: c-function ( "forth-name" "c-name" "{libcc-type}" "--" "libcc-type" -- )
-    create here >r 0 , \ place for the wrapper function pointer
+: c-function-ft ( xt-defr xt-cfr "c-name" "{libcc-type}" "--" "libcc-type" -- )
+    \ build time/first time action for c-function
+    init-c-source-file
+    noname create 2, lib-handle-addr @ ,
     parse-name { d: c-name }
-    parse-function-types c-name string,
-    r@ cell+
-    s" xxx.c" w/o create-file throw ( file-id )
-    dup >r >outfile gen-wrapper-function outfile<
-    r> close-file throw
-    compile-wrapper-function
-    link-wrapper-function
-    r> !
+    here parse-function-types c-name string,
+    ['] gen-wrapper-function c-source-file-execute
+  does> ( ... -- ... )
+    dup 2@ { xt-defer xt-cfr }
+    dup cff-lha @ @ 0= if
+	compile-wrapper-function
+    endif
+    link-wrapper-function xt-cfr >body !
+    xt-cfr xt-defer defer!
+    xt-cfr execute ;
+
+: c-function-rt ( -- )
+    \ run-time definition for c function; addr is the address where
+    \ the sym should be stored
+    noname create 0 ,
   does> ( ... -- ... )
     @ call-c ;
 
+: c-function ( "forth-name" "c-name" "@{type@}" "--" "type" -- ) \ gforth
+    \G Define a Forth word @i{forth-name}.  @i{Forth-name} has the
+    \G specified stack effect and calls the C function @code{c-name}.
+    defer lastxt dup c-function-rt lastxt c-function-ft
+    lastxt swap defer! ;
+
+: clear-libs ( -- ) \ gforth
+\G Clear the list of libs
+    c-source-file-id @ if
+	compile-wrapper-function
+    endif
+    0. c-libs 2! ;
+clear-libs
 
+: c-library-incomplete ( -- )
+    true abort" Called function of unfinished named C library" ;
 
+: c-library-name ( c-addr u -- ) \ gforth
+\G Start a C library interface with name @i{c-addr u}.
+    clear-libs
+    ['] c-library-incomplete is compile-wrapper-function
+    c-named-library-name ;
+
+: c-library ( "name" -- ) \ gforth
+\G Parsing version of @code{c-library-name}
+    parse-name save-mem c-library-name ;
+
+: end-c-library ( -- ) \ gforth
+\G Finish and (if necessary) build the latest C library interface.
+    ['] compile-wrapper-function1 is compile-wrapper-function
+    compile-wrapper-function1 ;
+
+: init-libcc ( -- )
+    s" ~/.gforth/libcc-named/" libcc-named-dir-v 2!
+[IFDEF] make-path
+    make-path to libcc-path
+    libcc-named-dir libcc-path also-path
+    [ s" libccdir" getenv ] sliteral libcc-path also-path
+[THEN]
+;
 
+init-libcc
 
-s" Library not found" exception constant err-nolib
-
-: library ( "name" "file" -- ) \ gforth
-\G Dynamically links the library specified by @i{file}.  Defines a
-\G word @i{name} ( -- lib ) that starts the declaration of a
-\G function from that library.
-    create parse-name open-lib dup 0= err-nolib and throw ,
-  does> ( -- lib )
-    @ ;
-
-\ test
-
-\ test all parameter and return types
-
-\ cr .( #include "engine/libcc.h")
-\ cr .( #include <unistd.h>)
-\ cr ." typedef void (* func)(int);
-\ cr ." int test1(int,char*,long,double,void (*)(int));"
-\ cr ." Cell *test2(void);"
-\ cr ." int test3(void);"
-\ cr ." float test4(void);"
-\ cr ." func test5(void);"
-\ cr ." void test6(void);"
-\ cr
-
-\ c-function dlseek lseek n d n -- d
-\ c-function n test1 n a d r func -- n
-\ c-function a test2 -- a
-\ c-function d test3 -- d
-\ c-function r test4 -- r
-\ c-function func test5 -- func
-\ c-function void test6 -- void
-
-c-function strlen strlen a -- n
-
-cr s\" fooo\0" 2dup dump drop .s strlen cr .s cr
+:noname ( -- )
+    defers 'cold
+    init-libcc ;
+is 'cold