add #line directive to generated C code to support source debugging

They are not equivalent, but it'll be allright in this
case. Probably. Passes tests at least.

.github/workflows/build.yaml

@@ -0,0 +1,55 @@
+name: Sex CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build-linux:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install chicken
+      run: |
+        wget -N https://code.call-cc.org/releases/5.4.0/chicken-5.4.0.tar.gz
+        tar zxf chicken-5.4.0.tar.gz
+        sudo apt install -y make
+        make -C chicken-5.4.0 PLATFORM=linux
+        sudo make -C chicken-5.4.0 PLATFORM=linux install
+    - name: Install dependencies
+      # FIXME: [project-local deps]: use venv or something
+      # run: make deps
+      run: sudo chicken-install $(cat dependencies.txt)
+    - name: Make sure that sexc builds
+      # FIXME: sexc being built with local .eggs/ libs needs the path at runtime.
+      # Without it there will be `Error: cannot load extension: fmt`.
+      run: make sexc && ./sexc --help
+    - name: Run tests
+      # FIXME: [project-local deps]: use local deps or build with -static
+      # run: make run-tests
+      run: make sex-tests && ./sex-tests
+
+  build-macos:
+
+    runs-on: macos-15
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install chicken
+      run: brew install chicken make
+    - name: Install dependencies
+      # FIXME: [project-local deps]: use venv or something
+      # run: make deps
+      run: chicken-install $(cat dependencies.txt)
+    - name: Make sure that sexc builds
+      # FIXME: sexc being built with local .eggs/ libs needs the path at runtime.
+      # Without it there will be `Error: cannot load extension: fmt`.
+      run: make sexc && ./sexc --help
+    - name: Run tests
+      # FIXME: [project-local deps]: use local deps or build with -static
+      # run: make run-tests
+      run: make sex-tests && ./sex-tests

.gitignore

@@ -0,0 +1,3 @@
+*.o
+sexc
+sex-tests

Makefile

@@ -1,20 +1,21 @@
 CHICKEN_C = csc
+CSC_FLAGS += -K prefix
 
-MODULES = sexc sex-macros sex-modules utils fmt-c
+MODULES = sexc fmt-c fmt-c-writer semen sex-macros sex-modules sex-reader utils
 OBJ = $(MODULES:%=%.o)
 
 sexc: main.o $(OBJ)
-	$(CHICKEN_C) $^ -o $@
+	$(CHICKEN_C) $(CSC_FLAGS) $^ -o $@
 
 main.o: main.scm
-	$(CHICKEN_C) $< -c -o $@
+	$(CHICKEN_C) $(CSC_FLAGS) $< -c -o $@
 
 %.o: %.scm
-	$(CHICKEN_C) $< -e -c -o $@
+	$(CHICKEN_C) $(CSC_FLAGS) $< -e -c -o $@
 
 sex-tests: $(OBJ) tests/*.scm
-	cd ./tests && $(CHICKEN_C) run.scm -c -o sex-tests.o
+	cd ./tests && $(CHICKEN_C) $(CSC_FLAGS) run.scm -c -o sex-tests.o
-	$(CHICKEN_C) $(OBJ) ./tests/sex-tests.o -o sex-tests
+	$(CHICKEN_C) $(CSC_FLAGS) $(OBJ) ./tests/sex-tests.o -o sex-tests
 
 clean:
 	rm -f $(OBJ) sexc sex-tests main.o ./tests/sex-tests.o

Readme.org

@@ -1,6 +1,12 @@
 * The Sex language
+
+#+NAME: the Sex logo
+#+ATTR_HTML: :width 300px
+[[sex.png][file:./sex.png]]
+
 Sex is a S-expressions language. Sex is written in Chicken, which is a
 [[https://call-cc.org][R5RS Scheme]].
+Sex is statically typed, compiled general purpose language.
 
 * Compilation
 First, get yourself a Chicken, then, some Chicken deps. You also will
@@ -12,7 +18,7 @@ to export ~CHICKEN_INSTALL_REPOSITORY~ and ~CHICKEN_REPOSITORY_PATH~
 environment variables. Refer to the documentation for more info:
 https://wiki.call-cc.org/man/5/Extension%20tools#changing-the-repository-location
 
-~chicken-install fmt getopt-long brev-separate test tree srfi-1 srfi-13~
+~chicken-install `cat dependencies.txt`~
 
 ** Compilation
 ~make~
@@ -25,10 +31,11 @@ Options:
  --c-compiler=ARG         Select C compiler. Defaults to value of SEX_CC
                           environment variable, or if it is empty, to cc
  -c, --compile-object     Compile object file instead of executable program
- -E, --preprocess         Emit C code
+ -C, --preprocess         Emit C code
  --public-interface       Get module's public interface
  -h, --help               Show this help
- -m, --macro-expand       Emit macro-expanded Sex code
+ -m, --macro-expand       Emit macro-expanded semantically processed Sex code
+                          (sort of IR). May be useful for debugging
  -o, --output=ARG         Write output to file. Default file name is a.out.
                           If -E or -m options are provided, defaults to stdout
 #+end_src
@@ -47,13 +54,13 @@ An example of Sex source:
 #+begin_src scheme
 (include stdio.h)
 
-(pub fn int main ((int argc) (char **argv))
+(pub fn main ((argc int) (argv  [* const char])) int
   (puts "Hello from Sex!")
-  (var (array char 512) name)
+  (var name [char 512])
   (puts "What is your name?")
-  (scanf "%s" (cast char* &name))
+  (scanf "%s" (cast (& name) (* char)))
   (printf "Hello, %s!\n" name)
-  0)
+  (return 0))
 #+end_src
 
 Compile and run:
@@ -71,9 +78,9 @@ Hello, Alex!
 Just ~(include "Your/Favourite/Library.h")~ and use it as you would
 have is C.
 
-** Auto unkebabification
+*** Auto kebabification
 For hardcore fans of traditional Lisp naming convention,
-Sex offers automatic unkebabification of all symbols, i.e. no more
+Sex offers automatic kebabification of all symbols, i.e. no more
 ugly ~GL_ARRAY_BUFFER~ s in your code, they may be written in their
 proper form: ~GL-ARRAY-BUFFER~.
 
@@ -99,38 +106,37 @@ return Sex code.
 (pub defmacro (list-T type)
   (let ((list-type (cat 'list- type)))
     `(struct ,list-type
-       ((,type value)
+       ((value ,type)
-        ((* ,list-type) next)))))
+        (next (* ,list-type))))))
 
 (list-T int)
 #+end_src
 ->
 #+begin_src scheme
 (struct list_int
- ((int value)
+ ((value int)
-  ((* list_int) next)))
+  (next (* list_int))))
 #+end_src
 
 **** Wrapper for checking return codes
 #+begin_src scheme
 (pub defmacro (check-sdl-return call message ret-code)
-  `((if (< 0 ,call)
+  `(if (< 0 ,call)
        (begin
          (puts ,message)
-         (return ,ret-code)))))
+         (return ,ret-code))))
 
-(pub fn int init ()
+(pub fn init () int
   (check-sdl-return
     (SDL-Init SDL-INIT-VIDEO) "Failed to initialize SDL" 1)
 ...)
 #+end_src
 ->
-#+begin_src c
+#+begin_src scheme
-(%fun int init ()
+(pub fn init () int
   (if (< 0 (SDL_Init SDL_INIT_VIDEO))
-    (%begin (puts "Failed to initialize SDL") (return 1)))
+    (begin (puts "Failed to initialize SDL") (return 1)))
   ...)
-}
 #+end_src
 
 ** Use an established environment for development

dependencies.txt

@@ -0,0 +1 @@
+fmt getopt-long brev-separate test tree srfi-1 srfi-13 srfi-69 matchable

example/fns.sex

@@ -0,0 +1,10 @@
+;;; Prototypes
+(fn puk () void)
+
+(pub fn plak () void)
+
+;;; Functions
+(fn foo () int (return 1))
+
+(pub fn bar ((a int) (b int)) void
+  (printf "%d\n" (+ a b)))

example/hello-world.sex

@@ -1,9 +1,9 @@
 (include stdio.h)
 
-(pub fn int main ((int argc) (char **argv))
+(pub fn main ((argc int) (argv  [* const char])) int
   (puts "Hello from Sex!")
-  (var (array char 512) name)
+  (var name [char 512])
   (puts "What is your name?")
-  (scanf "%s" (cast char* &name))
+  (scanf "%s" (cast (& name) (* char)))
   (printf "Hello, %s!\n" name)
   (return 0))

example/lambdas.sex

@@ -0,0 +1,51 @@
+(include stdio.h)
+
+(fn sum ((a int) (b int)) int
+  (return (+ a b)))
+
+(pub fn main () int
+  (var a int 10)
+  (var b int 20)
+  (var (fn ((int) (int)) int) sum-fn sum)
+
+  (var (fn ((int) (int)) int) sum-lambda
+
+       (lambda ((a int) (b int)) int ()
+         (return (+ a b))))
+
+  (var (fn ((int)) int) sum-lambda-2
+
+       (lambda ((a int)) int ()
+         (return (+ a 20))))
+
+  (printf "Hello from main fn!\n")
+  (printf "We will now perform some function calling.\n")
+
+  (printf "Calling fn ptr: %d\n" (sum-fn a b))
+  (printf "Calling lambda: %d\n" (sum-lambda a b))
+  (printf "Calling other lambda: %d\n" (sum-lambda-2 a))
+  (printf "Calling lambda inplace: %d\n" ((lambda ((a int) (b int)) int ()
+                                            (return (+ a b 100)))
+                                          a b))
+
+  (var (fn ((int)) int) l-1
+       (lambda ((a int)) int ()
+         (var (fn ((int)) int) l-2
+              (lambda ((a int)) int ()
+                (return (+ 60 a))))
+         (return (+ 600 (l-2 a)))))
+  (printf "Calling nested lambdas: %d\n" (l-1 6))
+
+  ;; Not supported yet
+  ;; Closure
+  ;;  (var (fn (fn ((int)) int) ((int))) make-adder
+  ;;       (lambda (fn int ((int a))) ()
+  ;;         (return (lambda int ((int b)) (a)
+  ;;                   (return (+ a b))))))
+  ;;
+  ;;  (var (fn int ((int))) add-10
+  ;;       (make-adder 10))
+  ;;  (var (fn int ((int))) add-20
+  ;;       (make-adder 20))
+  ;;  (printf "Calling closures: %d\n" (add-10 24))
+  (return 0))

example/list.sex

@@ -1,21 +1,21 @@
 (pub defmacro (list-T type)
   (let ((list-type (cat 'list- type)))
     `(struct ,list-type
-       ((,type value)
+       ((value ,type)
-        ((* ,list-type) next)))))
+        (next (* struct ,list-type))))))
 
 (pub defmacro (make-list-T type is-public?)
-  (let ((list-type (cat 'list- type))
+  (let ((list-type (list 'struct (cat 'list- type)))
         (fn-name (cat 'make-list- type)))
-    `(,@(if is-public? '(pub) '()) fn (* ,list-type) ,fn-name ()
+    `(,@(if is-public? '(pub) '()) fn ,fn-name () (* ,list-type)
-      (var (* ,list-type) list (cast (* ,list-type) (malloc (sizeof ,list-type))))
+      (var list (* ,list-type) (cast  (malloc (sizeof ,list-type)) (* ,list-type)))
       (= (-> list next) NULL)
       (return list))))
 
 (pub defmacro (add-value-list-T type is-public?)
-  (let ((list-type (cat 'list- type))
+  (let ((list-type (list 'struct (cat 'list- type)))
         (fn-name (cat 'add-value-list- type)))
-    `(,@(if is-public? '(pub) '()) fn void ,fn-name ((,list-type *list) (,type value))
+    `(,@(if is-public? '(pub) '()) fn ,fn-name ((list * ,list-type) (value ,type)) void
       (while (!= (-> list next) NULL)
         (= list (-> list next)))
       (= (-> list next) (,(cat 'make-list- type)))
@@ -23,25 +23,26 @@
 
 (pub defmacro (length-list-T type is-public?)
   (let ((fn-name (cat 'length-list- type))
-        (list-type (cat 'list- type)))
+        (list-type (list 'struct (cat 'list- type))))
-    `(,@(if is-public? '(pub) '()) fn size-t ,fn-name ((,list-type *list))
+    `(,@(if is-public? '(pub) '()) fn ,fn-name ((list ,(cons '* list-type))) size-t
-      (var size-t n 0)
+      (var n size-t 0)
       (while (!= (-> list next) NULL)
         (= list (-> list next))
         (++ n))
       (return n))))
 
 (pub defmacro (is-empty-list-T type is-public?)
-  `(,@(if is-public? '(pub) '()) fn bool ,(cat 'is-empty-list- type)
+  `(,@(if is-public? '(pub) '()) fn ,(cat 'is-empty-list- type)
-    ((,(cat 'list- type) *list))
+    ((list ,(list '* 'struct (cat 'list- type))))
+    bool
     (return (== (-> list next) NULL))))
 
 (pub defmacro (list-for-each list-type list-var elt-type elt-var what-do)
   (let ((list-var-2 (cat list-var '-2)))
-    `((begin
+    `(begin
-       (var (pointer ,list-type) ,list-var-2 ,list-var)
+      (var ,list-var-2 (* ,list-type) ,list-var)
-       (var ,elt-type ,elt-var (-> ,list-var-2 value))
+      (var ,elt-var ,elt-type (-> ,list-var-2 value))
       (while (!= (-> ,list-var-2 next) NULL)
         ,what-do
         (= ,list-var-2 (-> ,list-var-2 next))
-         (= ,elt-var (-> ,list-var-2 value)))))))
+        (= ,elt-var (-> ,list-var-2 value))))))

example/test-list.sex

@@ -2,20 +2,15 @@
 (include stddef.h)
 (include stdio.h)
 
-(chicken-import srfi-1 brev-separate)                 ; list routines, e.g. fold
-
 (import list)
 
-(chicken-define (imports-test a b c)
-  (fold + 0 (list 1 2 3 a b c)))
-
 (struct foo
-  ((float a-field)
+  ((a-field float)
-   (int b)
+   (b int)
-   ((const char *) c)
+   (c (* const char))
-   ((fn bool ((bool val))) not)))
+   (not (fn ((val bool)) bool))))
 
-(var foo f)
+(var f (struct foo))
 
 (list-T int)
 (make-list-T int #f)
@@ -23,27 +18,27 @@
 (length-list-T int #f)
 (is-empty-list-T int #f)
 
-(extern fn void puk ((int a) (float b)))
+(extern fn puk ((a int) (b float)) void)
-(fn int bar () (return ,(imports-test 10 20 30)))
+(pub fn baz () bool
-(pub fn bool baz () (return true))
+  (return true))
 
-(extern var int i)
+(extern var i int)
-(var int j)
+(var j int)
-(pub var int k)
+(pub var k int)
 
-(pub fn int main ()
+(pub fn main () int
-     (var (* list-int) l (make-list-int))
+     (var l (* struct list-int) (make-list-int))
      (printf "Size of the list: %lu\n" (length-list-int l))
      (add-value-list-int l 3)
      (add-value-list-int l 4)
      (printf "Size of the list: %lu\n" (length-list-int l))
-     (list-for-each list-int l int v
+     (list-for-each (struct list-int) l int v
        (printf "%d " v))
      (printf "\n")
      (printf "Size of the list: %lu\n" (length-list-int l))
-     (printf "%p\n" (cast void* l->next))
+     (printf "%p\n" (cast l->next (* void)))
      (return 0))
 
-(pub fn void print-list (((const list-int) *l))
+(pub fn print-list ((l (* const struct list-int))) void
-     (list-for-each (const list-int) l int v (printf "%d " v))
+     (list-for-each (const struct list-int) l int v (printf "%d " v))
      (printf "\n"))

fmt-c-writer.scm

@@ -0,0 +1,283 @@
+;;; Sex fmt-c output writer
+
+(declare (unit fmt-c-writer)
+         (uses fmt-c
+               semen
+               utils))
+
+(import (chicken string)
+        (chicken syntax)
+        brev-separate
+        fmt
+        matchable
+        regex
+        srfi-1                          ; lists
+        srfi-13                         ; strings
+        srfi-39                         ; parameters
+        tree)
+
+(define (unkebabify sym)
+  (case sym
+    ((-) sym)
+    ((--) sym)
+    ((->) sym)
+    ((-=) sym)
+    (else
+     (string->symbol
+      (string-substitute "-(?!>)" "_"
+                         (symbol->string sym) #t)))))
+
+(define (atom-to-fmt-c atom)
+  (case atom
+    ((fn) '%fun)
+    ((prototype) '%prototype)
+    ((begin) '%block-begin)
+    ((define) '%define)
+    ((pointer) '%pointer)
+    ((array) '%array)
+    ((attribute) '%attribute)
+    ((¤) 'vector-ref)
+    ((include) '%include)
+    ;; uh things we do for c89 compatibility
+    ((bool) 'int)
+    ((true) 1)
+    ((false) 0)
+    (else
+     (if (symbol? atom)
+         (unkebabify atom)
+         atom))))
+
+(define (maybe-unwrap-type type)
+  (if (and (list? type)
+           (= 1 (length type)))
+      (car type)
+      type))
+
+(define (make-field-access form)
+  (assert
+   (= 2 (length form)) "Wrong field access format")
+  (unkebabify
+   (string->symbol
+    (fmt #f (cadr form) (car form)))))
+
+(define (walk-generic-toplevel form)
+  (cond ((atom? form) (atom-to-fmt-c form))
+        ((list? form) (map walk-generic-toplevel form))
+        (else (error "Malformed form " form))))
+
+(define (field-access-form? form)
+  (and (symbol? (car form))
+       (char=? #\. (string-ref (symbol->string (car form)) 0))))
+
+(define (walk-expr form)
+  (match form
+    ((? vector?)
+     (list->vector
+      (walk-expr (vector->list form))))
+    ((? atom?)
+     (atom-to-fmt-c form))
+    ((? field-access-form?)
+     (make-field-access form))
+    (('var . _) (walk-var form))
+    (('cast expr type) (list '%cast
+                             (walk-type type)
+                             (walk-expr expr)))
+    (('enum . _) (walk-enum form))
+    ;; | is problematic... And c-or/bit-or/etc are actually
+    ;; procedures, so we have to call the procedure itself
+    (('c-or . rest) (apply c-or (map walk-expr rest)))
+    (('c-bit-or . rest) (apply c-bit-or (map walk-expr rest)))
+    (('c-bit-or= . rest) (apply c-bit-or= (map walk-expr rest)))
+    (else (map walk-expr form))))
+
+(define (walk-var form)
+  ;; (var a int) -> (%var int a)
+  ;; (var a (const int) 32) -> (%var (const int) a 32)
+  ;; (var b [const char 512]) -> (%var (%array (const char) 512) b)
+  ;; note: [...] is actually (¤ ...) after reading
+  ;; (var c (fn ((int) (float)) void)) -> (%var (%fun void ((int) (float))) c)
+  `(%var
+    ,(walk-type (third form))
+    ,(atom-to-fmt-c (second form))
+    .
+    ,(if (null? (drop form 3))
+         (list)
+         (walk-expr (drop form 3)))      ; optional init expression
+    ))
+
+(define (walk-type form)
+  ;; int -> int
+  ;; (const int) -> const int
+  ;; [const char 512] ->(¤ (const char) 512) -> (%array (const char) 512)
+  ;; [float 8] -> (%array float 8)
+  ;; (* const char) -> (const char *)
+  ;; (const * const * const char) -> (const char * const * const)
+  ;; (fn ((int) (float)) void) -> (%fun void ((int) (float)))
+  (match form
+    (('¤ . array-type)
+     (if (integer? (last array-type))
+         ;; sized array
+         (let* ((type-list (drop-right array-type 1))
+                (type (maybe-unwrap-type type-list))
+                (size (last array-type)))
+           `(%array ,(walk-type type)
+                    ,size))
+         ;; sugar for pointer... Do we really need it? Guess why not,
+         ;; it's a strong semantic cue
+         `(%array ,(walk-type (maybe-unwrap-type array-type)))))
+    (('fn arglist ret-type)
+     `(%fun ,(walk-type ret-type) ,(walk-arglist arglist)))
+    (('fn . _)
+     (assert #f "Malformed function type form"))
+
+    ;; Special case: nested structs/unions
+    ((or ('struct . _)
+         ('union . _)) (walk-struct form))
+
+    (('enum . _) (walk-enum form))
+    (else
+     (type-convert-to-c form))))
+
+(define (type-convert-to-c type)
+  ;; Our pointers to C pointers
+  ;; int -> int
+  ;; * const char -> const char *
+  ;; const * const char -> const char * const
+  (if (atom? type) (atom-to-fmt-c type)
+      (flatten
+       (tree-map atom-to-fmt-c
+                 (flatten
+                  (list-join (reverse (list-split type '*))
+                             '(*)))))))
+
+(define (walk-fn-def form)
+  (match form
+    (('fn name args ret-type . maybe-body)
+     `(%fun
+       ,(walk-type ret-type)
+       ,(atom-to-fmt-c name)
+       ,(walk-arglist args)
+       .
+       ,(walk-expr maybe-body)))))
+
+;;; TODO: isn't there a better way?
+(define (is-probably-type form)
+  (case (car form)
+    ((¤ * const volatile struct union) #t)
+    (else #f)))
+
+(define (walk-arglist form)
+  ;; E.g.:
+  ;; ((float) (int) (const char) (* const char) (¤ (* const struct res) 32))
+  ;; ((f1 float) (f2 float) (f3 float) (res (¤ float 4)))
+  (map (fn
+        (match x
+          (('¤ . _) (walk-type x))
+
+          ;; yeah shitty, but I don't know yet how to determine if the
+          ;; first entry is part of the type and not an argument name
+          ;; :(
+          ((? is-probably-type) (walk-type x))
+
+          ;; 1 element args are always type
+          ((_) (walk-type x))
+
+          ((var . type) (append (list (walk-type (maybe-unwrap-type type)))
+                                (list (walk-type var))))))
+       form))
+
+(define (walk-function form)
+  ;; (fn ret-type name arglist body) -> normal function
+  ;; (fn ret-type name arglist) -> prototype
+  (if (>= (length form) 5)
+      (walk-fn-def form)
+      (cons '%prototype (cdr (walk-fn-def form)))))
+
+(define (process-struct-fields fields)
+  (map (fn
+        (let ((type (walk-type (last x))))
+          (cons type (map atom-to-fmt-c (drop-right x 1)))))
+       fields))
+
+(define (walk-struct form)
+  (match form
+    ((type (fields ...) . attrs)        ; anonymous struct
+     `(,type ,(process-struct-fields fields)
+             . ,(tree-map atom-to-fmt-c attrs)))
+    ((type name)                        ; simple 'struct whatever', like in variable def
+     `(,type ,(atom-to-fmt-c name)))
+    ((type name (fields ...) . attrs)
+     `(,type ,(atom-to-fmt-c name)
+             ,(process-struct-fields fields)
+             . ,(tree-map atom-to-fmt-c attrs)))
+    (else (error "Malformed aggregate definition " form))))
+
+(define (walk-enum form)
+  (match form
+    (('enum (values ...))
+     `(enum ,(map atom-to-fmt-c values)))
+    (('enum name (values ...))
+     `(enum ,(atom-to-fmt-c name) ,(map atom-to-fmt-c values)))))
+
+(define (walk-extern form)
+  (match form
+    (('fn . _)
+     ;; extern function?.. What
+     (list 'extern (walk-function form)))
+    (('var . _)
+     (list 'extern (walk-var form)))
+    (else (error "Extern what?"))))
+
+(define (walk-public form)
+  (match form
+    (('fn . _)
+     (walk-function form))
+    (('var . _)
+     (walk-var form))
+    ((or ('define . _)
+         ('defmacro . _)
+
+         ('import . _)
+         ('include . _)
+
+         ('struct . _)
+         ('union . _)
+
+         ('typedef . _))
+     ;; ignore here, used in generating public interface
+     (process-toplevel-form form))
+    (else
+     (error "Pub what?" (cadr form)))))
+
+(define (process-toplevel-form form)
+  (match form
+    (('fn . _) (list 'static (walk-function form)))
+    (('var . _) (list 'static (walk-var form)))
+    (('extern . rest) (walk-extern rest))
+    (('pub . rest) (walk-public rest))
+    ((or ('struct . _)
+         ('union . _)) (walk-struct form))
+    (('enum . _) (walk-enum form))
+    (('define name . rest) `(%define ,(atom-to-fmt-c name) ,@rest))
+    (else (walk-expr form))))
+
+(define (get-line-num form)
+  (let ((num (get-line-number form)))
+    (if (string? num)
+        (last (string-split num ":"))
+        #f)))
+
+(define sex-fmt-current-file (make-parameter "/dev/null"))
+(define sex-fmt-line-num (make-parameter 0))
+
+(define (line-directive-string)
+  (fmt #f #\# "line " (sex-fmt-line-num) " " #\" (sex-fmt-current-file) #\"))
+
+(define (emit-c sex-forms)
+  (for-each (lambda (form)
+              (let ((start-line (get-line-num form)))
+                (when start-line
+                  (sex-fmt-line-num start-line)
+                  (fmt #t (line-directive-string) nl)))
+              (fmt #t (c-expr (process-toplevel-form form)) nl))
+            sex-forms))

fmt-c.scm

@@ -9,6 +9,7 @@
 (declare (unit fmt-c))
 
 (import fmt
+        srfi-1
         srfi-13)
 
 (define (fmt-in-macro? st) (fmt-ref st 'in-macro?))
@@ -84,7 +85,8 @@
 (define (c-maybe-paren op x)
   (lambda (st)
     ((fmt-let 'op op
-              (if (c-op<= (fmt-op st) op)
+              (if (and (c-op<= (fmt-op st) op)
+                       (not (vector? st)))
                   (c-paren x)
                   x))
      st)))
@@ -557,10 +559,13 @@
 ;; data structures
 
 (define (c-struct/aux type x . o)
+  ;; can be just pointer to SUC, need to support such case:
+  ;; struct whatever * - body is just '*'
   (let* ((name (if (null? o) (if (or (symbol? x) (string? x)) x #f) x))
          (body (if name (if (not (null? o)) (car o) '()) x))
          (o (if (null? o) o (cdr o))))
-    (if (not (null? body))
+    (if (and (not (null? body))
+             (not (eq? '* body)))
         (c-wrap-stmt
          (cat
           (c-braced-block
@@ -572,7 +577,13 @@
                  (c-wrap-stmt (c-expr body))))))
           (if (pair? o) (cat " " (apply c-begin o)) (dsp ""))))
         (c-wrap-stmt
-         (cat type (if (and name (not (equal? name ""))) (cat " " name) ""))))))
+         (cat type
+              (if (and name (not (equal? name "")))
+                  (cat " " name)
+                  "")
+              (if (not (null? body))
+                  (cat body)
+                  ""))))))
 
 (define (c-struct . args) (apply c-struct/aux "struct" args))
 (define (c-union . args) (apply c-struct/aux "union" args))
@@ -597,9 +608,11 @@
 
 (define (c-while check . body)
   (c-reset-newline
+   (if (null? body)
+       (cat "while (" (c-in-test (c-expr check)) ");")
        (cat (c-block (cat "while (" (c-in-test (c-expr check)) ")")
                      (c-in-stmt (apply c-begin body)))
-        fl)))
+            fl))))
 
 (define (c-for init check update . body)
   (c-reset-newline
@@ -614,7 +627,7 @@
 (define (c-param x)
   (cond
     ((procedure? x) x)
-    ((pair? x) (c-type (car x) (cadr x)))
+    ((pair? x) (c-param-type (car x) (cadr x)))
     (else (error "missing type" x))))
 
 (define (c-field x)
@@ -688,6 +701,33 @@
      (else
       (cat (if (eq? '%pointer type) '* type) (if name (cat " " name) ""))))))
 
+(define (c-param-type type . o)
+  (let ((name (and (pair? o) (car o))))
+    (cond
+     ((pair? type)
+      (case (car type)
+        ((%fun)
+         (cat (c-type (cadr type) #f)
+              " (*" (or name "") ")("
+              (fmt-join (lambda (x) (c-type x #f)) (caddr type) ", ") ")"))
+        ((%array)
+         (let ((name (cat name "[" (if (pair? (cddr type))
+                                       (c-expr (caddr type))
+                                       "")
+                          "]")))
+           (c-type (cadr type) name)))
+        ((%pointer *)
+         (let ((name (cat "*" (if name (c-expr name) ""))))
+           (c-type (cadr type)
+                   (if (and (pair? (cadr type)) (eq? '%array (caadr type)))
+                       (c-paren name)
+                       name))))
+        (else (fmt-join/last c-expr (lambda (x) (c-type x name)) type " "))))
+     ((not type)
+      (lambda (st) ((c-type (or (fmt-default-type st) 'int) name) st)))
+     (else
+      (cat (if (eq? '%pointer type) '* type) (if name (cat " " name) ""))))))
+
 (define (c-var type name . init)
   (c-wrap-stmt
    (if (pair? init)

semen.scm

@@ -0,0 +1,235 @@
+;;; Sex semantic engine
+
+(declare (unit semen)
+         (uses sex-macros
+               sex-modules
+               utils))
+
+(import
+  (chicken string)
+  fmt
+  matchable                             ; pattern matching
+  srfi-1                                ; list routines
+  srfi-69                               ; hash tables
+  )
+
+;;; for lambda extraction, docstring processing, macro expansion,
+;;; injection of module headers, i.e. all things that rearrange code
+;;; structurally, add or remove forms
+;;;
+;;; The algorithm: feed toplevel forms to appropriate handlers, then
+;;; append their return to the resulting list. Each handler can return
+;;; multiple forms, e.g. lambdas collected from a function may result
+;;; in auxiliary structures and functions.
+(define (semen-process raw-sex-forms)
+  (semen-process-rec raw-sex-forms (list)))
+
+(define (semen-process-rec forms acc)
+  (cond
+   ((null? forms) (reverse acc))
+   ((sex-macro? (car forms))
+    (semen-process-rec
+     (semen-apply-macro (car forms) (cdr forms))
+     acc))
+   (else
+    (semen-process-rec (cdr forms)
+                       (match-sex-form (car forms) acc)))))
+
+(define (semen-apply-macro macro-form rest-forms)
+;; We want to replace macro with its expansion. The problem is,
+;; top-level macro can return either a single form, or a list of
+;; forms, when it for example generates some aux
+;; structures/functions/typedefs.
+;;
+;; Single form we just cons to the top of rest-forms, but multiple
+;; forms have to be appended to the rest-forms.
+  (let ((res (apply-macro macro-form)))
+    (if (list? (car res))
+        (append res rest-forms)
+        (cons res rest-forms))))
+
+(define (match-sex-form sex-form acc)
+  (match sex-form
+    ((or ('fn . _)
+         ('pub 'fn . _)
+         ('extern 'fn . _)) (process-fn sex-form acc))
+    ((or ('struct . _)
+         ('pub 'struct . _)) (process-struct sex-form acc))
+    ((or ('union . _)
+         ('pub 'union . _)) (process-struct sex-form acc))
+    ((or ('enum . _)
+         ('pub 'enum . _)) (process-struct sex-form acc))
+    ((or ('var . _)
+         ('pub 'var . _)
+         ('extern 'var . _)) (process-global-var sex-form acc))
+    (('include _) (cons sex-form acc))
+    (('define . _) (cons sex-form acc))
+
+    (('import . modules)
+     (semen-process-imports (get-public-forms modules) acc))
+
+    ((or ('defmacro . rest)
+         ('pub 'defmacro . rest)) (defmacro rest) acc)
+
+    ((or ('typedef new-type target)
+         ('pub 'typedef new-type target))
+     (process-typedef new-type target acc))
+
+    (else (assert #f (fmt #f "Unknown top level form " sex-form)))))
+
+(define (semen-process-imports module-public-forms acc)
+  ;; Recursively process imports: register public macros, cons all
+  ;; other public things to our acc
+  (if (null? module-public-forms) acc
+      (match (car module-public-forms)
+        (('defmacro . rest)
+         (defmacro rest)
+         (semen-process-imports (cdr module-public-forms) acc))
+        (else
+         (semen-process-imports (cdr module-public-forms)
+                                (cons (car module-public-forms)
+                                      acc))))))
+
+(define (semen-macro-expand form)
+  "Walk the form recursively and expand all macros, until none is left."
+  (semen-walk-form
+   form
+   (lambda (subform env)
+     (if (sex-macro? subform)
+         (cons semen-walk-embed-result (semen-apply-macro subform (list)))
+         subform))
+   #f))
+
+;;; semen-walk-form and friends: form walker with various abilities.
+;;; By default, replaces walked form with walk-fn result But may
+;;; perform additional operations depending of what the walk function
+;;; has requested.
+
+;;; For inspiration, see SBCL's walk.lisp and their template
+;;; system.
+
+(define semen-walk-embed-result (gensym)
+  ;; For cases when result is a list which must be embedded in the
+  ;; form, e.g. when it returned from a macro
+  )
+
+(define (semen-walk-form form walk-fn env)
+  (if (atom? form) form
+      (let ((new-form (walk-fn form env)))
+        (cond ((not (eq? form new-form))
+               (semen-walk-form new-form walk-fn env))
+              (else
+               (let ((new-car (semen-walk-form (car new-form) walk-fn env))
+                     (new-cdr (semen-walk-form (cdr new-form) walk-fn env)))
+                 (cond ((and (pair? new-car)
+                             (eq? (car new-car) semen-walk-embed-result))
+                        (append (cdr new-car) new-cdr))
+                       (else
+                        (recons new-form new-car new-cdr)))))))))
+
+;;; Typdef
+
+(define (process-typedef new-type target acc)
+  (cons `(typedef ,target ,new-type) acc))
+
+;;; Fn processing
+
+(define (process-fn sex-fn acc)
+  (let* ((expanded (semen-macro-expand sex-fn))
+         (env (make-hash-table))
+         (processed
+          (semen-walk-form
+           expanded
+           semen-fn-walker
+           (begin
+             (set! (hash-table-ref env :fn-name) (sex-fn-name expanded))
+             (set! (hash-table-ref env :lambda-counter) 0)
+             (set! (hash-table-ref env :lambda-aux-code) (list))
+             env))))
+
+    (cons processed
+          (append (hash-table-ref env :lambda-aux-code) acc))))
+
+(define (semen-fn-walker form env)
+  (if (eq? 'lambda (car form))
+      (let ((lambda-name (semen-make-lambda-name (hash-table-ref env :fn-name)
+                                                 (hash-table-ref env :lambda-counter))))
+        (set! (hash-table-ref env :lambda-aux-code)
+          (append (semen-make-aux-lambda-struct lambda-name form)
+                  (hash-table-ref env :lambda-aux-code)))
+        (set! (hash-table-ref env :lambda-counter)
+          (+ (hash-table-ref env :lambda-counter) 1))
+        lambda-name)
+    form))
+
+(define (semen-make-lambda-name enclosing-fn-name counter)
+  (string->symbol
+   (fmt #f "__lambda_" counter "_" enclosing-fn-name)))
+
+(define (semen-make-aux-lambda-struct name form)
+  (match form
+    (('lambda ret-type arglist captures . body)
+     ;; Captures are ignored for now, but
+     ;; we'll need them for TODO: closures support
+     (process-fn `(fn ,ret-type ,name ,arglist ,@body) (list)))
+    (else (assert #f (fmt #f "Malformed lambda " form)))))
+
+;;; Structs
+
+(define (process-struct sex-struct acc)
+  (cons sex-struct acc))
+
+(define (process-global-var sex-var acc)
+  (cons sex-var acc))
+
+;;; Utils
+(define (non-empty-list? form)
+  (and (list? form)
+       (not (null? form))))
+
+(define (sex-fn? form)
+  "The `form` must be toplevel.
+Returns #f if the form is not a function, returns the form otherwise"
+  (match form
+    ((fn . _) form)
+    ((pub fn . _) form)
+    (else #f)))
+
+(define (sex-fn-public? fn-form)
+  (eq? (car fn-form) 'pub))
+
+(define (sex-fn-return-type fn-form)
+  (assert (sex-fn? fn-form)
+          (fmt #f "Form " fn-form " is not a function"))
+  (if (sex-fn-public? fn-form)
+      (third fn-form)
+      (second fn-form)))
+
+(define (sex-fn-name fn-form)
+  (assert (sex-fn? fn-form)
+          (fmt #f "Form " fn-form " is not a function"))
+  (if (sex-fn-public? fn-form)
+      (fourth fn-form)
+      (third fn-form)))
+
+(define (sex-fn-arglist fn-form)
+  (assert (sex-fn? fn-form)
+          (fmt #f "Form " fn-form " is not a function"))
+  (if (sex-fn-public? fn-form)
+      (fifth fn-form)
+      (fourth fn-form)))
+
+(define (sex-fn-prototype fn-form)
+  "Returns all except body"
+  (assert (sex-fn? fn-form)
+          (fmt #f "Form " fn-form " is not a function"))
+  (if (sex-fn-public? fn-form)
+      (take fn-form 5)
+      (take fn-form 4)))
+
+(define (sex-fn-body fn-form)
+  (assert (sex-fn? fn-form)
+          (fmt #f "Form " fn-form " is not a function"))
+  (if (sex-fn-public? fn-form)
+      (drop fn-form 5)
+      (drop fn-form 4)))

sex-macros.scm

@@ -19,11 +19,17 @@
 (define (get-macro name)
   (eval (get name 'sex-macro)))
 
-(define (macro? form)
+(define (sex-macro? form)
   (and (list? form)
        (symbol? (car form))
        (get (car form) 'sex-macro)))
 
+(define (apply-macro form)
+  (assert (sex-macro? form)
+          (fmt #f (car form) " is not a macro"))
+  (apply (get-macro (car form))
+         (cdr form)))
+
 (define (defmacro form)
   (let ((arglist (car form))
         (body (cdr form)))

sex-modules.scm

@@ -1,7 +1,8 @@
 ; Why `sex-modules`? Probably `modules` unit is reserved by chicken,
 ; things start to break.
 (declare (unit sex-modules)
-         (uses utils))
+         (uses sex-reader
+               utils))
 
 (import brev-separate
         (chicken file)
@@ -14,7 +15,7 @@
 
 (define +persistent-module-paths+ (list))
 
-(define (import-modules module-list)
+(define (get-public-forms module-list)
   ;; Module list is a list of symbols
   ;; How Sex handles modules:
   ;; For each module in a list, construct path, find module by path in
@@ -63,6 +64,7 @@
      (list)
      raw-forms)))
 
+;;; TODO: use semen facilities to analyze modules
 (define (process-public-interface-form form acc)
   (case (car form)
     ((pub)

sex-reader.scm

@@ -0,0 +1,63 @@
+(declare (unit sex-reader))
+
+(include "utils.macros.scm")
+
+(import
+  (chicken base)
+  (chicken io)
+  (chicken pathname)
+  (chicken port)
+  (chicken read-syntax)
+  (chicken string)
+  (chicken syntax)
+  brev-separate
+  fmt)
+
+(define (read-forms acc)
+  (let ((r (read-with-source-info (current-input-port))))
+    (if (eof-object? r) (reverse acc)
+        (read-forms (cons r acc)))))
+
+(define (read-from-file file)
+  (with-directory file
+    (with-input-from-file (pathname-strip-directory file)
+      (fn (read-forms (list))))))
+
+(define (read-bracket port)
+  (let loop ((c (read-char port))
+             (str (string)))
+    (cond ((char=? c #\])
+           (cons '¤
+                 (with-input-from-string str
+                   (fn (port-map identity read)))))
+          ((char=? c #\[)
+           (loop port (conc )))
+          (else
+           (loop (read-char port)
+                 (conc str c))))))
+
+(define open-bracket-counter (make-parameter 0))
+
+(define (read-raw-forms input-source)
+  (let ((bracket-end (gensym)))
+   (set-read-syntax!
+    #\]
+    (lambda (port)
+      (when (= 0 (open-bracket-counter))
+        (error "Unmatched closing bracket"))
+      (open-bracket-counter (- (open-bracket-counter) 1))
+      bracket-end))
+
+   (set-read-syntax!
+    #\[
+    (lambda (port)
+      (open-bracket-counter (+ (open-bracket-counter) 1))
+      (let loop ((r (read port))
+                 (acc (list)))
+        (if (eq? r bracket-end)
+            (cons '¤ (reverse acc))
+            (loop (read port)
+                  (cons r acc)))))))
+  (if (eq? input-source 'stdin)
+      (read-forms (list))
+      (read-from-file input-source)))

sexc.scm

@@ -1,207 +1,24 @@
 (declare (unit sexc)
-         (uses fmt-c
+         (uses fmt-c-writer
-               sex-macros
+               sex-reader
-               sex-modules))
+               utils
+               semen))
 
 (include "utils.macros.scm")
 
 (import brev-separate
         (chicken file)
-        (chicken pathname)
         (chicken plist)
         (chicken pretty-print)
         (chicken process)
         (chicken process-context)
         (chicken port)
-        (chicken string)
         fmt
         getopt-long
-        regex
         srfi-1                          ; list routines
-        srfi-13                         ; string routines
+        srfi-13
         tree)
 
-(define (unkebabify sym)
-  (case sym
-    ((-) sym)
-    ((--) sym)
-    ((->) sym)
-    ((-=) sym)
-    (else
-     (string->symbol
-      (string-substitute "-(?!>)" "_"
-                         (symbol->string sym) #t)))))
-
-(define (atom-to-fmt-c atom)
-  (case atom
-    ((fn) '%fun)
-    ((prototype) '%prototype)
-    ((var) '%var)
-    ((begin) '%block-begin)
-    ((define) '%define)
-    ((pointer) '%pointer)
-    ((array) '%array)
-    ((attribute) '%attribute)
-    ((@) 'vector-ref)
-    ((include) '%include)
-    ((cast) '%cast)
-    ;; uh things we do for c89 compatibility
-    ((bool) 'int)
-    ((true) 1)
-    ((false) 0)
-    (else
-     (if (symbol? atom)
-         (unkebabify atom)
-         atom))))
-
-(define (make-field-access form)
-  (assert (= 2 (length form)) "Wrong field access format")
-  (unkebabify
-   (string->symbol
-    (fmt #f (cadr form) (car form)))))
-
-(require-library chicken-syntax)
-
-(define (walk-generic form acc)
-  (cond
-   ((null? form) (cons '() acc))
-
-   ;; vector, e.g. {}-initializer
-   ((vector? form)
-    (cons
-     (list->vector
-      (car (walk-sex-tree (vector->list form) (list))))
-     acc))
-
-   ;; atom (hopefully)
-   ((not (list? form)) (cons (atom-to-fmt-c form) acc))
-
-   ;; special case - replace unquote with its expansion
-   ((eq? (car form) 'unquote)
-    (fold
-     cons
-     acc
-     (car                               ; bc walk-sex-tree always
-                                        ; wraps its result
-      (walk-sex-tree (eval (cadr form)) (list)))))
-
-   ;; another special case - field access
-   ((and (symbol? (car form))
-         (char=? #\. (string-ref (symbol->string (car form)) 0)))
-    (cons (make-field-access form) acc))
-
-   ;; another special case - macro
-   ((macro? form)
-    (append (fold-right
-             walk-generic
-             (list)
-             (apply (get-macro (car form)) (cdr form)))
-            acc))
-
-   ;; toplevel, or a start of a regular list form
-   (else
-    (let ((new-acc (list)))
-      (cons (fold-right
-             walk-generic
-             new-acc
-             form)
-            acc)))))
-
-(define (normalize-fn-form form)
-  ;; (fn ret-type name arglist body) -> normal function
-  ;; (fn ret-type name arglist) -> prototype
-  (if (>= (length form) 5)
-      form
-      (cons 'prototype (cdr form))))
-
-(define (walk-function form static acc)
-  (if static
-      (append (walk-generic (list 'static (normalize-fn-form form))
-                            (list))
-              acc)
-      (append (walk-generic (normalize-fn-form (cdr form))
-                            (list))
-              acc)))
-
-(define (walk-struct form acc)
-  (let ((name (unkebabify (cadr form))))
-    (append (walk-generic form (list))
-          (cons `(typedef struct ,name ,name) acc))))
-
-(define (walk-extern form acc)
-  (case (cadr form)
-    ((fn)
-     (append
-      (list (cons 'extern (walk-function form #f (list))))
-      acc))
-    ((var)
-     (append
-      (list (cons 'extern (walk-generic (cdr form) (list))))
-      acc))
-    (else (error "Extern what?"))))
-
-(define (walk-public form acc)
-  (case (cadr form)
-    ((fn)
-     (walk-function form #f acc))
-    ((var)
-     (append (walk-generic (list 'static (cdr form)) (list)) acc))
-    ((define defmacro import include struct typedef union var)
-     ;; ignore here, used in generating public interface
-     (process-form (cdr form) acc))
-    (else
-     (error "Pub what?" (cadr form)))))
-
-(define (walk-sex-tree form acc)
-  (if (list? form)
-      (if (macro? form)
-          (fold-right (fn (walk-sex-tree x y))
-                      acc
-                      (list (apply (get-macro (car form)) (cdr form))))
-          (case (car form)
-            ((fn) (walk-function form #t acc))
-            ((extern) (walk-extern form acc))
-            ((pub) (walk-public form acc))
-            ((struct union) (walk-struct form acc))
-            ((unquote) (fold (fn (walk-sex-tree x y))
-                             acc
-                             (eval (cadr form))))
-            (else (append (walk-generic form (list)) acc))))
-      ;; only for unquote support
-      (list (list (atom-to-fmt-c form)))))
-
-(define (process-form form acc)
-  (case (car form)
-    ((chicken-define) (eval (cons 'define (cdr form))) acc)
-    ((defmacro) (defmacro (cdr form)) acc)
-    ((chicken-load)
-     (load (cadr form)) acc)
-    ((chicken-import)
-     (eval (cons 'import (cdr form))) acc)
-    ((import)
-     (append (process-raw-forms
-
-              (import-modules (cdr form)) (list))
-             acc))
-    (else
-     (walk-sex-tree form acc))))
-
-(define (process-raw-forms raw-forms acc)
-  (if (null? raw-forms)
-      (reverse acc)
-      (process-raw-forms (cdr raw-forms)
-                         (process-form (car raw-forms) acc))))
-
-(define (read-forms acc)
-  (let ((r (read)))
-    (if (eof-object? r) (reverse acc)
-        (read-forms (cons r acc)))))
-
-(define (emit-c forms)
-  (for-each (lambda (form)
-              (fmt #t (c-expr form) nl))
-            forms))
-
 ;;; Main function facilities
 
 (define opts-grammar
@@ -214,10 +31,10 @@
                       (required #f)
                       (value #f)
                       (single-char #\c))
-      (preprocess "Emit C code"
+      (emit-c "Emit C code"
                   (required #f)
                   (value #f)
-                  (single-char #\E))
+                  (single-char #\C))
       (public-interface "Get module's public interface"
                         (required #f)
                         (value #f))
@@ -225,7 +42,7 @@
             (required #f)
             (value #f)
             (single-char #\h))
-      (macro-expand "Emit macro-expanded Sex code"
+      (macro-expand "Emit macro-expanded semantically processed Sex code"
                     (required #f)
                     (value #f)
                     (single-char #\m))
@@ -261,20 +78,14 @@
         'stdin
         (car rest-args))))
 
-(define (read-from-file file)
-  (with-directory file
-   (with-input-from-file (pathname-strip-directory file)
-     (fn (read-forms (list))))))
-
 (define (write-to-file-or-stdout output what)
   (if (eq? output 'default)
       (what)
       (with-output-to-file output
         (fn (what)))))
 
-(define (preprocess-or-macroexpand sex-forms output args)
+(define (emit-c-or-sex sex-forms output args)
-  (write-to-file-or-stdout
+  (write-to-file-or-stdout output
-   output
    (lambda ()
     (if (get-arg args 'macro-expand #f)
         (map pp sex-forms)
@@ -289,7 +100,7 @@
                       output)))
     (call-with-values
         (lambda ()
-          (process compiler (append (list "-o" out-file "-std=c89" "-pedantic" "-x" "c")
+          (process compiler (append (list "-o" out-file "-x" "c")
                                     (if (get-arg args 'compile-object #f)
                                         (list "-c")
                                         (list))
@@ -301,13 +112,23 @@
         (close-output-port in-port)
         (process-wait pid)))))
 
-(define (process-input input raw-forms)
+(define (semantic-process-forms raw-forms input-source)
-  (let ((current-dir (current-directory)))
+  (if (eq? input-source 'stdin)
-    (unless (eq? input 'stdin)
+      (semen-process raw-forms)
-      (set-working-directory input))
+      (with-directory input-source
-    (prog1
+        (semen-process raw-forms))))
-     (process-raw-forms raw-forms (list))
+
-     (change-directory current-dir))))
+(define prelude
+  '((include inttypes.h)
+
+    (typedef u8 uint8-t)
+    (typedef i8 int8-t)
+    (typedef u16 uint16-t)
+    (typedef i16 int16-t)
+    (typedef u32 uint32-t)
+    (typedef i32 int32-t)
+    (typedef u64 uint64-t)
+    (typedef i64 int64-t)))
 
 (define (main)
   (let* ((raw-args (command-line-arguments))
@@ -334,14 +155,12 @@
          (return #f))
        (load-persistent-module-paths)
 
-       (let* ((raw-forms
+       (sex-fmt-current-file (to-absolute-pathname input))
-               (if (eq? input 'stdin)
+       (let* ((raw-forms (append prelude (read-raw-forms input)))
-                   (read-forms (list))
+              (sex-forms (semantic-process-forms raw-forms input)))
-                   (read-from-file input)))
-              (sex-forms (process-input input raw-forms)))
          (if (or (get-arg args 'macro-expand #f)
-                 (get-arg args 'preprocess #f))
+                 (get-arg args 'emit-c #f))
-             ;; Preprocess or macroexpand
+             ;; Emit processed and macro-expanded sex code, or emit C code
-             (preprocess-or-macroexpand sex-forms output args)
+             (emit-c-or-sex sex-forms output args)
              ;; Compile file!
              (compile-to-file sex-forms output args)))))))

tests/basic.scm

@@ -1,31 +1,31 @@
-;;; unkebabify
+(test-group "basic"
-(test '- (unkebabify '-))
-(test '-- (unkebabify '--))
-(test '-> (unkebabify '->))
-(test '-= (unkebabify '-=))
-(test 'kebab_case (unkebabify 'kebab-case))
-(test '_what_ (unkebabify '-what-))
-(test 'this->member (unkebabify 'this->member))
-(test '_this_->_member_ (unkebabify '-this-->-member-))
-(test '__->>> (unkebabify '--->>>))
 
-;;; atom-to-fmt-c
+  ;; unkebabify
-(test '%fun (atom-to-fmt-c 'fn))
+  (test '- (unkebabify '-))
-(test '%prototype (atom-to-fmt-c 'prototype))
+  (test '-- (unkebabify '--))
-(test '%var (atom-to-fmt-c 'var))
+  (test '-> (unkebabify '->))
-(test '%block-begin (atom-to-fmt-c 'begin))
+  (test '-= (unkebabify '-=))
-(test '%define (atom-to-fmt-c 'define))
+  (test 'kebab_case (unkebabify 'kebab-case))
-(test '%pointer (atom-to-fmt-c 'pointer))
+  (test '_what_ (unkebabify '-what-))
-(test '%array (atom-to-fmt-c 'array))
+  (test 'this->member (unkebabify 'this->member))
-(test 'vector-ref (atom-to-fmt-c '@))
+  (test '_this_->_member_ (unkebabify '-this-->-member-))
-(test '%include (atom-to-fmt-c 'include))
+  (test '__->>> (unkebabify '--->>>))
-(test '%cast (atom-to-fmt-c 'cast))
 
-;;; c89 stuff
+  ;; atom-to-fmt-c
-(test 'int (atom-to-fmt-c 'bool))
+  (test '%fun (atom-to-fmt-c 'fn))
-(test 1 (atom-to-fmt-c 'true))
+  (test '%prototype (atom-to-fmt-c 'prototype))
-(test 0 (atom-to-fmt-c 'false))
+  (test '%block-begin (atom-to-fmt-c 'begin))
+  (test '%define (atom-to-fmt-c 'define))
+  (test '%pointer (atom-to-fmt-c 'pointer))
+  (test '%array (atom-to-fmt-c 'array))
+  (test 'vector-ref (atom-to-fmt-c '¤))
+  (test '%include (atom-to-fmt-c 'include))
 
-;;; make-field-access
+  ;; c89 stuff
-(test 'a.b (make-field-access '(.b a)))
+  (test 'int (atom-to-fmt-c 'bool))
-(test 'a.b.c (make-field-access '(.c a.b)))
+  (test 1 (atom-to-fmt-c 'true))
+  (test 0 (atom-to-fmt-c 'false))
+
+  ;; make-field-access
+  (test 'a.b (make-field-access '(.b a)))
+  (test 'a.b.c (make-field-access '(.c a.b))))

tests/fmt-c-writer.scm

@@ -0,0 +1,159 @@
+;;; Types
+(test-group "fmt-writer"
+
+  (test
+   '(const int)
+   (walk-type '(const int)))
+
+  (test
+   '(%array (const char) 512)
+   (walk-type '(¤ const char 512)))
+
+  (test
+   '(%array (float) 512)
+   (walk-type '(¤ (float) 512)))
+
+  (test
+   '(%array (const char) 512)
+   (walk-type '(¤ (const char) 512)))
+
+  (test
+   '(%array (const char))
+   (walk-type '(¤ const char)))
+
+  (test
+   '(%array (const char))
+   (walk-type '(¤ (const char))))
+
+  (test
+   '(%array float 8)
+   (walk-type '(¤ float 8)))
+
+  (test
+   "Pointer to const char"
+   '(const char *)
+   (walk-type '(* const char)))
+
+  (test
+   "Const pointer to const char"
+   '(const char * const)
+   (walk-type '(const * const char)))
+
+  (test
+   '(%fun void ((int) (float) (%array (struct what * const))))
+   (walk-type '(fn ((int) (float) (¤ (const * struct what))) void)))
+
+  (test
+   '(%fun void ((int) (%array float) (%array (struct what * const))))
+   (walk-type '(fn ((int) (¤ float) (¤ (const * struct what))) void)))
+
+  (test
+   '(%array (%fun void ((int) (%array float) (%array (struct what * const)))))
+   (walk-type '(¤ (fn ((int) (¤ float) (¤ (const * struct what))) void))))
+
+  ;; Type convert to C
+  (test
+   '(int)
+   (type-convert-to-c '(int)))
+
+  (test
+   '(* int)
+   (type-convert-to-c '(int *)))
+
+  (test
+   '(* const int)
+   (type-convert-to-c '(const int *)))
+
+  (test
+   '(const * const char)
+   (type-convert-to-c '(const char * const)))
+
+;;; Variable defs
+  (test
+   '(%var (%array float 8) a)
+   (walk-var '(var a (¤ float 8))))
+
+  (test
+   '(%var (int *) a (& n))
+   (walk-var '(var a (* int) (& n))))
+
+  (test
+   '(%var (const int *) a (& n))
+   (walk-var '(var a (* const int) (& n))))
+
+  (test
+   '(%var (struct suc) s)
+   (walk-var '(var s (struct suc))))
+
+  (test
+   '(%var (const struct suc *) s s1)
+   (walk-var '(var s (* const struct suc) s1)))
+
+  (test
+   '(%var (const struct suc *) s s1)
+   (walk-var '(var s (* (const struct suc)) s1)))
+
+  (test
+   '(%var (struct suc) s (hoge piyo))
+   (walk-var '(var s (struct suc) (hoge piyo))))
+
+  (test
+   '(%var (struct suc *) s (hoge piyo))
+   (walk-var '(var s (* struct suc) (hoge piyo))))
+
+;;; Fn defs
+  (test
+   '(%fun void puk ((int) (%array float 8)))
+   (walk-fn-def '(fn puk ((int) (¤ float 8)) void)))
+
+  (test
+   '(%fun int main ((int argc) ((%array (const char)) argv))
+          (return 0))
+   (walk-fn-def
+    '(fn main ((argc int) (argv (¤ const char))) int
+         (return 0))))
+  (test
+   '(%fun int quxu (((struct piq *) bar))
+          (return 0))
+   (walk-fn-def
+    '(fn quxu ((bar (* struct piq))) int
+         (return 0))))
+
+  (test
+   '(%fun int quxu (((struct piq *) bar) ((%array (struct tt *)) ploop))
+          (return 0))
+   (walk-fn-def
+    '(fn quxu ((bar (* struct piq)) (ploop (¤ * struct tt))) int
+         (return 0))))
+
+  ;; Structs
+  (test
+   '(struct no_kebab ((int a) (float f)))
+   (walk-struct '(struct no-kebab ((a int) (f float)))))
+
+  (test
+   '(struct settings ((u32 x y w h)
+                      ((%array (struct ((float r g b a))) 4) colors)))
+   (walk-struct
+    '(struct settings
+             ((x y w h u32)
+              (colors [¤ struct ((r g b a float)) 4])))))
+
+  (test
+   '(struct settings ((u32 x y w h)
+                      ((%array (struct color ((float r g b a))) 4) colors)))
+   (walk-struct
+    '(struct settings
+             ((x y w h u32)
+              (colors [¤ struct color ((r g b a float)) 4])))))
+
+  (test
+   '(struct mega_kebab ((int a)
+                        ((struct ((int year) (int month) (int day))) dob)
+                        ((%fun int ((int) (%array int))) min)))
+   (walk-struct '(struct mega-kebab
+                         ((a int)
+                          (dob (struct ((year int)
+                                        (month int)
+                                        (day int))))
+                          (min (fn ((int) (¤ int)) bool)))))))

tests/reader.scm

@@ -0,0 +1,17 @@
+(import (chicken port))
+
+(define-syntax reader-test
+  (syntax-rules ()
+    ((reader-test result string)
+     (test result
+           (with-input-from-string string
+             (lambda () (read-raw-forms 'stdin)))))))
+
+(test-group "reader"
+  ;; []-syntax. For array types and array access expressions
+  (reader-test '((¤ * char)) "[* char]")
+  (reader-test '((¤ * * char const 512)) "[* * char const 512]")
+  (reader-test '((¤)) "[]")
+  (reader-test '((¤ (¤))) "[[]]")
+  (reader-test '((¤ (¤ const char))) "[[const char]]")
+)

tests/run.scm

@@ -1,4 +1,5 @@
-(declare (uses sexc))
+(declare (uses fmt-c-writer
+               semen))
 
 (import
   (chicken process)
@@ -7,7 +8,10 @@
   test)
 
 (include "basic.scm")
-(include "types.scm")
+(include "semen.scm")
+(include "reader.scm")
+(include "fmt-c-writer.scm")
+(include "utils.scm")
 
 ;;; Should be the last in the test suite
 (test-exit)

tests/semen.scm

@@ -0,0 +1,56 @@
+(import srfi-69)
+
+(define print-str-fn
+  '(fn void print-str ((string s))
+       (printf "%s" s)))
+
+(define sum-fn
+  '(pub fn float sum ((int a) (int b))
+        (return (cast float (+ a b)))))
+
+(test-group "semen"
+  (test-assert (sex-fn? print-str-fn))
+  (test #f (sex-fn-public? print-str-fn))
+  (test 'void (sex-fn-return-type print-str-fn))
+  (test 'print-str (sex-fn-name print-str-fn))
+  (test '((string s)) (sex-fn-arglist print-str-fn))
+  (test '(fn void print-str ((string s))) (sex-fn-prototype print-str-fn))
+  (test '((printf "%s" s)) (sex-fn-body print-str-fn))
+
+  (test-assert (sex-fn? sum-fn))
+  (test #t (sex-fn-public? sum-fn))
+  (test 'float (sex-fn-return-type sum-fn))
+  (test 'sum (sex-fn-name sum-fn))
+  (test '((int a) (int b)) (sex-fn-arglist sum-fn))
+  (test '(pub fn float sum ((int a) (int b))) (sex-fn-prototype sum-fn))
+  (test '((return (cast float (+ a b)))) (sex-fn-body sum-fn))
+
+  (let ((sex-code
+         '((defmacro (sum-var name a b c)
+             `(var ,name ,(+ a b c)))
+
+           (sum-var v 1 2 3))))
+
+    (test '((var v 6)) (semen-process sex-code)))
+
+;;; Macro expansion
+
+  (define (form-identity form env)
+    form)
+
+  (test 'a (semen-walk-form 'a form-identity (make-hash-table)))
+  (test '(a b c) (semen-walk-form '(a b c) form-identity (make-hash-table)))
+
+  (test 'a (semen-macro-expand 'a))
+  (test '(a b c) (semen-macro-expand '(a b c)))
+
+  (let ((sex-code-macro
+         '((defmacro (x10 a)
+             `(* 10 ,a))
+
+           (fn void foo ((int a) (int b))
+               (return (+ a (x10 b)))))))
+
+    (test '((fn void foo ((int a) (int b))
+                (return (+ a (* 10 b)))))
+          (semen-process sex-code-macro))))

tests/types.scm

@@ -1 +0,0 @@
-(test "char *" (to-c-type '(%pointer char)))

tests/utils.scm

@@ -0,0 +1,22 @@
+(test-group "utils"
+
+  (test
+   '((1) (2) (3))
+   (list-split '(1 * 2 * 3) '*))
+
+  (test
+   '((1 2 3))
+   (list-split '(1 2 3) '*))
+
+  (test
+   '(() (1) (2) (3) ())
+   (list-split '(* 1 * 2 * 3 *) '*))
+
+  (test
+   '((const) (const struct something))
+   (list-split '(const * const struct something) '*))
+
+  (test
+   '(1 * 2 * 3)
+   (list-join '(1 2 3) '*))
+)

utils.macros.scm

@@ -1,3 +1,5 @@
+(import (chicken process-context))
+
 (define-syntax prog1
   (syntax-rules ()
     ((prog1 form . forms)

utils.scm

@@ -4,7 +4,8 @@
 
 (import
   (chicken pathname)
-  (chicken process-context))
+  (chicken process-context)
+  srfi-1)
 
 (define (get-env-var name)
   (get-environment-variable name))
@@ -17,3 +18,35 @@
         (make-absolute-pathname
          (current-directory)
          (pathname-directory file))))))
+
+(define (to-absolute-pathname pathname)
+  (if (absolute-pathname? pathname)
+      pathname
+      (make-absolute-pathname
+       (current-directory)
+       pathname)))
+
+(define (list-split src-list split-elt)
+  ;; split '(1 2 / 3 4 / 5 6) by '/ -> '((1 2) (3 4) (5 6))
+  (fold (lambda (elt acc)
+          (if (eq? elt split-elt)
+              (append acc (list (list)))
+              (append (drop-right acc 1)
+                      (list (append (last acc) (list elt))))))
+        (list (list))
+        src-list))
+
+(define (list-join lists join-by)
+  (drop-right
+   (fold (lambda (elt acc)
+           (append acc (list elt) (list join-by)))
+         (list)
+         lists)
+   1))
+
+;;; Reconstruct form
+(define (recons old-cons new-car new-cdr)
+  (if (and (eq? new-car (car old-cons))
+           (eq? new-cdr (cdr old-cons)))
+      old-cons
+      (cons new-car new-cdr)))