tokenizing MAL the hard way pasted by wasamasa on Mon Sep 26 10:16:48 2016

(de tokenizer (String)
   # [\s,]*(~@|[\[\]{}()'`~^@]|"(?:\\.|[^\\"])*"|;.*|[^\s\[\]{}('"`,;)]*)
   (let (Chars (chop String) Toks () Special " []{}()'\"`,;")
      (while Chars
         (let Char (car Chars)
            (cond
               ((or (= Char " ") (= Char ","))
                (pop 'Chars) )
               ((and (= Char "~") (= (cadr Chars) "@"))
                (push 'Toks "~@")
                (pop 'Chars)
                (pop 'Chars) )
               ((index Char (chop "[]{}()'`~^\@"))
                (push 'Toks (pop 'Chars)) )
               ((= Char "\"")
                (let (Tok (list (pop 'Chars)) Char (car Chars) Done NIL)
                   (while (and (not Done) Chars)
                      (push 'Tok (pop 'Chars))
                      (cond
                         ((= Char "\\")
                          (push 'Tok (pop 'Chars))
                          (setq Char (car Chars)) )
                         ((<> Char "\"")
                          (setq Char (car Chars)) )
                         ((= Char "\"")
                          (setq Done T) ) ) )
                   (push 'Toks (pack (flip Tok))) ) )
               ((= Char ";")
                (setq Char (pop 'Chars))
                (while (and Chars (<> Char "\n"))
                   (setq Char (pop 'Chars)) ) )
               ((not (index Char (chop Special)))
                (let (Tok (list (pop 'Chars)) Char (car Chars))
                   (while (and Chars (not (index Char (chop Special))) )
                      (push 'Tok (pop 'Chars))
                      (setq Char (car Chars)) )
                   (push 'Toks (pack (flip Tok))) ) )
               (T
                  (prinl "Error") ) ) ) )
      (flip Toks) ) )

(mapc prinl (tokenizer "`(abc;;foo\ndef ~@ghi)"))

(bye)

More idiomatic revision added by wasamasa on Tue Sep 27 11:31:55 2016

(de tokenizer (String)
   # [\s,]*(~@|[\[\]{}()'`~^@]|"(?:\\.|[^\\"])*"|;.*|[^\s\[\]{}('"`,;)]*)
   (let (Chars (chop String) Special " []{}()'\"`,;")
      (make
         (while Chars
            (let Char (pop 'Chars)
               (cond
                  ((member Char '(" " ","))
                   # do nothing, whitespace
                   )
                  ((and (= Char "~") (= (car Chars) "@"))
                   (link "~@")
                   (pop 'Chars) ) # remove @ token
                  ((index Char (chop "[]{}()'`~^\@"))
                   (link Char) )
                  ((= Char "\"")
                   (link
                      (pack
                         (make
                            (link Char)
                            (let (Char (car Chars) Done NIL)
                               (while (and (not Done) Chars)
                                  (link (pop 'Chars))
                                  (cond
                                     ((= Char "\\")
                                      (link (pop 'Chars))
                                      (setq Char (car Chars)) )
                                     ((<> Char "\"")
                                      (setq Char (car Chars)) )
                                     ((= Char "\"")
                                      (setq Done T) ) ) ) ) ) ) ) )
                  ((= Char ";")
                   (while (and Chars (<> Char "\n"))
                      (setq Char (pop 'Chars)) ) )
                  ((not (index Char (chop Special)))
                   (link
                      (pack
                         (make
                            (link Char)
                            (let Char (car Chars)
                               (while (and Chars (not (index Char (chop Special))))
                                  (link (pop 'Chars))
                                  (setq Char (car Chars)) ) ) ) ) ) ) ) ) ) ) ) )

(mapc prinl (tokenizer "`(abc;;foo\ndef ~@ghi\"ba\\\"r"))

(bye)