(de tokenizer (String)
   # [\s,]*(~@|[\[\]{}()'`~^@]|"(?:\\.|[^\\"])*"|;.*|[^\s\[\]{}('"`,;)]*)
   (let (Chars (chop String) Toks () Special " []{}()'\"`,;")
      (while Chars
         (let Char (car Chars)
            (cond
               ((or (= Char " ") (= Char ","))
                (pop 'Chars) )
               ((and (= Char "~") (= (cadr Chars) "@"))
                (push 'Toks "~@")
                (pop 'Chars)
                (pop 'Chars) )
               ((index Char (chop "[]{}()'`~^\@"))
                (push 'Toks (pop 'Chars)) )
               ((= Char "\"")
                (let (Tok (list (pop 'Chars)) Char (car Chars) Done NIL)
                   (while (and (not Done) Chars)
                      (push 'Tok (pop 'Chars))
                      (cond
                         ((= Char "\\")
                          (push 'Tok (pop 'Chars))
                          (setq Char (car Chars)) )
                         ((<> Char "\"")
                          (setq Char (car Chars)) )
                         ((= Char "\"")
                          (setq Done T) ) ) )
                   (push 'Toks (pack (flip Tok))) ) )
               ((= Char ";")
                (setq Char (pop 'Chars))
                (while (and Chars (<> Char "\n"))
                   (setq Char (pop 'Chars)) ) )
               ((not (index Char (chop Special)))
                (let (Tok (list (pop 'Chars)) Char (car Chars))
                   (while (and Chars (not (index Char (chop Special))) )
                      (push 'Tok (pop 'Chars))
                      (setq Char (car Chars)) )
                   (push 'Toks (pack (flip Tok))) ) )
               (T
                  (prinl "Error") ) ) ) )
      (flip Toks) ) )

(mapc prinl (tokenizer "`(abc;;foo\ndef ~@ghi)"))

(bye)