(de tokenizer (String)
   # [\s,]*(~@|[\[\]{}()'`~^@]|"(?:\\.|[^\\"])*"|;.*|[^\s\[\]{}('"`,;)]*)
   (let (Chars (chop String) Special " []{}()'\"`,;")
      (make
         (while Chars
            (let Char (pop 'Chars)
               (cond
                  ((member Char '(" " ","))
                   # do nothing, whitespace
                   )
                  ((and (= Char "~") (= (car Chars) "@"))
                   (link "~@")
                   (pop 'Chars) ) # remove @ token
                  ((index Char (chop "[]{}()'`~^\@"))
                   (link Char) )
                  ((= Char "\"")
                   (link
                      (pack
                         (make
                            (link Char)
                            (let (Char (car Chars) Done NIL)
                               (while (and (not Done) Chars)
                                  (link (pop 'Chars))
                                  (cond
                                     ((= Char "\\")
                                      (link (pop 'Chars))
                                      (setq Char (car Chars)) )
                                     ((<> Char "\"")
                                      (setq Char (car Chars)) )
                                     ((= Char "\"")
                                      (setq Done T) ) ) ) ) ) ) ) )
                  ((= Char ";")
                   (while (and Chars (<> Char "\n"))
                      (setq Char (pop 'Chars)) ) )
                  ((not (index Char (chop Special)))
                   (link
                      (pack
                         (make
                            (link Char)
                            (let Char (car Chars)
                               (while (and Chars (not (index Char (chop Special))))
                                  (link (pop 'Chars))
                                  (setq Char (car Chars)) ) ) ) ) ) ) ) ) ) ) ) )

(mapc prinl (tokenizer "`(abc;;foo\ndef ~@ghi\"ba\\\"r"))

(bye)