tokenizing MAL the hard way pasted by wasamasa on Mon Sep 26 10:16:48 2016
(de tokenizer (String) # [\s,]*(~@|[\[\]{}()'`~^@]|"(?:\\.|[^\\"])*"|;.*|[^\s\[\]{}('"`,;)]*) (let (Chars (chop String) Toks () Special " []{}()'\"`,;") (while Chars (let Char (car Chars) (cond ((or (= Char " ") (= Char ",")) (pop 'Chars) ) ((and (= Char "~") (= (cadr Chars) "@")) (push 'Toks "~@") (pop 'Chars) (pop 'Chars) ) ((index Char (chop "[]{}()'`~^\@")) (push 'Toks (pop 'Chars)) ) ((= Char "\"") (let (Tok (list (pop 'Chars)) Char (car Chars) Done NIL) (while (and (not Done) Chars) (push 'Tok (pop 'Chars)) (cond ((= Char "\\") (push 'Tok (pop 'Chars)) (setq Char (car Chars)) ) ((<> Char "\"") (setq Char (car Chars)) ) ((= Char "\"") (setq Done T) ) ) ) (push 'Toks (pack (flip Tok))) ) ) ((= Char ";") (setq Char (pop 'Chars)) (while (and Chars (<> Char "\n")) (setq Char (pop 'Chars)) ) ) ((not (index Char (chop Special))) (let (Tok (list (pop 'Chars)) Char (car Chars)) (while (and Chars (not (index Char (chop Special))) ) (push 'Tok (pop 'Chars)) (setq Char (car Chars)) ) (push 'Toks (pack (flip Tok))) ) ) (T (prinl "Error") ) ) ) ) (flip Toks) ) ) (mapc prinl (tokenizer "`(abc;;foo\ndef ~@ghi)")) (bye)
More idiomatic revision added by wasamasa on Tue Sep 27 11:31:55 2016
(de tokenizer (String) # [\s,]*(~@|[\[\]{}()'`~^@]|"(?:\\.|[^\\"])*"|;.*|[^\s\[\]{}('"`,;)]*) (let (Chars (chop String) Special " []{}()'\"`,;") (make (while Chars (let Char (pop 'Chars) (cond ((member Char '(" " ",")) # do nothing, whitespace ) ((and (= Char "~") (= (car Chars) "@")) (link "~@") (pop 'Chars) ) # remove @ token ((index Char (chop "[]{}()'`~^\@")) (link Char) ) ((= Char "\"") (link (pack (make (link Char) (let (Char (car Chars) Done NIL) (while (and (not Done) Chars) (link (pop 'Chars)) (cond ((= Char "\\") (link (pop 'Chars)) (setq Char (car Chars)) ) ((<> Char "\"") (setq Char (car Chars)) ) ((= Char "\"") (setq Done T) ) ) ) ) ) ) ) ) ((= Char ";") (while (and Chars (<> Char "\n")) (setq Char (pop 'Chars)) ) ) ((not (index Char (chop Special))) (link (pack (make (link Char) (let Char (car Chars) (while (and Chars (not (index Char (chop Special)))) (link (pop 'Chars)) (setq Char (car Chars)) ) ) ) ) ) ) ) ) ) ) ) ) (mapc prinl (tokenizer "`(abc;;foo\ndef ~@ghi\"ba\\\"r")) (bye)