ikarus/scheme/ikarus.reader.ss

1595 lines
61 KiB
Scheme
Raw Permalink Normal View History

;;; Ikarus Scheme -- A compiler for R6RS Scheme.
;;; Copyright (C) 2006,2007,2008 Abdulaziz Ghuloum
;;;
;;; This program is free software: you can redistribute it and/or modify
;;; it under the terms of the GNU General Public License version 3 as
;;; published by the Free Software Foundation.
;;;
;;; This program is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;;; General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with this program. If not, see <http://www.gnu.org/licenses/>.
2006-11-23 19:48:14 -05:00
(library (ikarus.reader)
(export read read-initial read-token comment-handler get-datum
read-annotated read-script-annotated annotation?
annotation-expression annotation-source
annotation-stripped)
2007-05-05 20:47:31 -04:00
(import
(only (ikarus.string-to-number) define-string->number-parser)
(ikarus system $chars)
(ikarus system $fx)
(ikarus system $pairs)
(ikarus system $bytevectors)
(only (ikarus.io) input-port-byte-position
input-port-column-number)
(except (ikarus) read-char read read-token comment-handler get-datum
read-annotated read-script-annotated annotation?
annotation-expression annotation-source annotation-stripped
input-port-column-number))
(define (die/lex id pos who msg arg*)
(raise
(condition
(make-lexical-violation)
(make-message-condition msg)
(if (null? arg*)
(condition)
(make-irritants-condition arg*))
(make-source-position-condition
id pos))))
(define (die/pos p off who msg arg*)
(die/lex (port-id p)
(let ([pos (input-port-byte-position p)])
(and pos (+ pos off)))
who msg arg*))
(define (die/p p who msg . arg*)
(die/pos p 0 who msg arg*))
(define (die/p-1 p who msg . arg*)
(die/pos p -1 who msg arg*))
(define (die/ann ann who msg . arg*)
(let ([src (annotation-source ann)])
(die/lex (car src) (cdr src) who msg arg*)))
(define (checked-integer->char n ac p)
(define (valid-integer-char? n)
(cond
[(<= n #xD7FF) #t]
[(< n #xE000) #f]
[(<= n #x10FFFF) #t]
[else #f]))
(if (valid-integer-char? n)
($fixnum->char n)
(die/p p 'tokenize
"invalid numeric value for character"
(list->string (reverse ac)))))
(define-syntax read-char
(syntax-rules ()
[(_ p) (get-char p)]))
2007-04-29 22:29:42 -04:00
2006-11-23 19:33:45 -05:00
(define delimiter?
(lambda (c)
(or (char-whitespace? c)
(memq c '(#\( #\) #\[ #\] #\" #\# #\; #\{ #\} #\|)))))
2006-11-23 19:33:45 -05:00
(define digit?
(lambda (c)
2006-11-23 19:48:14 -05:00
(and ($char<= #\0 c) ($char<= c #\9))))
2006-11-23 19:33:45 -05:00
(define char->num
(lambda (c)
2006-11-23 19:38:26 -05:00
(fx- ($char->fixnum c) ($char->fixnum #\0))))
2006-11-23 19:33:45 -05:00
(define initial?
(lambda (c)
(cond
[($char<= c ($fixnum->char 127))
(or (letter? c) (special-initial? c))]
[else (unicode-printable-char? c)])))
2006-11-23 19:33:45 -05:00
(define letter?
(lambda (c)
2006-11-23 19:48:14 -05:00
(or (and ($char<= #\a c) ($char<= c #\z))
(and ($char<= #\A c) ($char<= c #\Z)))))
2006-11-23 19:33:45 -05:00
(define special-initial?
(lambda (c)
(memq c '(#\! #\$ #\% #\& #\* #\/ #\: #\< #\= #\> #\? #\^ #\_ #\~))))
(define special-subsequent?
(lambda (c)
(memq c '(#\+ #\- #\. #\@))))
2008-08-07 18:02:53 -04:00
(define subsequent?
(lambda (c)
(cond
[($char<= c ($fixnum->char 127))
(or (letter? c)
(digit? c)
(special-initial? c)
(special-subsequent? c))]
[else
(or (unicode-printable-char? c)
(memq (char-general-category c) '(Nd Mc Me)))])))
2006-11-23 19:33:45 -05:00
(define tokenize-identifier
(lambda (ls p)
(let ([c (peek-char p)])
2006-11-23 19:33:45 -05:00
(cond
[(eof-object? c) ls]
[(subsequent? c)
(read-char p)
(tokenize-identifier (cons c ls) p)]
2006-11-23 19:33:45 -05:00
[(delimiter? c)
ls]
[(char=? c #\\)
(read-char p)
(tokenize-backslash ls p)]
[(eq? (port-mode p) 'r6rs-mode)
(die/p p 'tokenize "invalid identifier syntax"
(list->string (reverse (cons c ls))))]
[else ls]))))
(define (tokenize-string ls p)
(let ([c (read-char p)])
(cond
[(eof-object? c)
(die/p p 'tokenize "invalid eof inside string")]
[else (tokenize-string-char ls p c)])))
2009-06-26 03:07:26 -04:00
(define LF1 '(#\xA #\x85 #\x2028)) ;;; these are considered newlines
(define LF2 '(#\xA #\x85)) ;;; these are not newlines if they
;;; appear after CR
(define (tokenize-string-char ls p c)
(define (intraline-whitespace? c)
2009-06-26 03:07:26 -04:00
(or (eqv? c #\x9)
(eq? (char-general-category c) 'Zs)))
(define (tokenize-string-continue ls p c)
(cond
[(eof-object? c)
(die/p p 'tokenize "invalid eof inside string")]
[(intraline-whitespace? c)
(let f ()
(let ([c (read-char p)])
(cond
[(eof-object? c)
(die/p p 'tokenize "invalid eof inside string")]
[(intraline-whitespace? c) (f)]
[else (tokenize-string-char ls p c)])))]
[else (tokenize-string-char ls p c)]))
(cond
[($char= #\" c) ls]
[($char= #\\ c)
2006-11-23 19:33:45 -05:00
(let ([c (read-char p)])
(cond
[(eof-object? c)
(die/p p 'tokenize "invalid eof after string escape")]
[($char= #\a c) (tokenize-string (cons #\x7 ls) p)]
[($char= #\b c) (tokenize-string (cons #\x8 ls) p)]
[($char= #\t c) (tokenize-string (cons #\x9 ls) p)]
[($char= #\n c) (tokenize-string (cons #\xA ls) p)]
[($char= #\v c) (tokenize-string (cons #\xB ls) p)]
[($char= #\f c) (tokenize-string (cons #\xC ls) p)]
[($char= #\r c) (tokenize-string (cons #\xD ls) p)]
[($char= #\" c) (tokenize-string (cons #\x22 ls) p)]
[($char= #\\ c) (tokenize-string (cons #\x5C ls) p)]
[($char= #\x c) ;;; unicode escape \xXXX;
(let ([c (read-char p)])
(cond
[(eof-object? c)
(die/p p 'tokenize "invalid eof inside string")]
[(hex c) =>
(lambda (n)
(let f ([n n] [ac (cons c '(#\x))])
(let ([c (read-char p)])
(cond
[(eof-object? n)
(die/p p 'tokenize "invalid eof inside string")]
[(hex c) =>
(lambda (v) (f (+ (* n 16) v) (cons c ac)))]
[($char= c #\;)
(tokenize-string
(cons (checked-integer->char n ac p) ls) p)]
[else
(die/p-1 p 'tokenize
"invalid char in escape sequence"
(list->string (reverse (cons c ac))))]))))]
[else
(die/p-1 p 'tokenize
"invalid char in escape sequence" c)]))]
[(intraline-whitespace? c)
(let f ()
(let ([c (read-char p)])
(cond
[(eof-object? c)
(die/p p 'tokenize "invalid eof inside string")]
[(intraline-whitespace? c) (f)]
2009-06-26 03:07:26 -04:00
[(memv c LF1)
(tokenize-string-continue ls p (read-char p))]
2009-06-26 03:07:26 -04:00
[(eqv? c #\return)
(let ([c (read-char p)])
(cond
2009-06-26 03:07:26 -04:00
[(memv c LF2)
(tokenize-string-continue ls p (read-char p))]
[else
(tokenize-string-continue ls p c)]))]
[else
(die/p-1 p 'tokenize
"non-whitespace character after escape")])))]
2009-06-26 03:07:26 -04:00
[(memv c LF1)
(tokenize-string-continue ls p (read-char p))]
2009-06-26 03:07:26 -04:00
[(eqv? c #\return)
(let ([c (read-char p)])
(cond
2009-06-26 03:07:26 -04:00
[(memv c LF2)
(tokenize-string-continue ls p (read-char p))]
2009-06-26 03:07:26 -04:00
[else
(tokenize-string-continue ls p c)]))]
[else (die/p-1 p 'tokenize "invalid string escape" c)]))]
2009-06-26 03:07:26 -04:00
[(memv c LF1)
(tokenize-string (cons #\linefeed ls) p)]
2009-06-26 03:07:26 -04:00
[(eqv? c #\return)
(let ([c (peek-char p)])
2009-06-26 03:07:26 -04:00
(when (memv c LF2) (read-char p))
(tokenize-string (cons #\linefeed ls) p))]
[else
(tokenize-string (cons c ls) p)]))
2006-11-23 19:33:45 -05:00
(define skip-comment
(lambda (p)
(let ([c (read-char p)])
2009-06-26 03:07:26 -04:00
(unless (or (eof-object? c) (memv c LF1) (eqv? c #\return))
(skip-comment p)))))
2006-11-23 19:33:45 -05:00
(define tokenize-dot
(lambda (p)
(let ([c (peek-char p)])
(cond
[(eof-object? c) 'dot]
[(delimiter? c) 'dot]
2006-11-23 19:48:14 -05:00
[($char= c #\.) ; this is second dot
2006-11-23 19:33:45 -05:00
(read-char p)
(let ([c (peek-char p)])
2006-11-23 19:33:45 -05:00
(cond
[(eof-object? c)
(die/p p 'tokenize "invalid syntax .. near end of file")]
2006-11-23 19:48:14 -05:00
[($char= c #\.) ; this is the third
(read-char p)
2006-11-23 19:33:45 -05:00
(let ([c (peek-char p)])
(cond
[(eof-object? c) '(datum . ...)]
[(delimiter? c) '(datum . ...)]
[else
(die/p p 'tokenize "invalid syntax"
(string-append "..." (string c)))]))]
2006-11-23 19:33:45 -05:00
[else
(die/p p 'tokenize "invalid syntax"
(string-append ".." (string c)))]))]
2007-06-14 11:56:47 -04:00
[else
(cons 'datum
(u:dot p '(#\.) 10 #f #f +1))]))))
2006-11-23 19:33:45 -05:00
(define tokenize-char*
(lambda (i str p d)
(cond
[(fx= i (string-length str))
(let ([c (peek-char p)])
(cond
[(eof-object? c) d]
[(delimiter? c) d]
2009-06-26 03:07:26 -04:00
[else
(die/p p 'tokenize "invalid character after sequence"
(string-append str (string c)))]))]
2006-11-23 19:33:45 -05:00
[else
(let ([c (read-char p)])
(cond
[(eof-object? c)
2009-06-26 03:07:26 -04:00
(die/p p 'tokenize
"invalid eof in the middle of expected sequence" str)]
2006-11-23 19:48:14 -05:00
[($char= c (string-ref str i))
2006-11-23 19:33:45 -05:00
(tokenize-char* (fxadd1 i) str p d)]
[else
(die/p-1 p 'tokenize
"invalid char while scanning string"
c str)]))])))
2006-11-23 19:33:45 -05:00
(define tokenize-char-seq
(lambda (p str d)
(let ([c (peek-char p)])
(cond
[(eof-object? c) (cons 'datum (string-ref str 0))]
[(delimiter? c) (cons 'datum (string-ref str 0))]
2006-11-23 19:48:14 -05:00
[($char= (string-ref str 1) c)
2006-11-23 19:33:45 -05:00
(read-char p)
(tokenize-char* 2 str p d)]
2009-06-26 03:07:26 -04:00
[else
(die/p p 'tokenize "invalid syntax"
(string-ref str 0) c)]))))
2006-11-23 19:33:45 -05:00
(define tokenize-char
(lambda (p)
(let ([c (read-char p)])
(cond
[(eof-object? c)
(die/p p 'tokenize "invalid #\\ near end of file")]
2009-06-26 03:07:26 -04:00
[(eqv? #\n c)
(let ([c (peek-char p)])
(cond
[(eof-object? c)
(read-char p)
'(datum . #\n)]
[(eqv? #\u c)
(read-char p)
(tokenize-char-seq p "ul" '(datum . #\x0))]
[(eqv? #\e c)
(read-char p)
(tokenize-char-seq p "ewline" '(datum . #\xA))]
[(delimiter? c)
'(datum . #\n)]
[else
(die/p p 'tokenize "invalid syntax"
(string #\# #\\ #\n c))]))]
[(eqv? #\a c)
(tokenize-char-seq p "alarm" '(datum . #\x7))]
[(eqv? #\b c)
(tokenize-char-seq p "backspace" '(datum . #\x8))]
[(eqv? #\t c)
(tokenize-char-seq p "tab" '(datum . #\x9))]
[(eqv? #\l c)
(tokenize-char-seq p "linefeed" '(datum . #\xA))]
[(eqv? #\v c)
(tokenize-char-seq p "vtab" '(datum . #\xB))]
[(eqv? #\p c)
(tokenize-char-seq p "page" '(datum . #\xC))]
[(eqv? #\r c)
(tokenize-char-seq p "return" '(datum . #\xD))]
[(eqv? #\e c)
(tokenize-char-seq p "esc" '(datum . #\x1B))]
[(eqv? #\s c)
(tokenize-char-seq p "space" '(datum . #\x20))]
[(eqv? #\d c)
(tokenize-char-seq p "delete" '(datum . #\x7F))]
[(eqv? #\x c)
(let ([n (peek-char p)])
(cond
[(or (eof-object? n) (delimiter? n))
'(datum . #\x)]
[(hex n) =>
(lambda (v)
(read-char p)
(let f ([v v] [ac (cons n '(#\x))])
(let ([c (peek-char p)])
(cond
[(eof-object? c)
(cons 'datum (checked-integer->char v ac p))]
[(delimiter? c)
(cons 'datum (checked-integer->char v ac p))]
[(hex c) =>
(lambda (v0)
(read-char p)
(f (+ (* v 16) v0) (cons c ac)))]
[else
(die/p p 'tokenize
"invalid character sequence"
(list->string (reverse (cons c ac))))]))))]
[else
(die/p p 'tokenize "invalid character sequence"
(string-append "#\\" (string n)))]))]
2006-11-23 19:33:45 -05:00
[else
(let ([n (peek-char p)])
(cond
[(eof-object? n) (cons 'datum c)]
[(delimiter? n) (cons 'datum c)]
[else
(die/p p 'tokenize "invalid syntax"
(string-append "#\\" (string c n)))]))]))))
(define (hex x)
(cond
[(and ($char<= #\0 x) ($char<= x #\9))
($fx- ($char->fixnum x) ($char->fixnum #\0))]
[(and ($char<= #\a x) ($char<= x #\f))
($fx- ($char->fixnum x)
($fx- ($char->fixnum #\a) 10))]
[(and ($char<= #\A x) ($char<= x #\F))
($fx- ($char->fixnum x)
($fx- ($char->fixnum #\A) 10))]
[else #f]))
2006-11-23 19:33:45 -05:00
(define multiline-error
(lambda (p)
(die/p p 'tokenize
"end of file encountered while inside a #|-style comment")))
(define apprev
(lambda (str i ac)
(cond
[(fx= i (string-length str)) ac]
[else
(apprev str (fx+ i 1) (cons (string-ref str i) ac))])))
2006-11-23 19:33:45 -05:00
(define multiline-comment
(lambda (p)
(define f
(lambda (p ac)
(let ([c (read-char p)])
(cond
[(eof-object? c) (multiline-error p)]
[($char= #\| c)
(let g ([c (read-char p)] [ac ac])
(cond
[(eof-object? c) (multiline-error p)]
[($char= #\# c) ac]
[($char= #\| c)
(g (read-char p) (cons c ac))]
[else (f p (cons c ac))]))]
[($char= #\# c)
(let ([c (read-char p)])
(cond
[(eof-object? c) (multiline-error p)]
[($char= #\| c)
(let ([v (multiline-comment p)])
(if (string? v)
(f p (apprev v 0 ac))
(f p ac)))]
[else
(f p (cons c (cons #\# ac)))]))]
[else (f p (cons c ac))]))))
(let ([ac (f p '())])
((comment-handler)
(list->string (reverse ac))))))
2006-11-23 19:33:45 -05:00
(define tokenize-hash
(lambda (p)
(tokenize-hash/c (read-char p) p)))
(define (skip-whitespace p caller)
(let ([c (read-char p)])
(cond
[(eof-object? c)
(die/p p 'tokenize "invalid eof inside" caller)]
[(char-whitespace? c)
(skip-whitespace p caller)]
[else c])))
(define tokenize-hash/c
(lambda (c p)
(cond
[(eof-object? c) (die/p p 'tokenize "invalid # near end of file")]
[(memq c '(#\t #\T))
2008-12-08 06:28:41 -05:00
(let ([c1 (peek-char p)])
(cond
2008-12-08 06:28:41 -05:00
[(eof-object? c1) '(datum . #t)]
[(delimiter? c1) '(datum . #t)]
[else (die/p p 'tokenize
2008-12-08 06:28:41 -05:00
(format "invalid syntax near #~a~a" c c1))]))]
[(memq c '(#\f #\F))
2008-12-08 06:28:41 -05:00
(let ([c1 (peek-char p)])
(cond
2008-12-08 06:28:41 -05:00
[(eof-object? c1) '(datum . #f)]
[(delimiter? c1) '(datum . #f)]
[else (die/p p 'tokenize
2008-12-08 06:28:41 -05:00
(format "invalid syntax near #~a~a" c c1))]))]
[($char= #\\ c) (tokenize-char p)]
[($char= #\( c) 'vparen]
[($char= #\' c) '(macro . syntax)]
[($char= #\` c) '(macro . quasisyntax)]
[($char= #\, c)
(let ([c (peek-char p)])
(cond
[(eqv? c #\@) (read-char p)
'(macro . unsyntax-splicing)]
[else '(macro . unsyntax)]))]
[($char= #\! c)
(let ([e (read-char p)])
(when (eof-object? e)
(die/p p 'tokenize "invalid eof near #!"))
(case e
[(#\e)
2007-11-25 16:23:39 -05:00
(when (eq? (port-mode p) 'r6rs-mode)
(die/p-1 p 'tokenize "invalid syntax: #!e"))
(read-char* p '(#\e) "of" "eof sequence" #f #f)
(cons 'datum (eof-object))]
[(#\r)
(read-char* p '(#\r) "6rs" "#!r6rs comment" #f #f)
(set-port-mode! p 'r6rs-mode)
(tokenize/1 p)]
[(#\i)
(read-char* p '(#\i) "karus" "#!ikarus comment" #f #f)
(set-port-mode! p 'ikarus-mode)
(tokenize/1 p)]
[else
(die/p-1 p 'tokenize
(format "invalid syntax near #!~a" e))]))]
[(digit? c)
(when (eq? (port-mode p) 'r6rs-mode)
(die/p-1 p 'tokenize "graph syntax is invalid in #!r6rs mode"
(format "#~a" c)))
(tokenize-hashnum p (char->num c))]
[($char= #\: c)
(when (eq? (port-mode p) 'r6rs-mode)
(die/p-1 p 'tokenize "gensym syntax is invalid in #!r6rs mode"
(format "#~a" c)))
(let* ([c (skip-whitespace p "gensym")]
[id0
(cond
[(initial? c)
(list->string
(reverse (tokenize-identifier (cons c '()) p)))]
[($char= #\| c)
(list->string
(reverse (tokenize-bar p '())))]
[else
(die/p-1 p 'tokenize
"invalid char inside gensym" c)])])
(cons 'datum (gensym id0)))]
[($char= #\{ c)
(when (eq? (port-mode p) 'r6rs-mode)
(die/p-1 p 'tokenize "gensym syntax is invalid in #!r6rs mode"
(format "#~a" c)))
(let* ([c (skip-whitespace p "gensym")]
[id0
(cond
[(initial? c)
(list->string
(reverse (tokenize-identifier (cons c '()) p)))]
[($char= #\| c)
(list->string
(reverse (tokenize-bar p '())))]
[else
(die/p-1 p 'tokenize
"invalid char inside gensym" c)])]
[c (skip-whitespace p "gensym")])
(cond
[($char= #\} c)
(cons 'datum
(foreign-call "ikrt_strings_to_gensym" #f id0))]
[else
(let ([id1
(cond
[(initial? c)
(list->string
(reverse
(tokenize-identifier
(cons c '()) p)))]
[($char= #\| c)
(list->string
(reverse (tokenize-bar p '())))]
[else
(die/p-1 p 'tokenize
"invalid char inside gensym" c)])])
(let ([c (skip-whitespace p "gensym")])
(cond
[($char= #\} c)
(cons 'datum
(foreign-call "ikrt_strings_to_gensym"
id0 id1))]
[else
(die/p-1 p 'tokenize
"invalid char inside gensym" c)])))]))]
[($char= #\v c)
(let ([c (read-char p)])
(cond
[($char= #\u c)
(let ([c (read-char p)])
(cond
[($char= c #\8)
(let ([c (read-char p)])
(cond
[($char= c #\() 'vu8]
[(eof-object? c)
(die/p p 'tokenize "invalid eof object after #vu8")]
[else (die/p-1 p 'tokenize
(format "invalid sequence #vu8~a" c))]))]
[(eof-object? c)
(die/p p 'tokenize "invalid eof object after #vu")]
[else (die/p-1 p 'tokenize
(format "invalid sequence #vu~a" c))]))]
[(eof-object? c)
(die/p p 'tokenize "invalid eof object after #v")]
[else (die/p p 'tokenize
(format "invalid sequence #v~a" c))]))]
2007-06-14 11:56:47 -04:00
[(memq c '(#\e #\E))
(cons 'datum (parse-string p (list c #\#) 10 #f 'e))]
2007-06-14 11:56:47 -04:00
[(memq c '(#\i #\I))
(cons 'datum (parse-string p (list c #\#) 10 #f 'i))]
2007-06-14 11:56:47 -04:00
[(memq c '(#\b #\B))
(cons 'datum (parse-string p (list c #\#) 2 2 #f))]
2007-06-14 11:56:47 -04:00
[(memq c '(#\x #\X))
(cons 'datum (parse-string p (list c #\#) 16 16 #f))]
2007-06-14 11:56:47 -04:00
[(memq c '(#\o #\O))
(cons 'datum (parse-string p (list c #\#) 8 8 #f))]
2007-06-14 11:56:47 -04:00
[(memq c '(#\d #\D))
(cons 'datum (parse-string p (list c #\#) 10 10 #f))]
2009-06-26 03:07:26 -04:00
;[($char= #\@ c) DEAD: Unfixable due to port encoding
; that does not allow mixing binary and
; textual data in the same port.
; Left here for historical value
; (when (eq? (port-mode p) 'r6rs-mode)
; (die/p-1 p 'tokenize "fasl syntax is invalid in #!r6rs mode"
; (format "#~a" c)))
; (die/p-1 p 'read "FIXME: fasl read disabled")
; '(cons 'datum ($fasl-read p))]
[else
(die/p-1 p 'tokenize
(format "invalid syntax #~a" c))])))
(define (num-error p str ls)
(die/p-1 p 'read str
(list->string (reverse ls))))
(define-syntax port-config
(syntax-rules (GEN-TEST GEN-ARGS FAIL EOF-ERROR GEN-DELIM-TEST)
[(_ GEN-ARGS k . rest) (k (p ac) . rest)]
[(_ FAIL (p ac))
(num-error p "invalid numeric sequence" ac)]
[(_ FAIL (p ac) c)
(num-error p "invalid numeric sequence" (cons c ac))]
[(_ EOF-ERROR (p ac))
(num-error p "invalid eof while reading number" ac)]
[(_ GEN-DELIM-TEST c sk fk)
(if (delimiter? c) sk fk)]
[(_ GEN-TEST var next fail (p ac) eof-case char-case)
(let ([c (peek-char p)])
(if (eof-object? c)
(let ()
(define-syntax fail
(syntax-rules ()
[(_) (num-error p "invalid numeric sequence" ac)]))
eof-case)
(let ([var c])
(define-syntax fail
(syntax-rules ()
[(_)
(num-error p "invalid numeric sequence"
(cons var ac))]))
(define-syntax next
(syntax-rules ()
[(_ who args (... ...))
(who p (cons (get-char p) ac) args (... ...))]))
char-case)))]))
(define-string->number-parser port-config
(parse-string u:digit+ u:sign u:dot))
(define (read-char* p ls str who ci? delimited?)
(let f ([i 0] [ls ls])
(cond
[(fx= i (string-length str))
(when delimited?
(let ([c (peek-char p)])
(when (and (not (eof-object? c)) (not (delimiter? c)))
(die/p p 'tokenize
(format "invalid ~a: ~s" who
(list->string (reverse (cons c ls))))))))]
[else
(let ([c (read-char p)])
(cond
[(eof-object? c)
(die/p p 'tokenize