From 40cff81550d8ba5692868723c2c336916f768057 Mon Sep 17 00:00:00 2001 From: JeffBezanson Date: Tue, 3 Mar 2009 03:16:30 +0000 Subject: [PATCH] more reorganization of LLT adding io.readuntil, io.readline improvements to ios_readprep --- femtolisp/builtins.c | 1 + femtolisp/cvalues.c | 2 +- femtolisp/flisp.c | 10 +-- femtolisp/iostream.c | 25 +++++++ femtolisp/system.lsp | 2 + femtolisp/todo | 3 +- llt/Makefile | 5 +- llt/bswap.c | 87 +++++++++++++++++++++++ llt/fp.c | 22 ------ llt/hashing.c | 32 +-------- llt/hashing.h | 9 --- llt/ios.c | 41 ++++++++++- llt/ios.h | 2 +- llt/memalign.c | 50 +++++++++++++ llt/random.c | 63 +++++++++++++++++ llt/random.h | 14 ++++ llt/{utils.c => swapreverse.c} | 124 --------------------------------- 17 files changed, 295 insertions(+), 197 deletions(-) create mode 100644 llt/bswap.c create mode 100644 llt/memalign.c create mode 100644 llt/random.c create mode 100644 llt/random.h rename llt/{utils.c => swapreverse.c} (61%) diff --git a/femtolisp/builtins.c b/femtolisp/builtins.c index 0e4219d..c6040e6 100644 --- a/femtolisp/builtins.c +++ b/femtolisp/builtins.c @@ -14,6 +14,7 @@ #include #include "llt.h" #include "flisp.h" +#include "random.h" size_t llength(value_t v) { diff --git a/femtolisp/cvalues.c b/femtolisp/cvalues.c index f4bc44b..46c7539 100644 --- a/femtolisp/cvalues.c +++ b/femtolisp/cvalues.c @@ -838,7 +838,7 @@ value_t cbuiltin(char *name, builtin_t f) // directly-callable values are assumed not to move for // evaluator performance, so put builtin func metadata on the // unmanaged heap - cvalue_t *buf = malloc_aligned(nw * sizeof(value_t), 8); + cvalue_t *buf = malloc(nw * sizeof(value_t)); memcpy(buf, ptr(gf), nw*sizeof(value_t)); return tagptr(buf, TAG_BUILTIN); */ diff --git a/femtolisp/flisp.c b/femtolisp/flisp.c index 299311f..4cd90ca 100644 --- a/femtolisp/flisp.c +++ b/femtolisp/flisp.c @@ -202,8 +202,8 @@ static symbol_t *mk_symbol(char *str) symbol_t *sym; size_t len = strlen(str); - sym = (symbol_t*)malloc_aligned(sizeof(symbol_t)-sizeof(void*) + len + 1, - 8); + sym = (symbol_t*)malloc(sizeof(symbol_t)-sizeof(void*) + len + 1); + assert(((uptrint_t)sym & 0x7) == 0); // make sure malloc aligns 8 sym->left = sym->right = NULL; if (str[0] == ':') { value_t s = tagptr(sym, TAG_SYM); @@ -502,7 +502,7 @@ void gc(int mustgrow) // more space to fill next time. if we grew tospace last time, // grow the other half of the heap this time to catch up. if (grew || ((lim-curheap) < (int)(heapsize/5)) || mustgrow) { - temp = realloc_aligned(tospace, grew ? heapsize : heapsize*2, 16); + temp = realloc(tospace, grew ? heapsize : heapsize*2); if (temp == NULL) raise(memory_exception_value); tospace = temp; @@ -1442,8 +1442,8 @@ void lisp_init(void) llt_init(); - fromspace = malloc_aligned(heapsize, 16); - tospace = malloc_aligned(heapsize, 16); + fromspace = malloc(heapsize); + tospace = malloc(heapsize); curheap = fromspace; lim = curheap+heapsize-sizeof(cons_t); consflags = bitvector_new(heapsize/sizeof(cons_t), 1); diff --git a/femtolisp/iostream.c b/femtolisp/iostream.c index 6bb5d0f..d38ad0b 100644 --- a/femtolisp/iostream.c +++ b/femtolisp/iostream.c @@ -239,6 +239,30 @@ value_t fl_dump(value_t *args, u_int32_t nargs) return FL_T; } +value_t fl_ioreaduntil(value_t *args, u_int32_t nargs) +{ + argcount("io.readuntil", nargs, 2); + value_t str = cvalue_string(80); + cvalue_t *cv = (cvalue_t*)ptr(str); + char *data = cv_data(cv); + ios_t dest; + ios_mem(&dest, 0); + ios_setbuf(&dest, data, 80, 0); + char delim = (char)toulong(args[1], "io.readuntil"); + ios_t *src = toiostream(args[0], "io.readuntil"); + size_t n = ios_copyuntil(&dest, src, delim); + cv->len = n; + if (dest.buf != data) { + // outgrew initial space + cv->data = dest.buf; + cv_autorelease(cv); + } + ((char*)cv->data)[n] = '\0'; + if (n == 0 && ios_eof(src)) + return FL_F; + return str; +} + static builtinspec_t iostreamfunc_info[] = { { "iostream?", fl_iostreamp }, { "dump", fl_dump }, @@ -254,6 +278,7 @@ static builtinspec_t iostreamfunc_info[] = { { "io.discardbuffer", fl_iopurge }, { "io.read", fl_ioread }, { "io.write", fl_iowrite }, + { "io.readuntil", fl_ioreaduntil }, { NULL, NULL } }; diff --git a/femtolisp/system.lsp b/femtolisp/system.lsp index 3549ea9..878438d 100644 --- a/femtolisp/system.lsp +++ b/femtolisp/system.lsp @@ -560,6 +560,8 @@ (trim-start s at-start 0 L) (trim-end s at-end L)))) +(define (io.readline s) (io.readuntil s #byte(0xA))) + (define (repl) (define (prompt) (princ "> ") (io.flush *output-stream*) diff --git a/femtolisp/todo b/femtolisp/todo index a66ad5f..75beb59 100644 --- a/femtolisp/todo +++ b/femtolisp/todo @@ -858,7 +858,8 @@ IOStream API *io.write - (io.write s cvalue) *io.read - (io.read s ctype [len]) io.getc - get utf8 character(s) - io.readline +*io.readline +*io.readuntil io.copy - (io.copy to from [nbytes]) io.copyuntil - (io.copy to from byte) io.pos - (io.pos s [set-pos]) diff --git a/llt/Makefile b/llt/Makefile index b59a617..e4df4ab 100644 --- a/llt/Makefile +++ b/llt/Makefile @@ -1,8 +1,9 @@ CC = gcc -SRCS = bitvector.c hashing.c socket.c timefuncs.c utils.c dblprint.c ptrhash.c \ +SRCS = bitvector.c hashing.c socket.c timefuncs.c dblprint.c ptrhash.c \ utf8.c ios.c operators.c cplxprint.c dirpath.c htable.c \ - bitvector-ops.c fp.c int2str.c dump.c + bitvector-ops.c fp.c int2str.c dump.c random.c bswap.c memalign.c \ + swapreverse.c OBJS = $(SRCS:%.c=%.o) DOBJS = $(SRCS:%.c=%.do) TARGET = libllt.a diff --git a/llt/bswap.c b/llt/bswap.c new file mode 100644 index 0000000..bc1c23e --- /dev/null +++ b/llt/bswap.c @@ -0,0 +1,87 @@ +#include +#include +#include +#include +#include "dtypes.h" +#include "utils.h" + +void bswap_buffer(byte_t *data, size_t sz, size_t npts) +{ + size_t i, b; + byte_t *el; + byte_t temp; + + if (sz <= 1) + return; + + switch (sz) { + case 8: + for(i=0; i < npts; i++) { + ((u_int64_t*)data)[i] = bswap_64(((u_int64_t*)data)[i]); + } + break; + case 4: + for(i=0; i < npts; i++) { + ((u_int32_t*)data)[i] = bswap_32(((u_int32_t*)data)[i]); + } + break; + case 2: + for(i=0; i < npts; i++) { + ((u_int16_t*)data)[i] = bswap_16(((u_int16_t*)data)[i]); + } + break; + default: + for(i=0; i < sz * npts; i += sz) { + el = data + i; + for(b=0; b < sz/2; b++) { + temp = el[b]; + el[b] = el[sz-b-1]; + el[sz-b-1] = temp; + } + } + } +} + +void bswap(byte_t *s, size_t n) +{ + unsigned int i; + char temp; + + switch (n) { + case 8: + *(u_int64_t*)s = bswap_64(*(u_int64_t*)s); break; + case 4: + *(u_int32_t*)s = bswap_32(*(u_int32_t*)s); break; + case 2: + *(u_int16_t*)s = bswap_16(*(u_int16_t*)s); break; + case 1: + break; + default: + for(i=0; i < n/2; i++) { + temp = s[i]; + s[i] = s[n-i-1]; + s[n-i-1] = temp; + } + } +} + +void bswap_to(byte_t *dest, byte_t *src, size_t n) +{ + unsigned int i; + + switch (n) { + case 8: + *(u_int64_t*)dest = bswap_64(*(u_int64_t*)src); break; + case 4: + *(u_int32_t*)dest = bswap_32(*(u_int32_t*)src); break; + case 2: + *(u_int16_t*)dest = bswap_16(*(u_int16_t*)src); break; + case 1: + break; + default: + for(i=0; i < n; i++) { + dest[i] = src[n-i-1]; + } + } +} + diff --git a/llt/fp.c b/llt/fp.c index 679eea3..ffa1da9 100644 --- a/llt/fp.c +++ b/llt/fp.c @@ -86,25 +86,3 @@ int flt_equals(float a, float b) return 1; return 0; } - -double randn() -{ - double s, vre, vim, ure, uim; - static double next = -42; - - if (next != -42) { - s = next; - next = -42; - return s; - } - do { - ure = rand_double(); - uim = rand_double(); - vre = 2*ure - 1; - vim = 2*uim - 1; - s = vre*vre + vim*vim; - } while (s >= 1); - s = sqrt(-2*log(s)/s); - next = s * vre; - return s * vim; -} diff --git a/llt/hashing.c b/llt/hashing.c index 346b720..5129664 100644 --- a/llt/hashing.c +++ b/llt/hashing.c @@ -1,5 +1,5 @@ /* - Hashing and random numbers + Hashing */ #include #include @@ -10,6 +10,7 @@ #include "hashing.h" #include "timefuncs.h" #include "ios.h" +#include "random.h" uint_t nextipow2(uint_t i) { @@ -76,35 +77,6 @@ u_int32_t memhash32(char* buf, size_t n) return c; } -#include "mt19937ar.c" - -double rand_double() -{ - union ieee754_double d; - - d.ieee.mantissa0 = random(); - d.ieee.mantissa1 = random(); - d.ieee.negative = 0; - d.ieee.exponent = IEEE754_DOUBLE_BIAS + 0; /* 2^0 */ - return d.d - 1.0; -} - -float rand_float() -{ - union ieee754_float f; - - f.ieee.mantissa = random(); - f.ieee.negative = 0; - f.ieee.exponent = IEEE754_FLOAT_BIAS + 0; /* 2^0 */ - return f.f - 1.0; -} - -void randomize() -{ - u_int64_t tm = i64time(); - init_by_array((unsigned long*)&tm, 2); -} - double D_PNAN; double D_NNAN; double D_PINF; diff --git a/llt/hashing.h b/llt/hashing.h index 9e3741b..e4d4305 100644 --- a/llt/hashing.h +++ b/llt/hashing.h @@ -12,14 +12,5 @@ u_int32_t int64to32hash(u_int64_t key); #endif u_int64_t memhash(char* buf, size_t n); u_int32_t memhash32(char* buf, size_t n); -#define random() genrand_int32() -#define srandom(n) init_genrand(n) -double rand_double(); -float rand_float(); -double randn(); -u_int64_t i64time(); -void randomize(); -unsigned long genrand_int32(); -void init_genrand(unsigned long s); #endif diff --git a/llt/ios.c b/llt/ios.c index 9ba9ff1..9075c91 100644 --- a/llt/ios.c +++ b/llt/ios.c @@ -309,8 +309,17 @@ size_t ios_readprep(ios_t *s, size_t n) if (space >= n || s->bm == bm_mem || s->fd == -1) return space; if (s->maxsize < s->bpos+n) { - if (_buf_realloc(s, s->maxsize + n)==NULL) - return space; + // it won't fit. grow buffer or move data back. + if (n <= s->maxsize && space <= ((s->maxsize)>>5)) { + if (space) + memmove(s->buf, s->buf+s->bpos, space); + s->size -= s->bpos; + s->bpos = 0; + } + else { + if (_buf_realloc(s, s->bpos + n)==NULL) + return space; + } } size_t got; int result = _os_read(s->fd, s->buf+s->size, s->maxsize - s->size, &got); @@ -619,6 +628,34 @@ int ios_copyall(ios_t *to, ios_t *from) return ios_copy_(to, from, 0, 1); } +#define LINE_CHUNK_SIZE 160 + +size_t ios_copyuntil(ios_t *to, ios_t *from, char delim) +{ + size_t total = 0, avail; + if (!ios_eof(from)) { + do { + avail = ios_readprep(from, LINE_CHUNK_SIZE); + size_t written; + char *pd = (char*)memchr(from->buf+from->bpos, delim, avail); + if (pd == NULL) { + written = ios_write(to, from->buf+from->bpos, avail); + from->bpos += avail; + total += written; + } + else { + size_t ntowrite = pd - (from->buf+from->bpos) + 1; + written = ios_write(to, from->buf+from->bpos, ntowrite); + from->bpos += ntowrite; + total += written; + return total; + } + } while (!ios_eof(from) && avail >= LINE_CHUNK_SIZE); + } + from->_eof = 1; + return total; +} + static void _ios_init(ios_t *s) { // put all fields in a sane initial state diff --git a/llt/ios.h b/llt/ios.h index 3eb1e01..05d3532 100644 --- a/llt/ios.h +++ b/llt/ios.h @@ -80,6 +80,7 @@ void ios_set_readonly(ios_t *s); void ios_bswap(ios_t *s, int bswap); int ios_copy(ios_t *to, ios_t *from, size_t nbytes); int ios_copyall(ios_t *to, ios_t *from); +size_t ios_copyuntil(ios_t *to, ios_t *from, char delim); // ensure at least n bytes are buffered if possible. returns # available. size_t ios_readprep(ios_t *from, size_t n); //void ios_lock(ios_t *s); @@ -112,7 +113,6 @@ int ios_getutf8(ios_t *s, uint32_t *pwc); int ios_ungetutf8(ios_t *s, uint32_t wc); int ios_getstringz(ios_t *dest, ios_t *src); int ios_getstringn(ios_t *dest, ios_t *src, size_t nchars); -int ios_readline(ios_t *dest, ios_t *s, char delim); int ios_getline(ios_t *s, char **pbuf, size_t *psz); // discard data buffered for reading diff --git a/llt/memalign.c b/llt/memalign.c new file mode 100644 index 0000000..41d02aa --- /dev/null +++ b/llt/memalign.c @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include "dtypes.h" +#include "utils.h" + +#define ALIGNED_TO_ACTUAL(p) (((char*)p) - ((long*)p)[-1]) + +static void *aligned_ptr(char *ptr, size_t align_size) +{ + char *ptr2, *aligned_ptr; + + ptr2 = ptr + sizeof(long); + aligned_ptr = (char*)ALIGN(((uptrint_t)ptr2), align_size); + + ((long*)aligned_ptr)[-1] = (long)(aligned_ptr - ptr); + + return aligned_ptr; +} + +/* align_size has to be a power of two */ +void *malloc_aligned(size_t size, size_t align_size) +{ + char *ptr; + + ptr = (char*)malloc(size + align_size-1 + sizeof(long)); + if (ptr == NULL) + return NULL; + + return aligned_ptr(ptr, align_size); +} + +void free_aligned(void *ptr) +{ + free(ALIGNED_TO_ACTUAL(ptr)); +} + +void *realloc_aligned(void *ptr, size_t size, size_t align_size) +{ + char *pnew; + + if (ptr != NULL) + ptr = ALIGNED_TO_ACTUAL(ptr); + pnew = realloc(ptr, size + align_size-1 + sizeof(long)); + if (pnew == NULL) + return NULL; + + return aligned_ptr(pnew, align_size); +} diff --git a/llt/random.c b/llt/random.c new file mode 100644 index 0000000..43dd606 --- /dev/null +++ b/llt/random.c @@ -0,0 +1,63 @@ +/* + random numbers +*/ +#include +#include +#include +#include "ieee754.h" +#include "dtypes.h" +#include "utils.h" +#include "random.h" +#include "timefuncs.h" +#include "ios.h" + +#include "mt19937ar.c" + +double rand_double() +{ + union ieee754_double d; + + d.ieee.mantissa0 = random(); + d.ieee.mantissa1 = random(); + d.ieee.negative = 0; + d.ieee.exponent = IEEE754_DOUBLE_BIAS + 0; /* 2^0 */ + return d.d - 1.0; +} + +float rand_float() +{ + union ieee754_float f; + + f.ieee.mantissa = random(); + f.ieee.negative = 0; + f.ieee.exponent = IEEE754_FLOAT_BIAS + 0; /* 2^0 */ + return f.f - 1.0; +} + +double randn() +{ + double s, vre, vim, ure, uim; + static double next = -42; + + if (next != -42) { + s = next; + next = -42; + return s; + } + do { + ure = rand_double(); + uim = rand_double(); + vre = 2*ure - 1; + vim = 2*uim - 1; + s = vre*vre + vim*vim; + } while (s >= 1); + s = sqrt(-2*log(s)/s); + next = s * vre; + return s * vim; +} + +void randomize() +{ + u_int64_t tm = i64time(); + init_by_array((unsigned long*)&tm, 2); +} diff --git a/llt/random.h b/llt/random.h new file mode 100644 index 0000000..2216bbf --- /dev/null +++ b/llt/random.h @@ -0,0 +1,14 @@ +#ifndef __LLTRANDOM_H_ +#define __LLTRANDOM_H_ + +#define random() genrand_int32() +#define srandom(n) init_genrand(n) +double rand_double(); +float rand_float(); +double randn(); +void randomize(); +unsigned long genrand_int32(); +void init_genrand(unsigned long s); +u_int64_t i64time(); + +#endif diff --git a/llt/utils.c b/llt/swapreverse.c similarity index 61% rename from llt/utils.c rename to llt/swapreverse.c index e6b1847..68d22e0 100644 --- a/llt/utils.c +++ b/llt/swapreverse.c @@ -185,127 +185,3 @@ void memreverse_to(char *dest, char *a, size_t n, size_t elsz) break; } } - -void bswap_buffer(byte_t *data, size_t sz, size_t npts) -{ - size_t i, b; - byte_t *el; - byte_t temp; - - if (sz <= 1) - return; - - switch (sz) { - case 8: - for(i=0; i < npts; i++) { - ((u_int64_t*)data)[i] = bswap_64(((u_int64_t*)data)[i]); - } - break; - case 4: - for(i=0; i < npts; i++) { - ((u_int32_t*)data)[i] = bswap_32(((u_int32_t*)data)[i]); - } - break; - case 2: - for(i=0; i < npts; i++) { - ((u_int16_t*)data)[i] = bswap_16(((u_int16_t*)data)[i]); - } - break; - default: - for(i=0; i < sz * npts; i += sz) { - el = data + i; - for(b=0; b < sz/2; b++) { - temp = el[b]; - el[b] = el[sz-b-1]; - el[sz-b-1] = temp; - } - } - } -} - -void bswap(byte_t *s, size_t n) -{ - unsigned int i; - char temp; - - switch (n) { - case 8: - *(u_int64_t*)s = bswap_64(*(u_int64_t*)s); break; - case 4: - *(u_int32_t*)s = bswap_32(*(u_int32_t*)s); break; - case 2: - *(u_int16_t*)s = bswap_16(*(u_int16_t*)s); break; - case 1: - break; - default: - for(i=0; i < n/2; i++) { - temp = s[i]; - s[i] = s[n-i-1]; - s[n-i-1] = temp; - } - } -} - -void bswap_to(byte_t *dest, byte_t *src, size_t n) -{ - unsigned int i; - - switch (n) { - case 8: - *(u_int64_t*)dest = bswap_64(*(u_int64_t*)src); break; - case 4: - *(u_int32_t*)dest = bswap_32(*(u_int32_t*)src); break; - case 2: - *(u_int16_t*)dest = bswap_16(*(u_int16_t*)src); break; - case 1: - break; - default: - for(i=0; i < n; i++) { - dest[i] = src[n-i-1]; - } - } -} - -#define ALIGNED_TO_ACTUAL(p) (((char*)p) - ((long*)p)[-1]) - -static void *aligned_ptr(char *ptr, size_t align_size) -{ - char *ptr2, *aligned_ptr; - - ptr2 = ptr + sizeof(long); - aligned_ptr = (char*)ALIGN(((uptrint_t)ptr2), align_size); - - ((long*)aligned_ptr)[-1] = (long)(aligned_ptr - ptr); - - return aligned_ptr; -} - -/* align_size has to be a power of two */ -void *malloc_aligned(size_t size, size_t align_size) -{ - char *ptr; - - ptr = (char*)malloc(size + align_size-1 + sizeof(long)); - if (ptr == NULL) - return NULL; - - return aligned_ptr(ptr, align_size); -} - -void free_aligned(void *ptr) -{ - free(ALIGNED_TO_ACTUAL(ptr)); -} - -void *realloc_aligned(void *ptr, size_t size, size_t align_size) -{ - char *pnew; - - if (ptr != NULL) - ptr = ALIGNED_TO_ACTUAL(ptr); - pnew = realloc(ptr, size + align_size-1 + sizeof(long)); - if (pnew == NULL) - return NULL; - - return aligned_ptr(pnew, align_size); -}