1 |
#include "gcdb.h" |
2 |
#include <errno.h> |
3 |
#include <stdio.h> |
4 |
#include <stdarg.h> |
5 |
#if defined(__WIN32__) || defined(WIN32) |
6 |
#include <windows.h> |
7 |
/* m m a p === got from imagick sources |
8 |
% Method mmap emulates the Unix method of the same name. |
9 |
% The format of the mmap method is: |
10 |
% void *mmap(char *address,size_t length,int protection, |
11 |
% int access,int file,off_t offset) |
12 |
*/ |
13 |
void *mmap(char *address,size_t length,int protection,int access, |
14 |
int file, off_t offset) { |
15 |
void *map; |
16 |
HANDLE handle; |
17 |
map=(void *) NULL; |
18 |
handle=INVALID_HANDLE_VALUE; |
19 |
switch (protection) |
20 |
{ |
21 |
case PROT_READ: |
22 |
default: |
23 |
{ |
24 |
handle=CreateFileMapping((HANDLE) _get_osfhandle(file),0,PAGE_READONLY,0, |
25 |
length,0); |
26 |
if (!handle) |
27 |
break; |
28 |
map=(void *) MapViewOfFile(handle,FILE_MAP_READ,0,0,length); |
29 |
CloseHandle(handle); |
30 |
break; |
31 |
} |
32 |
case PROT_WRITE: |
33 |
{ |
34 |
handle=CreateFileMapping((HANDLE) _get_osfhandle(file),0,PAGE_READWRITE,0, |
35 |
length,0); |
36 |
if (!handle) |
37 |
break; |
38 |
map=(void *) MapViewOfFile(handle,FILE_MAP_WRITE,0,0,length); |
39 |
CloseHandle(handle); |
40 |
break; |
41 |
} |
42 |
case PROT_READWRITE: |
43 |
{ |
44 |
handle=CreateFileMapping((HANDLE) _get_osfhandle(file),0,PAGE_READWRITE,0, |
45 |
length,0); |
46 |
if (!handle) |
47 |
break; |
48 |
map=(void *) MapViewOfFile(handle,FILE_MAP_ALL_ACCESS,0,0,length); |
49 |
CloseHandle(handle); |
50 |
break; |
51 |
} |
52 |
} |
53 |
if (map == (void *) NULL) |
54 |
return((void *) MAP_FAILED); |
55 |
return((void *) ((char *) map+offset)); |
56 |
} |
57 |
|
58 |
/* =========== m u n m a p =========================== |
59 |
% |
60 |
% Method munmap emulates the Unix method with the same name. |
61 |
% The format of the munmap method is: |
62 |
% int munmap(void *map,size_t length) |
63 |
% A description of each parameter follows: |
64 |
% > status: Method munmap returns 0 on success; otherwise, it |
65 |
% returns -1 and sets errno to indicate the error. |
66 |
% > map: The address of the binary large object. |
67 |
% > length: The length of the binary large object. |
68 |
% |
69 |
*/ |
70 |
int munmap(void *map,size_t length) { |
71 |
if (!UnmapViewOfFile(map)) |
72 |
return(-1); |
73 |
return(0); |
74 |
} |
75 |
|
76 |
#endif |
77 |
|
78 |
|
79 |
|
80 |
int cdbInfoSIZE=offsetof(cdbInfo, tag)+4; |
81 |
|
82 |
|
83 |
//===================================================== |
84 |
//------------- buffer stuff ------------------- |
85 |
//===================================================== |
86 |
|
87 |
//------------------------------------- |
88 |
//--------- misc utility functions ----- |
89 |
|
90 |
static int gcdb_seek_set(int fd,gcdb_seek_pos pos) { |
91 |
if (lseek(fd, pos, 0) == -1) |
92 |
return -1; |
93 |
return 0; |
94 |
} |
95 |
|
96 |
#define gcdb_seek_begin(fd) (gcdb_seek_set((fd),(gcdb_seek_pos) 0)) |
97 |
|
98 |
static unsigned int gcdb_strlen(const char *s) { |
99 |
register char *t; |
100 |
t = (char*)s; |
101 |
for (;;) { |
102 |
if (!*t) return t - s; ++t; |
103 |
if (!*t) return t - s; ++t; |
104 |
if (!*t) return t - s; ++t; |
105 |
if (!*t) return t - s; ++t; |
106 |
} |
107 |
} |
108 |
|
109 |
|
110 |
static int byte_diff(char *s, unsigned int n,char *t) { |
111 |
for (;;) { |
112 |
if (!n) return 0; if (*s != *t) break; ++s; ++t; --n; |
113 |
if (!n) return 0; if (*s != *t) break; ++s; ++t; --n; |
114 |
if (!n) return 0; if (*s != *t) break; ++s; ++t; --n; |
115 |
if (!n) return 0; if (*s != *t) break; ++s; ++t; --n; |
116 |
} |
117 |
return ((int)(unsigned int)(unsigned char) *s) |
118 |
- ((int)(unsigned int)(unsigned char) *t); |
119 |
} |
120 |
|
121 |
static void gcdb_byte_copy(char *to, unsigned int n, char *from) { |
122 |
for (;;) { |
123 |
if (!n) return; *to++ = *from++; --n; |
124 |
if (!n) return; *to++ = *from++; --n; |
125 |
if (!n) return; *to++ = *from++; --n; |
126 |
if (!n) return; *to++ = *from++; --n; |
127 |
} |
128 |
} |
129 |
|
130 |
static void gcdb_byte_copyr(char *to, unsigned int n, char *from) { |
131 |
to += n; |
132 |
from += n; |
133 |
for (;;) { |
134 |
if (!n) return; *--to = *--from; --n; |
135 |
if (!n) return; *--to = *--from; --n; |
136 |
if (!n) return; *--to = *--from; --n; |
137 |
if (!n) return; *--to = *--from; --n; |
138 |
} |
139 |
} |
140 |
|
141 |
#define ALIGNMENT 16 /* XXX: assuming that this alignment is enough */ |
142 |
#define SPACE 4096 /* must be multiple of ALIGNMENT */ |
143 |
|
144 |
typedef union { char irrelevant[ALIGNMENT]; double d; } aligned; |
145 |
static aligned realspace[SPACE / ALIGNMENT]; |
146 |
#define space ((char *) realspace) |
147 |
|
148 |
static unsigned int avail = SPACE; /* multiple of ALIGNMENT; 0<=avail<=SPACE */ |
149 |
|
150 |
offt_conv gcvt_offt; |
151 |
uint_conv gcvt_uint; |
152 |
|
153 |
|
154 |
char *gcdb_alloc(unsigned int n) { |
155 |
char *x; |
156 |
n = ALIGNMENT + n - (n & (ALIGNMENT - 1)); /* XXX: could overflow */ |
157 |
if (n <= avail) { avail -= n; return space + avail; } |
158 |
x = (char*) malloc(n); |
159 |
if (!x) return NULL; |
160 |
//if (!x) GError("Error: mgcdb_alloc(%d) failed !\n", n); |
161 |
return x; |
162 |
} |
163 |
|
164 |
|
165 |
int GCDBuffer::write_all(char* buf, unsigned int len) { |
166 |
int w; |
167 |
while (len) { |
168 |
w = op(fd,buf,len); |
169 |
if (w == -1) { |
170 |
if (errno == error_intr) continue; |
171 |
return -1; /* note that some data may have been written */ |
172 |
} |
173 |
/* if (w == 0) ; luser's fault */ |
174 |
buf += w; |
175 |
len -= w; |
176 |
} |
177 |
return 0; |
178 |
} |
179 |
|
180 |
int GCDBuffer::flush() { |
181 |
int pt=p; |
182 |
if (!pt) return 0; |
183 |
p = 0; |
184 |
//return allwrite(op,fd,x,pt); |
185 |
return write_all(x,pt); |
186 |
} |
187 |
|
188 |
int GCDBuffer::putalign(char *buf,unsigned int len) { |
189 |
unsigned int bn; |
190 |
|
191 |
while (len > (bn = n-p)) { |
192 |
gcdb_byte_copy(x + p,bn,buf); |
193 |
p += bn; buf += bn; len -= bn; |
194 |
if (GCDBuffer::flush() == -1) return -1; |
195 |
} |
196 |
|
197 |
/* now len <= s->n - s->p */ |
198 |
gcdb_byte_copy(x + p,len,buf); |
199 |
p += len; |
200 |
return 0; |
201 |
} |
202 |
|
203 |
int GCDBuffer::put(char *buf,unsigned int len) { |
204 |
unsigned int bn=n; |
205 |
if (len > bn - p) { |
206 |
if (GCDBuffer::flush() == -1) return -1; |
207 |
/* now s->p == 0 */ |
208 |
if (bn < GCDBUFFER_OUTSIZE) bn = GCDBUFFER_OUTSIZE; |
209 |
while (len > n) { |
210 |
if (bn > len) bn = len; |
211 |
if (write_all(buf, bn) == -1) return -1; |
212 |
buf += bn; |
213 |
len -= bn; |
214 |
} |
215 |
} |
216 |
/* now len <= s->n - s->p */ |
217 |
gcdb_byte_copy(x + p,len,buf); |
218 |
p += len; |
219 |
return 0; |
220 |
} |
221 |
|
222 |
int GCDBuffer::putflush(char *buf,unsigned int len) { |
223 |
if (flush() == -1) return -1; |
224 |
return write_all(buf,len); |
225 |
} |
226 |
|
227 |
int GCDBuffer::putsalign(char *buf) { |
228 |
return GCDBuffer::putalign(buf, gcdb_strlen(buf)); |
229 |
} |
230 |
|
231 |
int GCDBuffer::puts(char *buf) { |
232 |
return GCDBuffer::put(buf, gcdb_strlen(buf)); |
233 |
} |
234 |
|
235 |
int GCDBuffer::putsflush(char *buf) { |
236 |
return GCDBuffer::putflush(buf, gcdb_strlen(buf)); |
237 |
} |
238 |
|
239 |
static int oneread(opfunc op,int fd, char *buf,unsigned int len) { |
240 |
int r; |
241 |
for (;;) { |
242 |
r = op(fd,buf,len); |
243 |
if (r == -1 && errno == error_intr) continue; |
244 |
return r; |
245 |
} |
246 |
} |
247 |
|
248 |
int GCDBuffer::oneRead(char* buf, unsigned int len) { |
249 |
return op(fd,buf,len); |
250 |
/*int r; |
251 |
for (;;) { |
252 |
r = op(fd,buf,len); |
253 |
if (r == -1 && errno == error_intr) continue; |
254 |
return r; |
255 |
}*/ |
256 |
} |
257 |
|
258 |
int GCDBuffer::getthis(char *buf,unsigned int len) { |
259 |
if (len > p) len = p; |
260 |
p -= len; |
261 |
gcdb_byte_copy(buf, len,x + n); |
262 |
n += len; |
263 |
return len; |
264 |
} |
265 |
|
266 |
int GCDBuffer::feed() { |
267 |
int r; |
268 |
if (p) return p; |
269 |
r = oneRead(x,n); |
270 |
if (r <= 0) |
271 |
return r; |
272 |
p = r; |
273 |
n -= r; |
274 |
if (n > 0) gcdb_byte_copyr(x + n,r,x); |
275 |
return r; |
276 |
} |
277 |
|
278 |
int GCDBuffer::bget(char *buf,unsigned int len) { |
279 |
int r; |
280 |
if (p > 0) return getthis(buf,len); |
281 |
if (n <= len) return oneRead(buf,n); |
282 |
r = GCDBuffer::feed(); if (r <= 0) return r; |
283 |
return getthis(buf,len); |
284 |
} |
285 |
|
286 |
int GCDBuffer::get(char *buf,unsigned int len) { |
287 |
int r; |
288 |
if (p > 0) return getthis(buf,len); |
289 |
if (n <= len) return oneread(op,fd,buf,len); |
290 |
r = GCDBuffer::feed(); |
291 |
if (r <= 0) |
292 |
return r; |
293 |
return getthis(buf,len); |
294 |
} |
295 |
|
296 |
char* GCDBuffer::peek() { |
297 |
return x + n; |
298 |
} |
299 |
|
300 |
void GCDBuffer::seek(unsigned int len) { |
301 |
n += len; |
302 |
p -= len; |
303 |
} |
304 |
|
305 |
int GCDBuffer::copy(GCDBuffer* bin) { |
306 |
int n_in; |
307 |
char *x_in; |
308 |
for (;;) { |
309 |
n_in = bin->feed(); |
310 |
if (n_in < 0) return -2; |
311 |
if (!n_in) return 0; |
312 |
x_in = bin->peek(); |
313 |
if (GCDBuffer::put(x_in,n_in) == -1) return -3; |
314 |
bin->seek(n_in); |
315 |
} |
316 |
} |
317 |
|
318 |
//===================================================== |
319 |
//------------- cdb utils ------------------- |
320 |
//===================================================== |
321 |
|
322 |
int error_intr = |
323 |
#ifdef EINTR |
324 |
EINTR; |
325 |
#else |
326 |
-1; |
327 |
#endif |
328 |
|
329 |
int error_nomem = |
330 |
#ifdef ENOMEM |
331 |
ENOMEM; |
332 |
#else |
333 |
-2; |
334 |
#endif |
335 |
|
336 |
int error_proto = |
337 |
#ifdef EPROTO |
338 |
EPROTO; |
339 |
#else |
340 |
-15; |
341 |
#endif |
342 |
//------------------------------------------------ |
343 |
//------------ allocation routines: |
344 |
|
345 |
/* |
346 |
big/little endian check |
347 |
*/ |
348 |
int endian_test(void) { |
349 |
unsigned short v=0x0001; |
350 |
unsigned char* b = (unsigned char*)&v; |
351 |
return b[1]; |
352 |
} |
353 |
/* conversion of unsigned int offsets read from a file |
354 |
can also be used to prepare unsigned integers to be written |
355 |
into a file in an independent platform manner |
356 |
*/ |
357 |
|
358 |
unsigned int uint32_sun(void* x86int) { |
359 |
unsigned char b[4]; |
360 |
b[3]=((unsigned char*)x86int)[0]; |
361 |
b[0]=((unsigned char*)x86int)[3]; |
362 |
b[1]=((unsigned char*)x86int)[2]; |
363 |
b[2]=((unsigned char*)x86int)[1]; |
364 |
return *((unsigned int*)b); |
365 |
} |
366 |
|
367 |
|
368 |
unsigned int uint32_x86(void* offt) { |
369 |
return *((unsigned int*)offt); |
370 |
} |
371 |
|
372 |
//-------- 64bit types, if that's the case: |
373 |
|
374 |
off_t offt_sun(void* offt) { |
375 |
unsigned char b[8]; |
376 |
if (sizeof(off_t)==8) { //64 bit? |
377 |
// upper words: |
378 |
b[3]=((unsigned char*)offt)[4]; |
379 |
b[0]=((unsigned char*)offt)[7]; |
380 |
b[1]=((unsigned char*)offt)[6]; |
381 |
b[2]=((unsigned char*)offt)[5]; |
382 |
//-- |
383 |
b[7]=((unsigned char*)offt)[0]; |
384 |
b[4]=((unsigned char*)offt)[3]; |
385 |
b[5]=((unsigned char*)offt)[2]; |
386 |
b[6]=((unsigned char*)offt)[1]; |
387 |
} |
388 |
else { |
389 |
b[3]=((unsigned char*)offt)[0]; |
390 |
b[0]=((unsigned char*)offt)[3]; |
391 |
b[1]=((unsigned char*)offt)[2]; |
392 |
b[2]=((unsigned char*)offt)[1]; |
393 |
} |
394 |
return *((off_t*)b); |
395 |
} |
396 |
|
397 |
|
398 |
|
399 |
off_t offt_x86(void* offt) { |
400 |
return *((off_t*)offt); |
401 |
} |
402 |
|
403 |
|
404 |
|
405 |
//------------------------ platform independent uint32 : |
406 |
|
407 |
void uint32_pack(char s[4],uint32 u) |
408 |
{ |
409 |
s[0] = u & 255; |
410 |
u >>= 8; |
411 |
s[1] = u & 255; |
412 |
u >>= 8; |
413 |
s[2] = u & 255; |
414 |
s[3] = u >> 8; |
415 |
} |
416 |
|
417 |
void uint32_pack_big(char s[4],uint32 u) |
418 |
{ |
419 |
s[3] = u & 255; |
420 |
u >>= 8; |
421 |
s[2] = u & 255; |
422 |
u >>= 8; |
423 |
s[1] = u & 255; |
424 |
s[0] = u >> 8; |
425 |
} |
426 |
|
427 |
/* unpacking: */ |
428 |
|
429 |
|
430 |
void uint32_unpack(char s[4],uint32 *u) |
431 |
{ |
432 |
uint32 result; |
433 |
|
434 |
result = (unsigned char) s[3]; |
435 |
result <<= 8; |
436 |
result += (unsigned char) s[2]; |
437 |
result <<= 8; |
438 |
result += (unsigned char) s[1]; |
439 |
result <<= 8; |
440 |
result += (unsigned char) s[0]; |
441 |
|
442 |
*u = result; |
443 |
} |
444 |
|
445 |
void uint32_unpack_big(char s[4],uint32 *u) |
446 |
{ |
447 |
uint32 result; |
448 |
|
449 |
result = (unsigned char) s[0]; |
450 |
result <<= 8; |
451 |
result += (unsigned char) s[1]; |
452 |
result <<= 8; |
453 |
result += (unsigned char) s[2]; |
454 |
result <<= 8; |
455 |
result += (unsigned char) s[3]; |
456 |
|
457 |
*u = result; |
458 |
} |
459 |
|
460 |
//===================================================== |
461 |
//------------- cdb index ------------------- |
462 |
//===================================================== |
463 |
|
464 |
GCdbWrite::GCdbWrite(int afd) { |
465 |
//check endianness :) |
466 |
gcvt_uint=(endian_test())? &uint32_sun : &uint32_x86; |
467 |
gcvt_offt=(endian_test())? &offt_sun : &offt_x86; |
468 |
cdbuf=new GCDBuffer((opfunc)&write,(int) afd,(char*)bspace,sizeof bspace); |
469 |
head = NULL; |
470 |
split = 0; |
471 |
hash = 0; |
472 |
numentries = 0; |
473 |
fd = afd; |
474 |
pos = sizeof final; |
475 |
gcdb_seek_set(fd, pos); |
476 |
|
477 |
fname[0]='\0'; |
478 |
//should return and test the result of gcdb_seek_set!!! |
479 |
} |
480 |
|
481 |
GCdbWrite::GCdbWrite(char* afname) { |
482 |
#if defined(__WIN32__) || defined(WIN32) |
483 |
fd = open(afname,O_WRONLY | O_TRUNC | O_BINARY | O_CREAT, S_IREAD|S_IWRITE); |
484 |
#else |
485 |
fd = open(afname,O_WRONLY | O_NDELAY | O_TRUNC | O_CREAT, 0664); |
486 |
#endif |
487 |
if (fd == -1) |
488 |
GError("GCdbWrite: Error creating file '%s'\n", fname); |
489 |
|
490 |
//check endianness :) |
491 |
gcvt_uint=(endian_test())? &uint32_sun : &uint32_x86; |
492 |
gcvt_offt=(endian_test())? &offt_sun : &offt_x86; |
493 |
|
494 |
cdbuf=new GCDBuffer((opfunc)&write,(int) fd,(char*)bspace,sizeof bspace); |
495 |
head = NULL; |
496 |
split = 0; |
497 |
hash = 0; |
498 |
numentries = 0; |
499 |
pos = sizeof final; |
500 |
gcdb_seek_set(fd, pos); |
501 |
strcpy(fname, afname); |
502 |
|
503 |
//should return and test the result of gcdb_seek_set!!! |
504 |
} |
505 |
|
506 |
GCdbWrite::~GCdbWrite() { |
507 |
cdbuf->flush(); |
508 |
#if !(defined(__WIN32__) || defined(WIN32)) |
509 |
/* NFS silliness */ |
510 |
if (fsync(fd) == -1) |
511 |
GError("GCdbWrite: Error at fsync() for file '%s'\n", |
512 |
fname); |
513 |
#endif |
514 |
if (::close(fd) == -1) |
515 |
GError("GCdbWrite: Error at closing file '%s'\n", |
516 |
fname); |
517 |
delete cdbuf; |
518 |
if (head!=NULL) free(head); |
519 |
} |
520 |
|
521 |
int GCdbWrite::posplus(uint32 len) { |
522 |
uint32 newpos = pos + len; |
523 |
if (newpos < len) { //errno = error_nomem; |
524 |
return -1; } |
525 |
pos = newpos; |
526 |
return 0; |
527 |
} |
528 |
|
529 |
int GCdbWrite::addend(unsigned int keylen,unsigned int datalen,uint32 h) { |
530 |
struct cdb_hplist *chead = head; |
531 |
if (!chead || (chead->num >= CDB_HPLIST)) { |
532 |
chead = (struct cdb_hplist *) gcdb_alloc(sizeof(struct cdb_hplist)); |
533 |
if (!chead) return -1; |
534 |
chead->num = 0; |
535 |
chead->next = head; |
536 |
head = chead; |
537 |
} |
538 |
chead->hp[head->num].h = h; |
539 |
chead->hp[head->num].p = pos; |
540 |
++chead->num; |
541 |
++numentries; |
542 |
if (posplus(8) == -1) return -1; |
543 |
if (posplus(keylen) == -1) return -1; |
544 |
if (posplus(datalen) == -1) return -1; |
545 |
return 0; |
546 |
} |
547 |
|
548 |
int GCdbWrite::addbegin(unsigned int keylen,unsigned int datalen) { |
549 |
char buf[8]; |
550 |
//if (keylen > MAX_UINT) { /* errno = error_nomem; */return -1; } |
551 |
// if (datalen > MAX_UINT) { /*errno = error_nomem;*/ return -1; } |
552 |
uint32_pack(buf,keylen); |
553 |
uint32_pack(buf + 4,datalen); |
554 |
if (cdbuf->putalign(buf,8) == -1) return -1; |
555 |
return 0; |
556 |
} |
557 |
|
558 |
#define cdbuffer_PUTC(s,c) \ |
559 |
( ((s).n != (s).p) \ |
560 |
? ( (s).x[(s).p++] = (c), 0 ) \ |
561 |
: (s).put(&(c),1) \ |
562 |
) |
563 |
|
564 |
int GCdbWrite::add(const char* key, char* recdata, unsigned int datalen) { |
565 |
unsigned int i; |
566 |
unsigned int klen=strlen(key); |
567 |
if (klen<1) { |
568 |
GMessage("Warning: zero length key found\n"); |
569 |
return 0; |
570 |
} |
571 |
//------------ adding record ----------------- |
572 |
if (addbegin(klen,datalen)==-1) |
573 |
GError("GCdbWrite: Error at addbegin(%d, %d)\n",klen, datalen); |
574 |
uint32 h=CDB_HASHSTART; |
575 |
for (i = 0;i < klen; ++i) { |
576 |
//if (cdbuffer_PUTC(c.cdbuf,key[i]) == -1) |
577 |
if ( ((cdbuf->n!=cdbuf->p) ? (cdbuf->x[cdbuf->p++]=(key[i]),0 ) |
578 |
: cdbuf->put((char*)&(key[i]),1) )==-1) |
579 |
GError("GCdbWrite: Error at cdbbuf.put, key '%s'\n", key); |
580 |
h = cdb_hashadd(h,key[i]); |
581 |
} |
582 |
if (cdbuf->put(recdata,datalen) == -1) |
583 |
GError("GCdbWrite: Error at final cdbuf.put() at key='%s', datalen=%d\n", |
584 |
key, datalen); |
585 |
if (addend(klen,datalen,h) == -1) |
586 |
GError("GCdbWrite: Error at addend(%d, %d, h)\n", klen, datalen); |
587 |
return 1; |
588 |
} |
589 |
|
590 |
int GCdbWrite::addrec(const char *key,unsigned int keylen,char *data,unsigned int datalen) { |
591 |
if (GCdbWrite::addbegin(keylen,datalen) == -1) return -1; |
592 |
if (cdbuf->putalign((char*)key,keylen) == -1) return -1; |
593 |
if (cdbuf->putalign(data,datalen) == -1) return -1; |
594 |
return GCdbWrite::addend(keylen,datalen,cdb_hash(key,keylen)); |
595 |
} |
596 |
|
597 |
|
598 |
int GCdbWrite::finish() { |
599 |
char buf[8]; |
600 |
int i; |
601 |
uint32 len; |
602 |
uint32 u; |
603 |
uint32 memsize; |
604 |
uint32 icount; |
605 |
uint32 where; |
606 |
struct cdb_hplist *x; |
607 |
struct cdb_hp *hp; |
608 |
|
609 |
for (i = 0;i < 256;++i) |
610 |
count[i] = 0; |
611 |
|
612 |
for (x = head;x;x = x->next) { |
613 |
i = x->num; |
614 |
while (i--) |
615 |
++count[255 & x->hp[i].h]; |
616 |
} |
617 |
|
618 |
memsize = 1; |
619 |
for (i = 0;i < 256;++i) { |
620 |
u = count[i] * 2; |
621 |
if (u > memsize) |
622 |
memsize = u; |
623 |
} |
624 |
|
625 |
memsize += numentries; /* no overflow possible up to now */ |
626 |
u = (uint32) 0 - (uint32) 1; |
627 |
u /= sizeof(struct cdb_hp); |
628 |
if (memsize > u) { /* errno = error_nomem;*/ return -1; } |
629 |
|
630 |
split = (struct cdb_hp *) gcdb_alloc(memsize * sizeof(struct cdb_hp)); |
631 |
if (!split) return -1; |
632 |
|
633 |
hash = split + numentries; |
634 |
|
635 |
u = 0; |
636 |
for (i = 0;i < 256;++i) { |
637 |
u += count[i]; /* bounded by numentries, so no overflow */ |
638 |
start[i] = u; |
639 |
} |
640 |
|
641 |
for (x = head;x;x = x->next) { |
642 |
i = x->num; |
643 |
while (i--) |
644 |
split[--start[255 & x->hp[i].h]] = x->hp[i]; |
645 |
} |
646 |
|
647 |
for (i = 0;i < 256;++i) { |
648 |
icount = count[i]; |
649 |
|
650 |
len = icount + icount; /* no overflow possible */ |
651 |
uint32_pack(final + 8 * i,pos); |
652 |
uint32_pack(final + 8 * i + 4,len); |
653 |
|
654 |
for (u = 0;u < len;++u) |
655 |
hash[u].h = hash[u].p = 0; |
656 |
|
657 |
hp = split + start[i]; |
658 |
for (u = 0;u < icount;++u) { |
659 |
where = (hp->h >> 8) % len; |
660 |
while (hash[where].p) |
661 |
if (++where == len) |
662 |
where = 0; |
663 |
hash[where] = *hp++; |
664 |
} |
665 |
|
666 |
for (u = 0;u < len;++u) { |
667 |
uint32_pack(buf,hash[u].h); |
668 |
uint32_pack(buf + 4,hash[u].p); |
669 |
if (cdbuf->putalign(buf,8) == -1) return -1; |
670 |
if (posplus(8) == -1) return -1; |
671 |
} |
672 |
} |
673 |
|
674 |
if (cdbuf->flush() == -1) return -1; |
675 |
if (gcdb_seek_begin(fd) == -1) return -1; |
676 |
return cdbuf->putflush(final,sizeof final); |
677 |
} |
678 |
|
679 |
//===================================================== |
680 |
//------------- cdb ------------------- |
681 |
//===================================================== |
682 |
uint32 cdb_hashadd(uint32 h,unsigned char c) { |
683 |
h += (h << 5); |
684 |
return h ^ c; |
685 |
} |
686 |
|
687 |
uint32 cdb_hash(const char *buf,unsigned int len) { |
688 |
uint32 h; |
689 |
h = CDB_HASHSTART; |
690 |
while (len) { |
691 |
h = cdb_hashadd(h,*buf++); |
692 |
--len; |
693 |
} |
694 |
return h; |
695 |
} |
696 |
|
697 |
//--------------------------------------------------------------- |
698 |
//-------------------------- cdb methods ------------------------ |
699 |
|
700 |
GCdbRead::GCdbRead(int afd) { |
701 |
struct stat st; |
702 |
char *x; |
703 |
map=NULL; |
704 |
//check endianness :) |
705 |
gcvt_uint=(endian_test())? &uint32_sun : &uint32_x86; |
706 |
gcvt_offt=(endian_test())? &offt_sun : &offt_x86; |
707 |
|
708 |
findstart(); |
709 |
fd = afd; |
710 |
if (fstat(fd,&st) == 0) |
711 |
if (st.st_size <= MAX_UINT) { |
712 |
#ifndef NO_MMAP |
713 |
x = (char *) mmap(0,st.st_size,PROT_READ,MAP_SHARED,fd,0); |
714 |
if (x + 1) { |
715 |
size = st.st_size; |
716 |
map = x; |
717 |
} |
718 |
else { |
719 |
GError("Error mapping the file (size=%ld)!\n",st.st_size); |
720 |
} |
721 |
#endif |
722 |
} |
723 |
else { |
724 |
GError("Error mapping the file (size %ld > MAX_UINT)\n", |
725 |
st.st_size); |
726 |
} |
727 |
} |
728 |
|
729 |
GCdbRead::GCdbRead(char* afname) { |
730 |
struct stat st; |
731 |
char *x; |
732 |
map=NULL; |
733 |
//check endianness :) |
734 |
gcvt_uint=(endian_test())? &uint32_sun : &uint32_x86; |
735 |
gcvt_offt=(endian_test())? &offt_sun : &offt_x86; |
736 |
|
737 |
|
738 |
findstart(); |
739 |
#ifdef __WIN32__ |
740 |
fd = open(afname, O_RDONLY|O_BINARY); |
741 |
#else |
742 |
fd = open(afname, O_RDONLY); |
743 |
#endif |
744 |
if (fd == -1) |
745 |
GError("Error: cannot open file %s\n", afname); |
746 |
strcpy(fname, afname); |
747 |
if (fstat(fd,&st) == 0) |
748 |
if (st.st_size <= MAX_UINT) { |
749 |
#ifndef NO_MMAP |
750 |
x = (char *) mmap(0,st.st_size,PROT_READ,MAP_SHARED,fd,0); |
751 |
if (x + 1) { |
752 |
size = st.st_size; |
753 |
map = x; |
754 |
} |
755 |
else { |
756 |
GError("GCdbRead: Error mapping the file (size=%ld)!\n",st.st_size); |
757 |
} |
758 |
#endif |
759 |
} |
760 |
else { |
761 |
GError("GCdbRead: Error mapping the file (size %ld > MAX_UINT)\n", |
762 |
st.st_size); |
763 |
} |
764 |
} |
765 |
|
766 |
|
767 |
GCdbRead::~GCdbRead() { |
768 |
if (map!=NULL) { |
769 |
munmap(map,size); |
770 |
map = 0; |
771 |
} |
772 |
} |
773 |
|
774 |
int GCdbRead::read(char *buf,unsigned int len, uint32 pos) { |
775 |
#ifndef NO_MMAP |
776 |
if (map) { |
777 |
if ((pos > size) || (size - pos < len)) { |
778 |
/* errno = error_proto; */ |
779 |
return -1; |
780 |
} |
781 |
gcdb_byte_copy(buf, len, map + pos); |
782 |
} |
783 |
else |
784 |
#endif |
785 |
{ |
786 |
if (gcdb_seek_set(fd,pos) == -1) return -1; |
787 |
while (len > 0) { |
788 |
int r; |
789 |
do { |
790 |
r = ::read(fd,buf,len); |
791 |
} while ((r == -1) && (errno == error_intr)); |
792 |
if (r == -1) return -1; |
793 |
if (r == 0) { |
794 |
//errno = error_proto; |
795 |
return -1; |
796 |
} |
797 |
buf += r; |
798 |
len -= r; |
799 |
} |
800 |
} |
801 |
return 0; |
802 |
} |
803 |
|
804 |
int GCdbRead::match(const char *key, unsigned int len, uint32 pos) { |
805 |
char buf[32]; |
806 |
unsigned int n; |
807 |
while (len > 0) { |
808 |
n = sizeof buf; |
809 |
if (n > len) n = len; |
810 |
if (GCdbRead::read(buf,n,pos) == -1) return -1; |
811 |
if (byte_diff(buf,n,(char*)key)) return 0; |
812 |
pos += n; |
813 |
key += n; |
814 |
len -= n; |
815 |
} |
816 |
return 1; |
817 |
} |
818 |
|
819 |
int GCdbRead::findnext(const char *key,unsigned int len) { |
820 |
char buf[8]; |
821 |
uint32 pos; |
822 |
uint32 u; |
823 |
if (!loop) { |
824 |
u = cdb_hash(key,len); |
825 |
if (GCdbRead::read(buf,8,(u << 3) & 2047) == -1) return -1; |
826 |
uint32_unpack(buf + 4,&hslots); |
827 |
if (!hslots) return 0; |
828 |
uint32_unpack(buf,&hpos); |
829 |
khash = u; |
830 |
u >>= 8; |
831 |
u %= hslots; |
832 |
u <<= 3; |
833 |
kpos = hpos + u; |
834 |
} |
835 |
while (loop < hslots) { |
836 |
if (GCdbRead::read(buf,8,kpos) == -1) return - 1; |
837 |
uint32_unpack(buf + 4, &pos); |
838 |
if (!pos) return 0; |
839 |
loop += 1; |
840 |
kpos += 8; |
841 |
if (kpos == hpos + (hslots << 3)) kpos = hpos; |
842 |
uint32_unpack(buf,&u); |
843 |
if (u == khash) { |
844 |
if (GCdbRead::read(buf,8,pos) == -1) return -1; |
845 |
uint32_unpack(buf,&u); |
846 |
if (u == len) |
847 |
switch(GCdbRead::match(key,len,pos + 8)) { |
848 |
case -1: |
849 |
return -1; |
850 |
case 1: |
851 |
uint32_unpack(buf + 4,&dlen); |
852 |
dpos = pos + 8 + len; |
853 |
return 1; |
854 |
} |
855 |
} |
856 |
} |
857 |
return 0; |
858 |
} |
859 |
|
860 |
int GCdbRead::find(const char *key) { |
861 |
GCdbRead::findstart(); |
862 |
return GCdbRead::findnext(key,gcdb_strlen(key)); |
863 |
} |
864 |
|
865 |
//----- GReadBuf and GReadBufLine |
866 |
|
867 |
char* GReadBufLine::readline(int idx) { |
868 |
//reads a char at a time until \n and/or \r are encountered |
869 |
GFREE(buf[idx].chars); |
870 |
buf[idx].len=0; |
871 |
if (isEOF) return NULL; |
872 |
int len=0; |
873 |
buf[idx].fpos=filepos; |
874 |
int c=0; |
875 |
int allocated=256; |
876 |
GMALLOC(buf[idx].chars, allocated); |
877 |
while ((c=getc(file))!=EOF) { |
878 |
if (len>=allocated-1) { |
879 |
allocated+=256; |
880 |
GREALLOC(buf[idx].chars, allocated); |
881 |
} |
882 |
if (c=='\n' || c=='\r') { |
883 |
buf[idx].chars[len]='\0'; |
884 |
if (c=='\r') { //DOS file -- special case |
885 |
if ((c=getc(file))!='\n') ungetc(c,file); |
886 |
else filepos++; |
887 |
} |
888 |
filepos++; |
889 |
buf[idx].len=len; |
890 |
return buf[idx].chars; |
891 |
} |
892 |
filepos++; |
893 |
buf[idx].chars[len]=(char)c; |
894 |
len++; |
895 |
} //while i<buf_cap-1 |
896 |
if (c==EOF) { //end of file reached while reading chars |
897 |
isEOF=true; |
898 |
} |
899 |
buf[idx].len=len; |
900 |
if (len==0 && isEOF) { |
901 |
GFREE(buf[idx].chars); |
902 |
} |
903 |
else { |
904 |
buf[idx].chars[len]='\0'; |
905 |
} |
906 |
return buf[idx].chars; |
907 |
} |
908 |
|
909 |
|
910 |
|
911 |
int GReadBufLine::fillbuf() { |
912 |
if (isEOF) return -1; |
913 |
if (bufidx==0 || bufidx==1) return 0; //buffer was just filled! |
914 |
int bufstart=0; |
915 |
GASSERT( (bufidx<=bufcap) ); |
916 |
if (bufidx>0) { //preserve the lines already in buffer |
917 |
int bidx=bufidx-1;//always leave room for PREVIOUS line, for putLine() |
918 |
for (int i=0;i<bufcap-bidx;i++) { |
919 |
GFREE(buf[i].chars); |
920 |
buf[i]=buf[bidx+i]; |
921 |
buf[bidx+i].chars=NULL; |
922 |
} |
923 |
//memmove((void*)&buf[0], (void*)&buf[bidx], (bufcap-bidx)*sizeof(BufLine)); |
924 |
bufstart=bufcap-bidx; |
925 |
bufidx=1; |
926 |
} |
927 |
else { |
928 |
bufidx=0; //only the first time, before the first getLine() |
929 |
} |
930 |
int rlines=0; |
931 |
for (int i=bufstart;i<bufcap;i++) { |
932 |
if (readline(i)!=NULL) rlines++; |
933 |
} |
934 |
return rlines; |
935 |
} |
936 |
|
937 |
//get a line from the buffer, update "current line" pointer |
938 |
const char* GReadBufLine::line() { |
939 |
if (isEOB) return NULL; |
940 |
GASSERT( (bufidx>=0 && bufidx<bufcap) ); |
941 |
char* r=buf[bufidx].chars; |
942 |
lno++; |
943 |
if (r==NULL) { |
944 |
isEOB=true; |
945 |
return NULL; |
946 |
} |
947 |
bufidx++; |
948 |
if (bufidx==bufcap) { |
949 |
if (isEOF) isEOB=true; |
950 |
else fillbuf(); |
951 |
} |
952 |
return r; |
953 |
} |
954 |
|
955 |
off_t GReadBufLine::fpos() { |
956 |
if (isEOB || bufidx==0) return -1; |
957 |
GASSERT( (bufidx>0 && bufidx<bufcap) ); |
958 |
return buf[bufidx-1].fpos; |
959 |
} |
960 |
|
961 |
int GReadBufLine::len() { |
962 |
if (isEOB || bufidx==0) return -1; |
963 |
GASSERT( (bufidx>0 && bufidx<bufcap) ); |
964 |
return buf[bufidx-1].len; |
965 |
} |
966 |
|
967 |
void GReadBufLine::putLine() { |
968 |
if (bufidx==0) GError("Error: calling putLine() before getLine()!\n"); |
969 |
bufidx--; |
970 |
isEOB=false; |
971 |
lno--; |
972 |
} |