blockFile.c (16801B)
1
2
3 /* Add class methods and modify the base functions (free, duplicate, ...) where there are the TODOs (TODO)*/
4
5 #include "libsheepyObject.h"
6 #include "blockFile.h"
7 #include "blockFileInternal.h"
8 #include "shpPackages/lz4/lz4.h"
9
10 #include <stdlib.h>
11 #include <string.h>
12 #include <stdio.h>
13
14 void initiateBlockFile(blockFilet *self);
15 void registerMethodsBlockFile(blockFileFunctionst *f);
16 void initiateAllocateBlockFile(blockFilet **self);
17 void finalizeBlockFile(void);
18 blockFilet* allocBlockFile(char *filename);
19 internal void freeBlockFile(blockFilet *self);
20 internal void terminateBlockFile(blockFilet **self);
21 internal char* toStringBlockFile(blockFilet *self);
22 internal blockFilet* duplicateBlockFile(blockFilet *self);
23 internal void smashBlockFile(blockFilet **self);
24 internal void finishBlockFile(blockFilet **self);
25 internal bool openBlockFile(blockFilet *self, const char *filename);
26 internal void closeBlockFile(blockFilet *self);
27 internal void deleteBlockFile(blockFilet *self);
28 internal void deleteFBlockFile(char *filename);
29 internal u64 addBlockBlockFile(blockFilet *self, void* buf, i64 len, flagsBlockFilet flags);
30 internal u64 addBlockFile(blockFilet *self, void* buf, i64 len);
31 internal bufBlockFilet getBlockFile(blockFilet *self, u64 block);
32 internal bool removeBlockFile(blockFilet *self, u64 block);
33 internal bool loadBlockFile(blockFilet *self, void *closure, loadFBlockFileFt callback);
34 /* TODO add prototypes */
35
36 void initiateBlockFile(blockFilet *self) {
37
38 self->type = "blockFile";
39 if (!blockFileF) {
40 blockFileF = malloc(sizeof(blockFileFunctionst));
41 registerMethodsBlockFile(blockFileF);
42 pErrorNot0(atexit(finalizeBlockFile));
43 }
44 self->f = blockFileF;
45 self->file = malloc(sizeof(privateBlockFilet));
46 // no database opened: no name
47 fil.name = NULL;
48 /* TODO Initialize object data */
49 }
50
51 void registerMethodsBlockFile(blockFileFunctionst *f) {
52
53 f->free = freeBlockFile;
54 f->terminate = terminateBlockFile;
55 f->toString = toStringBlockFile;
56 f->duplicate = duplicateBlockFile;
57 f->smash = smashBlockFile;
58 f->finish = finishBlockFile;
59 f->open = openBlockFile;
60 f->close = closeBlockFile;
61 f->delete = deleteBlockFile;
62 f->deleteF = deleteFBlockFile;
63 f->addBlock = addBlockBlockFile;
64 f->add = addBlockFile;
65 f->get = getBlockFile;
66 f->remove = removeBlockFile;
67 f->load = loadBlockFile;
68 /* TODO add class functions */
69 }
70
71 void initiateAllocateBlockFile(blockFilet **self) {
72
73 if (self) {
74 (*self) = malloc(sizeof(blockFilet));
75 if (*self) {
76 initiateBlockFile(*self);
77 }
78 }
79 }
80
81 void finalizeBlockFile(void) {
82
83 if (blockFileF) {
84 free(blockFileF);
85 blockFileF = NULL;
86 }
87 }
88
89 blockFilet* allocBlockFile(char* filename) {
90 blockFilet *r = NULL;
91
92 initiateAllocateBlockFile(&r);
93
94 if (!openBlockFile(r, filename)) {
95 terminateBlockFile(&r);
96 }
97 return(r);
98 }
99
100
101 internal void freeBlockFile(blockFilet *self) {
102
103 if (fil.name) {
104 closeBlockFile(self);
105 }
106 free(self->file);
107 /* TODO free internal data (not the structure holding the function pointers) */
108 return;
109 }
110
111 internal void terminateBlockFile(blockFilet **self) {
112
113 freeBlockFile(*self);
114 free(*self);
115 *self = NULL;
116 }
117
118
119 internal char* toStringBlockFile(blockFilet UNUSED *self) {
120
121 /* TODO convert object data to string */
122 return(strdup("TODO - blockFile"));
123 }
124
125 internal blockFilet* duplicateBlockFile(blockFilet UNUSED *self) {
126
127 createAllocateBlockFile(dup);
128 /* TODO COPY data */
129 return(dup);
130 }
131
132 internal void smashBlockFile(blockFilet **self) {
133
134 finishBlockFile(self);
135 }
136
137 internal void finishBlockFile(blockFilet **self) {
138
139 free(*self);
140 *self = NULL;
141 }
142
143 internal bool openBlockFile(blockFilet *self, const char *filename) {
144 if (!filename || isBlankS(filename))
145 return false;
146
147 bool dbExists = true;
148
149 // init
150 fil.name = strdup(filename);
151 fil.blockSize = BLOCKSIZE;
152 fil.freeName = catS(filename, "x");
153 fil.defaultFlags = COMPRESSED;
154
155 // check if db file exists
156 if (!fileExists(filename)) {
157 dbExists = false;
158
159 // save info in block 0
160 block0 b0;
161 b0.blockSize = fil.blockSize;
162 b0.blockIndexSize = 0;
163
164 char blockBuf[BLOCKSIZE];
165
166 memcpy(blockBuf, &b0, sizeof b0);
167
168 pError0(writeFile(filename, blockBuf, fil.blockSize));
169 }
170 fil.f = fopen(filename, "r+");
171 if (!fil.f) pFuncError;
172 fil.fmode = READWRITE;
173
174 // TODO read block 0 for configuration: blockSize, blockIndexSize 32/64 bits
175
176 fil.count = fileSize(filename) / fil.blockSize;
177
178 // initialize freeF
179 if (!fileExists(fil.freeName) || !dbExists) {
180 dArrayInit(&fil.freeBlocks);
181 // create empty freeF file
182 fil.freeF = fopen(fil.freeName, "w");
183 fclose(fil.freeF);
184 }
185 else {
186 // load freeBlocks
187 dArrayInit(&fil.freeBlocks);
188 dArrayReadFilename(&fil.freeBlocks, fil.freeName);
189 }
190
191 // open freeF - read mode to keep the content (fixes the duplicate test)
192 // this file is freopen in write mode later
193 fil.freeF = fopen(fil.freeName, "r");
194
195 return true;
196 }
197
198 internal void closeBlockFile(blockFilet *self) {
199 // save free blocks
200 freopen(NULL, "w", fil.freeF);
201 dArrayWrite(&fil.freeBlocks, fil.freeF);
202 // free dArray
203 dArrayFree(&fil.freeBlocks);
204
205 // close FILES
206 fclose(fil.f);
207 fclose(fil.freeF);
208 freeManyS(fil.name, fil.freeName);
209 }
210
211 internal void deleteBlockFile(blockFilet *self) {
212 char *s1 = strdup(fil.name);
213 char *s2 = strdup(fil.freeName);
214
215 // close file
216 closeBlockFile(self);
217
218 // remove file and freeF
219 pError0(rmAll(s1));
220 pError0(rmAll(s2));
221 freeManyS(s1, s2);
222 }
223
224 internal void deleteFBlockFile(char *filename) {
225 if (!filename || isBlankS(filename))
226 return;
227
228 char *s = catS(filename, "x");
229
230 // remove file and freeF
231 pError0(rmAll(filename));
232 pError0(rmAll(s));
233 free(s);
234 }
235
236 /**
237 * set file mode
238 * read/write or append
239 */
240 internal void setFMode(blockFilet *self, fmodet fmode) {
241 if ((fil.fmode == APPEND) and (fmode == READWRITE)) {
242 freopen(NULL, "r+", fil.f);
243 fil.fmode = READWRITE;
244 }
245 if ((fil.fmode == READWRITE) and (fmode == APPEND)) {
246 freopen(NULL, "a", fil.f);
247 fil.fmode = APPEND;
248 }
249 }
250
251 internal uI insertBlock(blockFilet *self, void *blockData) {
252 if (!dArrayCount(&fil.freeBlocks)) {
253 // append new block
254 //logI("no free blocks");
255 setFMode(self, APPEND);
256 fwrite(blockData, 1, fil.blockSize, fil.f);
257 fil.count++;
258 //logVarG(fil.count-1);
259 return fil.count-1;
260 }
261 else {
262 // reuse free block
263 uI block = dArrayPop(&fil.freeBlocks);
264 setFMode(self, READWRITE);
265 fseek(fil.f, block * fil.blockSize, SEEK_SET);
266 fwrite(blockData, 1, fil.blockSize, fil.f);
267 //logVarG(block);
268 return block;
269 }
270 }
271
272 internal uI nextBlock(blockFilet *self) {
273 if (!dArrayCount(&fil.freeBlocks)) {
274 return fil.count+1;
275 }
276 else if (dArrayCount(&fil.freeBlocks) == 1) {
277 return fil.count;
278 }
279 else {
280 return dArrayAt(&fil.freeBlocks, dArrayLastIndex(&fil.freeBlocks)-1);
281 }
282 }
283
284 internal u64 addBlockBlockFile(blockFilet *self, void* buf, i64 len, flagsBlockFilet flags) {
285 uI firstBlock = 0;
286 size_t freeB = dArrayCount(&fil.freeBlocks);
287
288 //logVarG(len);
289
290 // compress with lz4
291 uint32_t decompressedSize;
292 if (flags == COMPRESSED) {
293 if (len < 192) {
294 // too small to have benefits from compression
295 flags = NOT_COMPRESSED;
296 //AT;
297 }
298 else {
299 decompressedSize = len;
300 const char *uncompressed = buf;
301
302 int max_dst_size = LZ4_compressBound(decompressedSize);
303
304 char* compressed_data = malloc(max_dst_size);
305 if (compressed_data == NULL) {
306 shEPrintfS("Failed to allocate memory for *compressed_data.");
307 return 0;
308 }
309
310 len = LZ4_compress_default(uncompressed, compressed_data, decompressedSize, max_dst_size);
311
312 if (len < 0) {
313 shEPrintfS("A negative result from LZ4_compress_default indicates a failure trying to compress the data. Value returned %d\n", len);
314 return 0;
315 }
316 if (len == 0) {
317 shEPrintfS("A result of 0 means compression worked, but was stopped because the destination buffer couldn't hold all the information.");
318 return 0;
319 }
320
321 buf = compressed_data;
322 }
323 }
324
325 // header sizes
326 u64 firstHeaderSize;
327 if (flags == COMPRESSED) {
328 firstHeaderSize = sizeof(struct firstBlockHeaderCompressed);
329 }
330 else {
331 firstHeaderSize = sizeof(struct firstBlockHeader);
332 }
333 u64 headerSize = sizeof(struct blockheader);
334 i64 firstDataSize = fil.blockSize - firstHeaderSize;
335 i64 dataSize = fil.blockSize - headerSize;
336
337 // create block chain
338 // user data in first block: blockSize - firstHeaderSize
339 // user data in other blocks: blockSize - headerSize
340
341 char blockBuf[fil.blockSize];
342 blockt *header = (blockt *)blockBuf;
343 bool isFirstBlock = true;
344 char *offset = buf;
345 u64 currentDataSize;
346
347 //loghex(buf, len);
348
349 while(len > 0) {
350 if (isFirstBlock) {
351 isFirstBlock = false;
352 header->chain = 1;
353 header->z = flags == COMPRESSED ? 1 : 0;
354 header->dataSize = len;
355
356 if (flags == COMPRESSED) {
357 header->decompressedSize = decompressedSize;
358 }
359
360 if (len <= firstDataSize) {
361 header->nextBlock = 0;
362 memcpy(blockBuf+firstHeaderSize, offset, len);
363 firstBlock = insertBlock(self, blockBuf);
364 //len = 0;
365 break;
366 }
367 else {
368 // more blocks to save
369 header->nextBlock = nextBlock(self);
370 currentDataSize = firstDataSize;
371 memcpy(blockBuf+firstHeaderSize, offset, firstDataSize);
372 firstBlock = insertBlock(self, blockBuf);
373 }
374 }
375 else {
376 // other blocks
377 header->chain = 0;
378 if (len <= dataSize) {
379 header->nextBlock = 0;
380 memcpy(blockBuf+headerSize, offset, len);
381 insertBlock(self, blockBuf);
382 //len = 0;
383 break;
384 }
385 else {
386 // more blocks to save
387 header->nextBlock = nextBlock(self);
388 currentDataSize = dataSize;
389 memcpy(blockBuf+headerSize, offset, currentDataSize);
390 insertBlock(self, blockBuf);
391 }
392 }
393 //logVarG(header->nextBlock);
394 len -= currentDataSize;
395 offset += currentDataSize;
396 }
397 //logVarG(header->nextBlock);
398
399 if (flags == COMPRESSED) {
400 // free the compressed buffer
401 free(buf);
402 }
403
404 // save free blocks if necessary
405 if (freeB) {
406 // there were free block when we started, save free blocks
407 freopen(NULL, "w", fil.freeF);
408 dArrayWrite(&fil.freeBlocks, fil.freeF);
409 }
410 return firstBlock;
411 }
412
413 internal u64 addBlockFile(blockFilet *self, void* buf, i64 len) {
414 return addBlockBlockFile(self, buf, len, fil.defaultFlags);
415 }
416
417 /**
418 * \return
419 * bufBlockFilet {data, len}, data has to be freed
420 */
421 internal bufBlockFilet getBlockFile(blockFilet *self, u64 block) {
422 char blockBuf[fil.blockSize];
423 blockt *header = (blockt *)blockBuf;
424 bufBlockFilet result = {NULL, 0};
425
426 // block 0 is reserved
427 if (!block) return result;
428
429 // check if block is free
430 range(i, (size_t)(dArrayCount(&fil.freeBlocks))) {
431 if (block == dArrayAt(&fil.freeBlocks, i)) {
432 return result;
433 }
434 }
435
436 setFMode(self, READWRITE);
437 fseek(fil.f, block * fil.blockSize, SEEK_SET);
438 fread(blockBuf, 1, fil.blockSize, fil.f);
439
440 if (header->chain != 1) {
441 // error not first block
442 return result;
443 }
444
445 // header sizes
446 u64 firstHeaderSize;
447 if (header->z) {
448 firstHeaderSize = sizeof(struct firstBlockHeaderCompressed);
449 }
450 else {
451 firstHeaderSize = sizeof(struct firstBlockHeader);
452 }
453 u64 headerSize = sizeof(struct blockheader);
454 u64 firstDataSize = fil.blockSize - firstHeaderSize;
455 u64 dataSize = fil.blockSize - headerSize;
456
457 u64 currentDataSize;
458
459 // decompress
460 uint32_t decompressedSize = 0;
461 if (header->z) {
462 decompressedSize = header->decompressedSize;
463 }
464
465 u64 len = header->dataSize;
466 const u64 compressedSize = header->dataSize;
467 void *r = malloc(len);
468 if (!r) {
469 return result;
470 }
471
472 // set result length: dataSize or decompressedSize
473 if (header->z) {
474 result.len = decompressedSize;
475 }
476 else {
477 result.len = len;
478 }
479
480 /* first block already loaded */
481 char *offset = r;
482
483 bool isFirstBlock = true;
484 while(len > 0) {
485 if (isFirstBlock) {
486 isFirstBlock = false;
487 if (len <= firstDataSize) {
488 if (header->nextBlock) {
489 // error this block should be the last block
490 goto error;
491 }
492 // all the data is in the first block
493 memcpy(offset, blockBuf+firstHeaderSize, len);
494 //len = 0;
495 break;
496 }
497 else {
498 block = header->nextBlock;
499 if (!block) {
500 // error there should be a next block
501 goto error;
502 }
503 currentDataSize = firstDataSize;
504 memcpy(offset, blockBuf+firstHeaderSize, currentDataSize);
505 }
506 }
507 else {
508 fseek(fil.f, block * fil.blockSize, SEEK_SET);
509 fread(blockBuf, 1, fil.blockSize, fil.f);
510
511 if (header->chain != 0) {
512 // error not a block in a chain
513 goto error;
514 }
515
516 if (len <= dataSize) {
517 if (header->nextBlock) {
518 // error this block should be the last block
519 goto error;
520 }
521 // all the data is in the first block
522 memcpy(offset, blockBuf+headerSize, len);
523 //len = 0;
524 break;
525 }
526 else {
527 block = header->nextBlock;
528 if (!block) {
529 // error there should be a next block
530 goto error;
531 }
532 currentDataSize = dataSize;
533 memcpy(offset, blockBuf+headerSize, currentDataSize);
534 }
535 }
536 //logVarG(header->nextBlock);
537 len -= currentDataSize;
538 offset += currentDataSize;
539 }
540
541 //loghex(r, compressedSize);
542
543 // decompress
544 if (decompressedSize) {
545 // the data is compressed
546 char *compressed_data = r;
547 char* regen_buffer = malloc(decompressedSize);
548 if (regen_buffer == NULL) {
549 free(r);
550 shEPrintfS("Failed to allocate memory for *regen_buffer.");
551 return result;
552 }
553
554 const int decompressed_size = LZ4_decompress_safe(compressed_data, regen_buffer, compressedSize, decompressedSize);
555 free(r);
556
557 if (decompressed_size < 0) {
558 free(regen_buffer);
559 shEPrintfS("A negative result from LZ4_decompress_safe indicates a failure trying to decompress the data. Value returned %d\n", decompressed_size);
560 return result;
561 }
562 if (decompressed_size == 0) {
563 free(regen_buffer);
564 shEPrintfS("I'm not sure this function can ever return 0. Documentation in lz4.h doesn't indicate so.");
565 return result;
566 }
567
568 r = regen_buffer;
569 }
570
571 result.data = r;
572 return result;
573
574 error:
575 //logVarG(header->nextBlock);
576 free(r);
577 return result;
578 }
579
580 internal bool removeBlockFile(blockFilet *self, u64 block) {
581 char blockBuf[sizeof(blockt)];
582 blockt *header = (blockt *)blockBuf;
583
584 // block 0 is reserved
585 if (!block) return false;
586
587 // check if block is already free
588 range(i, (size_t)(dArrayCount(&fil.freeBlocks))) {
589 if (block == dArrayAt(&fil.freeBlocks, i)) {
590 return false;
591 }
592 }
593
594 setFMode(self, READWRITE);
595 fseek(fil.f, block * fil.blockSize, SEEK_SET);
596 fread(blockBuf, 1, sizeof(blockt), fil.f);
597
598 if (header->chain != 1) {
599 // error not first block
600 return false;
601 }
602
603 // delete first block
604 dArrayAppend(&fil.freeBlocks, block);
605
606 while(header->nextBlock) {
607 block = header->nextBlock;
608 fseek(fil.f, block * fil.blockSize, SEEK_SET);
609 fread(blockBuf, 1, sizeof(blockt), fil.f);
610 if (header->chain != 0) {
611 // error not other block
612 return false;
613 }
614 // delete block
615 dArrayAppend(&fil.freeBlocks, block);
616 }
617
618 // save free blocks
619 freopen(NULL, "w", fil.freeF);
620 dArrayWrite(&fil.freeBlocks, fil.freeF);
621
622 return true;
623 }
624
625 internal bool loadBlockFile(blockFilet *self, void *closure, loadFBlockFileFt callback) {
626 char blockBuf[sizeof(blockt)];
627 blockt *header = (blockt *)blockBuf;
628
629 setFMode(self, READWRITE);
630
631 rangeFrom(i, 1, fil.count) {
632 // check if block i is free
633 range(j, (size_t)(dArrayCount(&fil.freeBlocks))) {
634 if (i == dArrayAt(&fil.freeBlocks, j)) goto cont;
635 }
636
637 fseek(fil.f, i * fil.blockSize, SEEK_SET);
638 fread(blockBuf, 1, sizeof(blockt), fil.f);
639
640 if (header->chain != 1) {
641 // not first block, skip
642 goto cont;
643 }
644
645 bufBlockFilet data = getBlockFile(self, i);
646 if (!callback(closure, i, data)) return false;
647 cont:;
648 }
649 return true;
650 }
651
652 /* TODO add method implementations */
653
654 bool checkLibsheepyVersionBlockFile(const char *currentLibsheepyVersion) {
655 return eqG(currentLibsheepyVersion, LIBSHEEPY_VERSION);
656 }
657