Project

General

Profile

Statistics
| Branch: | Revision:

root / src / scave / indexfile.cc @ a3be1d55

History | View | Annotate | Download (18 KB)

1 01873262 Georg Kunz
//=========================================================================
2
//  INDEXFILE.CC - part of
3
//                  OMNeT++/OMNEST
4
//           Discrete System Simulation in C++
5
//
6
//  Author: Tamas Borbely
7
//
8
//=========================================================================
9
10
/*--------------------------------------------------------------*
11
  Copyright (C) 2006-2008 OpenSim Ltd.
12

13
  This file is distributed WITHOUT ANY WARRANTY. See the file
14
  `license' for details on this and other legal matters.
15
*--------------------------------------------------------------*/
16
17
#include <algorithm>
18
#include <locale.h>
19
#include <sys/stat.h>
20
#include "intxtypes.h"
21
#include "exception.h"
22
#include "filereader.h"
23
#include "linetokenizer.h"
24
#include "stringutil.h"
25
#include "scaveutils.h"
26
#include "scaveexception.h"
27
#include "indexfile.h"
28
29
#define LL INT64_PRINTF_FORMAT
30
31
USING_NAMESPACE
32
33
static bool serialLess(const Block &first, const Block &second)
34
{
35
    return first.endSerial() < second.endSerial();
36
}
37
38
const Block* VectorData::getBlockBySerial(long serial) const
39
{
40
    if (serial < 0 || serial >= getCount())
41
        return NULL;
42
43
    Block blockToFind;
44
    blockToFind.startSerial = serial;
45
    Blocks::const_iterator first = std::upper_bound(blocks.begin(), blocks.end(), blockToFind, serialLess);
46
    assert(first == blocks.end() || first->endSerial() > serial); // first block ending after serial
47
48
    if (first != blocks.end()) {
49
        assert(first->contains(serial));
50
        return &(*first);
51
    }
52
    else
53
        return NULL;
54
}
55
56
// ordering of blocks
57
static bool simtimeLess(const Block &first, const Block &second)
58
{
59
    return first.endTime < second.startTime;
60
}
61
62
// ordering of reversed blocks
63
static bool simtimeGreater(const Block &first, const Block &second)
64
{
65
    return first.startTime > second.endTime;
66
}
67
68
69
const Block *VectorData::getBlockBySimtime(simultime_t simtime, bool after) const
70
{
71
    Block blockToFind;
72
    blockToFind.startTime = simtime;
73
    blockToFind.endTime = simtime;
74
75
    if (after)
76
    {
77
        Blocks::const_iterator first = std::lower_bound(blocks.begin(), blocks.end(), blockToFind, simtimeLess);
78
        return first != blocks.end() ? &(*first) : NULL;
79
    }
80
    else
81
    {
82
        Blocks::const_reverse_iterator last = std::lower_bound(blocks.rbegin(), blocks.rend(), blockToFind, simtimeGreater);
83
        return last != blocks.rend() ? &(*last) : NULL;
84
    }
85
}
86
87
Blocks::size_type VectorData::getBlocksInSimtimeInterval(simultime_t startTime, simultime_t endTime, Blocks::size_type &startIndex, Blocks::size_type &endIndex) const
88
{
89
    Block blockToFind;
90
    blockToFind.startTime = startTime;
91
    blockToFind.endTime = endTime;
92
93
    Blocks::const_iterator first = std::lower_bound(blocks.begin(), blocks.end(), blockToFind, simtimeLess);
94
    Blocks::const_iterator last = std::upper_bound(blocks.begin(), blocks.end(), blockToFind, simtimeLess);
95
96
    assert(first == blocks.end() || first->endTime >= startTime);
97
    assert(last == blocks.end() || last->startTime > endTime);
98
    assert(first <= last);
99
100
    startIndex = first - blocks.begin();
101
    endIndex = last - blocks.begin();
102
    return endIndex - startIndex;
103
}
104
105
// ordering of blocks
106
static bool eventnumLess(const Block &first, const Block &second)
107
{
108
    return first.endEventNum < second.startEventNum;
109
}
110
111
// ordering of reversed blocks
112
static bool eventnumGreater(const Block &first, const Block &second)
113
{
114
    return first.startEventNum > second.endEventNum;
115
}
116
117
const Block *VectorData::getBlockByEventnum(eventnumber_t eventNum, bool after) const
118
{
119
    Block blockToFind;
120
    blockToFind.startEventNum = eventNum;
121
    blockToFind.endEventNum = eventNum;
122
123
    if (after)
124
    {
125
        Blocks::const_iterator first = std::lower_bound(blocks.begin(), blocks.end(), blockToFind, eventnumLess);
126
        return first != blocks.end() ? &(*first) : NULL;
127
    }
128
    else
129
    {
130
        Blocks::const_reverse_iterator last = std::lower_bound(blocks.rbegin(), blocks.rend(), blockToFind, eventnumGreater);
131
        return last != blocks.rend() ? &(*last) : NULL;
132
    }
133
}
134
135
Blocks::size_type VectorData::getBlocksInEventnumInterval(eventnumber_t startEventNum, eventnumber_t endEventNum, Blocks::size_type &startIndex, Blocks::size_type &endIndex) const
136
{
137
    Block blockToFind;
138
    blockToFind.startEventNum = startEventNum;
139
    blockToFind.endEventNum = endEventNum;
140
141
    Blocks::const_iterator first = std::lower_bound(blocks.begin(), blocks.end(), blockToFind, eventnumLess);
142
    Blocks::const_iterator last = std::upper_bound(blocks.begin(), blocks.end(), blockToFind, eventnumLess);
143
144
    assert(first == blocks.end() || first->endEventNum >= startEventNum);
145
    assert(last == blocks.end() || last->startEventNum > endEventNum);
146
    assert(first <= last);
147
148
    startIndex = first - blocks.begin();
149
    endIndex = last - blocks.begin();
150
    return endIndex - startIndex;
151
}
152
153
//=========================================================================
154
155
#ifdef CHECK
156
#undef CHECK
157
#endif
158
#define CHECK(cond,msg) if (!(cond)) throw ResultFileFormatException(msg, filename, lineNo);
159
160
bool RunData::parseLine(char **tokens, int numTokens, const char *filename, int64 lineNo)
161
{
162
    if (tokens[0][0] == 'a' && strcmp(tokens[0], "attr") == 0)
163
    {
164
        CHECK(numTokens >= 3, "'attr <name> <value>' expected");
165
        this->attributes[tokens[1]] = tokens[2];
166
        // the "runNumber" attribute is also stored separately
167
        if (strcmp(tokens[1], "runNumber") == 0) {
168
            CHECK(parseInt(tokens[2], this->runNumber), "runNumber: an integer expected");
169
        }
170
        return true;
171
    }
172
    else if (tokens[0][0] == 'p' && strcmp(tokens[0], "param") == 0)
173
    {
174
        CHECK(numTokens >= 3, "'param <namePattern> <value>' expected");
175
        this->moduleParams[tokens[1]] = tokens[2];
176
        return true;
177
    }
178
    else if (tokens[0][0] == 'r' && strcmp(tokens[0], "run") == 0)
179
    {
180
        CHECK(numTokens >= 2, "missing run name");
181
        this->runName = tokens[1];
182
        return true;
183
    }
184
185
    return false;
186
}
187
188
#undef CHECK
189
#define CHECK(fprintf)    if (fprintf<0) throw opp_runtime_error("Cannot write output file `%s'", filename)
190
191
void RunData::writeToFile(FILE *file, const char *filename) const
192
{
193
    if (runName.size() > 0)
194
    {
195
        CHECK(fprintf(file, "run %s\n", runName.c_str()));
196
    }
197
    for (StringMap::const_iterator it = attributes.begin(); it != attributes.end(); ++it)
198
    {
199
        CHECK(fprintf(file, "attr %s %s\n", it->first.c_str(), QUOTE(it->second.c_str())));
200
    }
201
    for (StringMap::const_iterator it = moduleParams.begin(); it != moduleParams.end(); ++it)
202
    {
203
        CHECK(fprintf(file, "param %s %s\n", it->first.c_str(), QUOTE(it->second.c_str())));
204
    }
205
}
206
//=========================================================================
207
static bool isFileReadable(const char *filename)
208
{
209
    FILE *f = fopen(filename, "r");
210
    if (f != NULL)
211
        fclose(f);
212
    return f != NULL;
213
}
214
215
bool IndexFile::isIndexFile(const char *filename)
216
{
217
    int len = strlen(filename);
218
    return (len >= 4) && (strcmp(filename+len-4, ".vci") == 0);
219
}
220
221
bool IndexFile::isVectorFile(const char *filename)
222
{
223
    // XXX check contents?
224
    int len = strlen(filename);
225
    return (len >= 4) && (strcmp(filename+len-4, ".vec") == 0);
226
}
227
228
std::string IndexFile::getVectorFileName(const char *filename)
229
{
230
    std::string vectorFileName(filename);
231
    std::string::size_type pos = vectorFileName.rfind('.');
232
    if (pos != std::string::npos)
233
        vectorFileName.replace(vectorFileName.begin()+pos, vectorFileName.end(), ".vec");
234
    else
235
        vectorFileName.append(".vec");
236
    return vectorFileName;
237
}
238
239
std::string IndexFile::getIndexFileName(const char *filename)
240
{
241
    std::string indexFileName(filename);
242
    std::string::size_type pos = indexFileName.rfind('.');
243
    if (pos != std::string::npos)
244
        indexFileName.replace(indexFileName.begin()+pos, indexFileName.end(), ".vci");
245
    else
246
        indexFileName.append(".vci");
247
    return indexFileName;
248
}
249
250
bool IndexFile::isIndexFileUpToDate(const char *filename)
251
{
252
    std::string indexFileName, vectorFileName;
253
254
    if (isIndexFile(filename))
255
    {
256
        indexFileName = std::string(filename);
257
        vectorFileName = getVectorFileName(filename);
258
    }
259
    else
260
    {
261
        indexFileName = getIndexFileName(filename);
262
        vectorFileName = std::string(filename);
263
    }
264
265
    if (!isFileReadable(indexFileName.c_str()))
266
        return false;
267
268
    IndexFileReader reader(indexFileName.c_str());
269
    VectorFileIndex *index = reader.readFingerprint();
270
271
    // when the fingerprint not found assume the index file is being written therefore it is up to date
272
    if (!index)
273
        return true;
274
275
    bool uptodate = index->fingerprint.check(vectorFileName.c_str());
276
    delete index;
277
    return uptodate;
278
}
279
280
//=========================================================================
281
FingerPrint::FingerPrint(const char *vectorFileName)
282
{
283
    struct opp_stat_t s;
284
    if (opp_stat(vectorFileName, &s) != 0)
285
        throw opp_runtime_error("vector file '%s' does not exist", vectorFileName);
286
287
    this->lastModified = (int64)s.st_mtime;
288
    this->fileSize = (int64)s.st_size;
289
}
290
291
bool FingerPrint::check(const char *vectorFileName)
292
{
293
    struct opp_stat_t s;
294
    if (opp_stat(vectorFileName, &s) == 0)
295
    {
296
        return (this->lastModified == (int64)s.st_mtime) && (this->fileSize == (int64)s.st_size);
297
    }
298
    return false;
299
}
300
301
//=========================================================================
302
303
#ifdef CHECK
304
#undef CHECK
305
#endif
306
#define CHECK(cond,msg,line) if (!(cond)) throw ResultFileFormatException(msg, filename.c_str(), line);
307
308
// see ifilemgr.h
309
#define MIN_BUFFER_SIZE 512
310
311
IndexFileReader::IndexFileReader(const char *filename)
312
    : filename(filename)
313
{
314
}
315
316
VectorFileIndex *IndexFileReader::readAll()
317
{
318
    // read vector and blocks
319
    FileReader reader(filename.c_str());
320
    LineTokenizer tokenizer(1024);
321
    int numTokens;
322
    char *line, **tokens;
323
324
    VectorFileIndex *index = new VectorFileIndex();
325
    while ((line=reader.getNextLineBufferPointer())!=NULL)
326
    {
327
        int64 lineNum = reader.getNumReadLines();
328
        int len=reader.getCurrentLineLength();
329
        numTokens=tokenizer.tokenize(line, len);
330
        tokens=tokenizer.tokens();
331
        parseLine(tokens, numTokens, index, lineNum);
332
    }
333
    return index;
334
}
335
336
VectorFileIndex *IndexFileReader::readFingerprint()
337
{
338
    FileReader reader(filename.c_str());
339
    LineTokenizer tokenizer(1024);
340
    int numTokens;
341
    char *line, **tokens;
342
343
    VectorFileIndex *index = NULL;
344
    while ((line=reader.getNextLineBufferPointer())!=NULL)
345
    {
346
        int64 lineNum = reader.getNumReadLines();
347
        int len = reader.getCurrentLineLength();
348
        numTokens = tokenizer.tokenize(line,len);
349
        tokens = tokenizer.tokens();
350
351
        if (numTokens == 0 || tokens[0][0] == '#')
352
            continue;
353
        else if (tokens[0][0] == 'f' && strcmp(tokens[0], "file") == 0)
354
        {
355
            index = new VectorFileIndex();
356
            parseLine(tokens, numTokens, index, lineNum);
357
        }
358
        else
359
            break;
360
    }
361
362
    // missing fingerprint: possible if the writing of the index file is in progress
363
    // CHECK(index, "missing fingerprint", -1);
364
365
    return index;
366
}
367
368
369
void IndexFileReader::parseLine(char **tokens, int numTokens, VectorFileIndex *index, int64 lineNum)
370
{
371
    if (numTokens == 0 || tokens[0][0] == '#')
372
        return;
373
374
    long count;
375
    double min;
376
    double max;
377
    double sum;
378
    double sumSqr;
379
380
    if (tokens[0][0] == 'v' && strcmp(tokens[0], "vector") == 0)
381
    {
382
        CHECK(numTokens >= 5, "invalid vector declaration", lineNum);
383
384
        VectorData vector;
385
        CHECK(parseInt(tokens[1], vector.vectorId), "invalid vector id", lineNum);
386
        vector.moduleName = tokens[2];
387
        vector.name = tokens[3];
388
        vector.columns = tokens[4];
389
390
        index->addVector(vector);
391
    }
392
    else if (tokens[0][0] == 'a' && strcmp(tokens[0], "attr") == 0 && index->getNumberOfVectors() > 0) // vector attr
393
    {
394
        CHECK(numTokens == 3, "malformed vector attribute", lineNum);
395
        VectorData *lastVector = index->getVectorAt(index->getNumberOfVectors()-1);
396
        lastVector->attributes[tokens[1]] = tokens[2];
397
    }
398
    else if (tokens[0][0] == 'f' && strcmp(tokens[0], "file") == 0)
399
    {
400
        int64 fileSize;
401
        int64 lastModified;
402
        CHECK(numTokens >= 3, "missing file attributes", lineNum);
403
        CHECK(parseInt64(tokens[1], fileSize), "file size is not a number", lineNum);
404
        CHECK(parseInt64(tokens[2], lastModified), "modification date is not a number", lineNum);
405
        index->fingerprint.fileSize = fileSize;
406
        index->fingerprint.lastModified = lastModified;
407
    }
408
    else if (tokens[0][0] == 'v' && strcmp(tokens[0], "version") == 0)
409
    {
410
        int version;
411
        CHECK(numTokens >= 2, "missing version number", lineNum);
412
        CHECK(parseInt(tokens[1], version), "version is not a number", lineNum);
413
        CHECK(version <= 2, "expects version 2 or lower", lineNum);
414
    }
415
    else if (index->run.parseLine(tokens, numTokens, filename.c_str(), lineNum))
416
    {
417
        return;
418
    }
419
    else // blocks
420
    {
421
        CHECK(numTokens >= 10, "missing fields from block", lineNum);
422
423
        int id;
424
        CHECK(parseInt(tokens[0], id), "malformed vector id", lineNum);
425
        VectorData *vector = index->getVectorById(id);
426
        CHECK(vector, "missing vector definition", lineNum);
427
428
        Block block;
429
        block.startSerial = vector->blocks.size() > 0 ? vector->blocks.back().endSerial() : 0;
430
        int i = 1; // column index
431
        CHECK(parseInt64(tokens[i++], block.startOffset), "invalid file offset", lineNum);
432
        CHECK(parseInt64(tokens[i++], block.size), "invalid block size", lineNum);
433
        if (vector->hasColumn('E'))
434
        {
435
            CHECK(parseInt64(tokens[i++], block.startEventNum) && parseInt64(tokens[i++], block.endEventNum),
436
                "invalid event numbers", lineNum);
437
        }
438
        if (vector->hasColumn('T'))
439
        {
440
            CHECK(parseSimtime(tokens[i++], block.startTime) && parseSimtime(tokens[i++], block.endTime),
441
                "invalid simulation time", lineNum);
442
        }
443
        if (vector->hasColumn('V'))
444
        {
445
            CHECK(parseLong(tokens[i++], count) && parseDouble(tokens[i++], min) && parseDouble(tokens[i++], max) &&
446
                    parseDouble(tokens[i++], sum) && parseDouble(tokens[i++], sumSqr), "invalid statistics data", lineNum);
447
            block.stat = Statistics(count, min, max, sum, sumSqr);
448
        }
449
        vector->addBlock(block);
450
    }
451
}
452
453
//=========================================================================
454
455
#ifdef CHECK
456
#undef CHECK
457
#endif
458
#define CHECK(fprintf)    if (fprintf<0) throw opp_runtime_error("Cannot write output file `%s'", filename.c_str())
459
#define INDEX_FILE_VERSION 2
460
461
IndexFileWriter::IndexFileWriter(const char *filename, int precision)
462
    : filename(filename), precision(precision), file(NULL)
463
{
464
}
465
466
IndexFileWriter::~IndexFileWriter()
467
{
468
    if (file != NULL)
469
    {
470
        closeFile();
471
    }
472
}
473
474
void IndexFileWriter::writeAll(const VectorFileIndex& index)
475
{
476
    openFile();
477
    writeFingerprint(index.vectorFileName);
478
    writeRun(index.run);
479
480
    int numOfVectors = index.getNumberOfVectors();
481
    for (int i = 0; i < numOfVectors; ++i)
482
    {
483
        const VectorData *vectorRef = index.getVectorAt(i);
484
        writeVector(*vectorRef);
485
    }
486
487
    closeFile();
488
}
489
490
void IndexFileWriter::writeFingerprint(std::string vectorFileName)
491
{
492
    FingerPrint fingerprint(vectorFileName.c_str());
493
494
    if (file == NULL)
495
        openFile();
496
497
    file_offset_t saveOffset = opp_ftell(file);
498
    opp_fseek(file, 0, SEEK_SET);
499
    CHECK(fprintf(file, "file %"LL"d %"LL"d", fingerprint.fileSize, fingerprint.lastModified));
500
    opp_fseek(file, saveOffset, SEEK_SET);
501
}
502
503
void IndexFileWriter::writeRun(const RunData &run)
504
{
505
    if (file == NULL)
506
        openFile();
507
    run.writeToFile(file, filename.c_str());
508
}
509
510
void IndexFileWriter::writeVector(const VectorData &vector)
511
{
512
    if (file == NULL)
513
        openFile();
514
515
    int numBlocks = vector.blocks.size();
516
    if (numBlocks > 0)
517
    {
518
        writeVectorDeclaration(vector);
519
        writeVectorAttributes(vector);
520
521
        for (int i=0; i<numBlocks; i++)
522
        {
523
            writeBlock(vector, vector.blocks[i]);
524
        }
525
    }
526
}
527
528
void IndexFileWriter::writeVectorDeclaration(const VectorData &vector)
529
{
530
    CHECK(fprintf(file, "vector %d  %s  %s  %s\n",
531
          vector.vectorId, QUOTE(vector.moduleName.c_str()), QUOTE(vector.name.c_str()), vector.columns.c_str()));
532
533
}
534
535
void IndexFileWriter::writeVectorAttributes(const VectorData &vector)
536
{
537
    for (StringMap::const_iterator it=vector.attributes.begin(); it != vector.attributes.end(); ++it)
538
    {
539
        CHECK(fprintf(file, "attr %s %s\n", QUOTE(it->first.c_str()), QUOTE(it->second.c_str())));
540
    }
541
}
542
543
void IndexFileWriter::writeBlock(const VectorData &vector, const Block &block)
544
{
545
    static char buff1[64], buff2[64];
546
    char *e;
547
548
    if (block.getCount() > 0)
549
    {
550
        CHECK(fprintf(file, "%d\t%"LL"d %"LL"d", vector.vectorId, (int64)block.startOffset, (int64)block.size));
551
        if (vector.hasColumn('E')) { CHECK(fprintf(file, " %"LL"d %"LL"d", block.startEventNum, block.endEventNum)); }
552
        if (vector.hasColumn('T')) { CHECK(fprintf(file, " %s %s",
553
                                                BigDecimal::ttoa(buff1, block.startTime, e),
554
                                                BigDecimal::ttoa(buff2, block.endTime, e))); }
555
        if (vector.hasColumn('V')) { CHECK(fprintf(file, " %ld %.*g %.*g %.*g %.*g",
556
                                                block.getCount(), precision, block.getMin(), precision, block.getMax(),
557
                                                precision, block.getSum(), precision, block.getSumSqr())); }
558
        CHECK(fprintf(file, "\n"));
559
    }
560
}
561
562
void IndexFileWriter::openFile()
563
{
564
    file = fopen(filename.c_str(), "w");
565
    if (file == NULL)
566
        throw opp_runtime_error("Cannot open index file: %s", filename.c_str());
567
568
    setlocale(LC_NUMERIC, "C");
569
570
    // space for header
571
    CHECK(fprintf(file, "%64s\n", ""));
572
    // version
573
    CHECK(fprintf(file, "version %d\n", INDEX_FILE_VERSION));
574
}
575
576
void IndexFileWriter::closeFile()
577
{
578
    if (file != NULL)
579
    {
580
        fclose(file);
581
        file = NULL;
582
    }
583
}