Project

General

Profile

Statistics
| Branch: | Revision:

root / src / scave / vectorfileindexer.cc @ a3be1d55

History | View | Annotate | Download (14.4 KB)

1 01873262 Georg Kunz
//=========================================================================
2
//  VECTORFILEINDEXER.CC - part of
3
//                  OMNeT++/OMNEST
4
//           Discrete System Simulation in C++
5
//
6
//  Author: Tamas Borbely
7
//
8
//=========================================================================
9
10
/*--------------------------------------------------------------*
11
  Copyright (C) 2006-2008 OpenSim Ltd.
12

13
  This file is distributed WITHOUT ANY WARRANTY. See the file
14
  `license' for details on this and other legal matters.
15
*--------------------------------------------------------------*/
16
17
#include <sys/stat.h>
18
#include <errno.h>
19
#include <sstream>
20
#include <ostream>
21
#include <stdlib.h>
22
#include "opp_ctype.h"
23
#include "platmisc.h"
24
#include "stringutil.h"
25
#include "scaveutils.h"
26
#include "scaveexception.h"
27
#include "filereader.h"
28
#include "linetokenizer.h"
29
#include "dataflowmanager.h"
30
#include "indexfile.h"
31
#include "indexedvectorfile.h"
32
#include "nodetyperegistry.h"
33
#include "vectorfileindexer.h"
34
35
USING_NAMESPACE
36
37
using namespace std;
38
39
static inline bool existsFile(const string fileName)
40
{
41
    struct stat s;
42
    return stat(fileName.c_str(), &s)==0;
43
}
44
45
static string createTempFileName(const string baseFileName)
46
{
47
    string prefix = baseFileName;
48
    prefix.append(".temp");
49
    string tmpFileName = prefix;
50
    int serial = 0;
51
52
    while (existsFile(tmpFileName))
53
        tmpFileName = opp_stringf("%s%d", prefix.c_str(), serial++);
54
    return tmpFileName;
55
}
56
57
// TODO: adjacent blocks are merged
58
void VectorFileIndexer::generateIndex(const char *vectorFileName, IProgressMonitor *monitor)
59
{
60
    FileReader reader(vectorFileName);
61
    LineTokenizer tokenizer(1024);
62
    VectorFileIndex index;
63
    index.vectorFileName = vectorFileName;
64
65
    char *line;
66
    char **tokens;
67
    int64 lineNo;
68
    int numTokens, numOfUnrecognizedLines = 0;
69
    VectorData *currentVectorRef = NULL;
70
    VectorData *lastVectorDecl = NULL;
71
    Block currentBlock;
72
73
    int64 onePercentFileSize = reader.getFileSize() / 100;
74
    int readPercentage = 0;
75
76
    if (monitor)
77
        monitor->beginTask(string("Indexing ")+vectorFileName, 110);
78
79
    try
80
    {
81
82
        while ((line=reader.getNextLineBufferPointer())!=NULL)
83
        {
84
            if (monitor)
85
            {
86
                if (monitor->isCanceled())
87
                {
88
                    monitor->done();
89
                    return;
90
                }
91
                if (onePercentFileSize > 0)
92
                {
93
                    int64 readBytes = reader.getNumReadBytes();
94
                    int currentPercentage = readBytes / onePercentFileSize;
95
                    if (currentPercentage > readPercentage)
96
                    {
97
                        monitor->worked(currentPercentage - readPercentage);
98
                        readPercentage = currentPercentage;
99
                    }
100
                }
101
            }
102
103
            tokenizer.tokenize(line, reader.getCurrentLineLength());
104
            numTokens = tokenizer.numTokens();
105
            tokens = tokenizer.tokens();
106
            lineNo = reader.getNumReadLines();
107
108
            if (numTokens == 0 || tokens[0][0] == '#')
109
                continue;
110
            else if ((tokens[0][0] == 'r' && strcmp(tokens[0], "run") == 0) ||
111
                     (tokens[0][0] == 'p' && strcmp(tokens[0], "param") == 0))
112
            {
113
                index.run.parseLine(tokens, numTokens, vectorFileName, lineNo);
114
            }
115
            else if (tokens[0][0] == 'a' && strcmp(tokens[0], "attr") == 0)
116
            {
117
                if (lastVectorDecl == NULL) // run attribute
118
                {
119
                    index.run.parseLine(tokens, numTokens, vectorFileName, lineNo);
120
                }
121
                else // vector attribute
122
                {
123
                    if (numTokens < 3)
124
                        throw ResultFileFormatException("vector file indexer: missing attribute name or value", vectorFileName, lineNo);
125
                    lastVectorDecl->attributes[tokens[1]] = tokens[2];
126
                }
127
            }
128
            else if (tokens[0][0] == 'v' && strcmp(tokens[0], "vector") == 0)
129
            {
130
                if (numTokens < 4)
131
                    throw ResultFileFormatException("vector file indexer: broken vector declaration", vectorFileName, lineNo);
132
133
                VectorData vector;
134
                if (!parseInt(tokens[1], vector.vectorId))
135
                    throw ResultFileFormatException("vector file indexer: malformed vector in vector declaration", vectorFileName, lineNo);
136
                vector.moduleName = tokens[2];
137
                vector.name = tokens[3];
138
                vector.columns = (numTokens < 5 || opp_isdigit(tokens[4][0]) ? "TV" : tokens[4]);
139
                vector.blockSize = 0;
140
141
                index.addVector(vector);
142
                lastVectorDecl = index.getVectorAt(index.getNumberOfVectors() - 1);
143
                currentVectorRef = NULL;
144
            }
145
            else if (tokens[0][0] == 'v' && strcmp(tokens[0], "version") == 0)
146
            {
147
                int version;
148
                if(numTokens < 2)
149
                    throw ResultFileFormatException("vector file indexer: missing version number", vectorFileName, lineNo);
150
                if(!parseInt(tokens[1], version))
151
                    throw ResultFileFormatException("vector file indexer: version is not a number", vectorFileName, lineNo);
152
                if(version > 2)
153
                    throw ResultFileFormatException("vector file indexer: expects version 2 or lower", vectorFileName, lineNo);
154
            }
155
            else // data line
156
            {
157
                int vectorId;
158
                simultime_t simTime;
159
                double value;
160
                eventnumber_t eventNum = -1;
161
162
                if (!parseInt(tokens[0], vectorId))
163
                {
164
                    numOfUnrecognizedLines++;
165
                    continue;
166
                }
167
168
                if (currentVectorRef == NULL || vectorId != currentVectorRef->vectorId)
169
                {
170
                    if (currentVectorRef != NULL)
171
                    {
172
                        currentBlock.size = (int64)(reader.getCurrentLineStartOffset() - currentBlock.startOffset);
173
                        if (currentBlock.size > currentVectorRef->blockSize)
174
                            currentVectorRef->blockSize = currentBlock.size;
175
                        currentVectorRef->addBlock(currentBlock);
176
                    }
177
178
                    currentBlock = Block();
179
                    currentBlock.startOffset = reader.getCurrentLineStartOffset();
180
                    currentVectorRef = index.getVectorById(vectorId);
181
                    if (currentVectorRef == NULL)
182
                        throw ResultFileFormatException("vector file indexer: missing vector declaration", vectorFileName, lineNo);
183
                }
184
185
                for (int i = 0; i < (int)currentVectorRef->columns.size(); ++i)
186
                {
187
                    char column = currentVectorRef->columns[i];
188
                    if (i+1 >= numTokens)
189
                        throw ResultFileFormatException("vector file indexer: data line too short", vectorFileName, lineNo);
190
191
                    char *token = tokens[i+1];
192
                    switch (column)
193
                    {
194
                    case 'T':
195
                        if (!parseSimtime(token, simTime))
196
                            throw ResultFileFormatException("vector file indexer: malformed simulation time", vectorFileName, lineNo);
197
                        break;
198
                    case 'V':
199
                        if (!parseDouble(token, value))
200
                            throw ResultFileFormatException("vector file indexer: malformed data value", vectorFileName, lineNo);
201
                        break;
202
                    case 'E':
203
                        if (!parseInt64(token, eventNum))
204
                            throw ResultFileFormatException("vector file indexer: malformed event number", vectorFileName, lineNo);
205
                        break;
206
                    }
207
                }
208
209
                currentBlock.collect(eventNum, simTime, value);
210
            }
211
        }
212
213
        // finish last block
214
        if (currentBlock.getCount() > 0)
215
        {
216
            assert(currentVectorRef != NULL);
217
            currentBlock.size = (int64)(reader.getFileSize() - currentBlock.startOffset);
218
            if (currentBlock.size > currentVectorRef->blockSize)
219
                currentVectorRef->blockSize = currentBlock.size;
220
            currentVectorRef->addBlock(currentBlock);
221
        }
222
223
        if (numOfUnrecognizedLines > 0)
224
        {
225
            fprintf(stderr, "Found %d unrecognized lines in %s.\n", numOfUnrecognizedLines, vectorFileName);
226
        }
227
    }
228
    catch (exception&)
229
    {
230
        if (monitor)
231
            monitor->done();
232
        throw;
233
    }
234
235
236
    if (monitor)
237
    {
238
        if (monitor->isCanceled())
239
        {
240
            monitor->done();
241
            return;
242
        }
243
        if (readPercentage < 100)
244
            monitor->worked(100 - readPercentage);
245
    }
246
247
    // generate index file: first write it to a temp file then rename it to .vci;
248
    // we do this in order to prevent race conditions from other processes/threads
249
    // reading an incomplete .vci file
250
    string indexFileName = IndexFile::getIndexFileName(vectorFileName);
251
    string tempIndexFileName = createTempFileName(indexFileName);
252
253
    try
254
    {
255
        IndexFileWriter writer(tempIndexFileName.c_str());
256
        writer.writeAll(index);
257
258
        if (monitor)
259
            monitor->worked(10);
260
261
        // rename generated index file
262
        if (unlink(indexFileName.c_str())!=0 && errno!=ENOENT)
263
            throw opp_runtime_error("Cannot remove original index file `%s': %s", indexFileName.c_str(), strerror(errno));
264
        if (rename(tempIndexFileName.c_str(), indexFileName.c_str())!=0)
265
            throw opp_runtime_error("Cannot rename index file from '%s' to '%s': %s", tempIndexFileName.c_str(), indexFileName.c_str(), strerror(errno));
266
    }
267
    catch (exception&)
268
    {
269
        if (monitor)
270
            monitor->done();
271
272
        // if something wrong happened, we remove the temp files
273
        unlink(indexFileName.c_str());
274
        unlink(tempIndexFileName.c_str());
275
        throw;
276
    }
277
278
    if (monitor)
279
        monitor->done();
280
}
281
282
void VectorFileIndexer::rebuildVectorFile(const char *vectorFileName, IProgressMonitor *monitor)
283
{
284
    string indexFileName = IndexFile::getIndexFileName(vectorFileName);
285
    string tempIndexFileName = createTempFileName(indexFileName);
286
    string tempVectorFileName = createTempFileName(vectorFileName);
287
288
    try
289
    {
290
        // load file
291
        ResultFileManager resultFileManager;
292
        ResultFile *f = resultFileManager.loadFile(vectorFileName);
293
        if (!f)
294
            throw opp_runtime_error("Cannot load %s", vectorFileName);
295
        else if (f->numUnrecognizedLines>0)
296
            fprintf(stderr, "WARNING: %s: %d invalid/incomplete lines out of %d\n", vectorFileName, f->numUnrecognizedLines, f->numLines);
297
298
        IDList vectorIDList = resultFileManager.getAllVectors();
299
        if (vectorIDList.isEmpty()) {
300
            fprintf(stderr, "WARNING: %s: no vectors found\n", vectorFileName);
301
            return;
302
        }
303
304
        const RunList &runList = resultFileManager.getRuns();
305
        if (runList.size() != 1)
306
            throw opp_runtime_error("Multiple runs found in %s", vectorFileName);
307
        const Run *runPtr = runList[0];
308
309
        //
310
        // assemble dataflow network for vectors
311
        //
312
        {
313
            DataflowManager dataflowManager;
314
            NodeTypeRegistry *registry = NodeTypeRegistry::getInstance();
315
316
            // create reader node
317
            NodeType *readerNodeType = registry->getNodeType("vectorfilereader");
318
            if (!readerNodeType)
319
                throw opp_runtime_error("There is no node type 'vectorfilereader' in the registry\n");
320
            StringMap attrs;
321
            attrs["filename"] = vectorFileName;
322
            Node *readerNode = readerNodeType->create(&dataflowManager, attrs);
323
324
            // create writer node
325
            NodeType *writerNodeType = registry->getNodeType("indexedvectorfilewriter");
326
            if (!writerNodeType)
327
                throw opp_runtime_error("There is no node type 'indexedvectorfilewriter' in the registry\n");
328
            StringMap writerAttrs;
329
            writerAttrs["filename"] = tempVectorFileName;
330
            writerAttrs["indexfilename"] = tempIndexFileName;
331
            writerAttrs["blocksize"] = "65536"; // TODO
332
            writerAttrs["fileheader"] = "# generated by scavetool";
333
            IndexedVectorFileWriterNode *writerNode =
334
                dynamic_cast<IndexedVectorFileWriterNode*>(writerNodeType->create(&dataflowManager, writerAttrs));
335
            if (!writerNode)
336
                throw opp_runtime_error("Cannot create the indexedvectorfilewriternode.");
337
            writerNode->setRun(runPtr->runName.c_str(), runPtr->attributes, runPtr->moduleParams);
338
339
340
            // create a ports
341
            for (int i=0; i<vectorIDList.size(); i++)
342
            {
343
                char portName[30];
344
                const VectorResult& vector = resultFileManager.getVector(vectorIDList.get(i));
345
                sprintf(portName, "%d", vector.vectorId);
346
                Port *outPort = readerNodeType->getPort(readerNode, portName);
347
                Port *inPort = writerNode->addVector(vector);
348
                dataflowManager.connect(outPort, inPort);
349
            }
350
351
            // run!
352
            dataflowManager.execute(monitor);
353
354
            // dataflowManager deleted here, and its destructor closes the files
355
        }
356
357
        // rename temp to orig
358
        if (unlink(indexFileName.c_str())!=0 && errno!=ENOENT)
359
            throw opp_runtime_error("Cannot remove original index file `%s': %s", indexFileName.c_str(), strerror(errno));
360
        if (unlink(vectorFileName)!=0)
361
            throw opp_runtime_error("Cannot remove original vector file `%s': %s", vectorFileName, strerror(errno));
362
        if (rename(tempVectorFileName.c_str(), vectorFileName)!=0)
363
            throw opp_runtime_error("Cannot move generated vector file '%s' to the original '%s': %s",
364
                    tempVectorFileName.c_str(), vectorFileName, strerror(errno));
365
        if (rename(tempIndexFileName.c_str(), indexFileName.c_str())!=0)
366
            throw opp_runtime_error("Cannot move generated index file from '%s' to '%s': %s", tempIndexFileName.c_str(), indexFileName.c_str(), strerror(errno));
367
    }
368
    catch (exception& e)
369
    {
370
        // cleanup temp files
371
        unlink(tempIndexFileName.c_str());
372
        if (existsFile(vectorFileName))
373
            unlink(tempVectorFileName.c_str());
374
375
        throw;
376
    }
377
}
378