Project

General

Profile

Statistics
| Branch: | Revision:

root / src / scave / resultfilemanager.h @ a3be1d55

History | View | Annotate | Download (18.3 KB)

1
//=========================================================================
2
//  RESULTFILEMANAGER.H - part of
3
//                  OMNeT++/OMNEST
4
//           Discrete System Simulation in C++
5
//
6
//  Author: Andras Varga, Tamas Borbely
7
//
8
//=========================================================================
9

    
10
/*--------------------------------------------------------------*
11
  Copyright (C) 1992-2008 Andras Varga
12
  Copyright (C) 2006-2008 OpenSim Ltd.
13

14
  This file is distributed WITHOUT ANY WARRANTY. See the file
15
  `license' for details on this and other legal matters.
16
*--------------------------------------------------------------*/
17

    
18
#ifndef _RESULTFILEMANAGER_H_
19
#define _RESULTFILEMANAGER_H_
20

    
21
#include <assert.h>
22
#include <math.h>
23
#include <string>
24
#include <vector>
25
#include <set>
26
#include <map>
27
#include <list>
28

    
29
#include "idlist.h"
30
#include "enumtype.h"
31
#include "exception.h"
32
#include "commonutil.h"
33
#include "statistics.h"
34
#include "scaveutils.h"
35

    
36
#ifdef THREADED
37
#include "rwlock.h"
38
#endif
39

    
40
NAMESPACE_BEGIN
41

    
42
class Run;
43
class ResultFile;
44
class FileRun;
45
class ResultFileManager;
46

    
47
typedef std::map<std::string, std::string> StringMap;
48

    
49
typedef int64 ComputationID;
50
typedef void* ComputationNode;
51

    
52
/**
53
 * Item in an output scalar or output vector file. Represents common properties
54
 * of an output vector or output scalar.
55
 */
56
struct SCAVE_API ResultItem
57
{
58
    enum Type { TYPE_INT, TYPE_DOUBLE, TYPE_ENUM };
59

    
60
    FileRun *fileRunRef; // backref to containing FileRun
61
    const std::string *moduleNameRef; // points into ResultFileManager's StringSet
62
    const std::string *nameRef; // scalarname or vectorname; points into ResultFileManager's StringSet
63
    StringMap attributes; // metadata in key/value form
64
    ComputationNode computation;
65

    
66
    ResultItem() : fileRunRef(NULL), moduleNameRef(NULL), nameRef(NULL), computation(NULL) {}
67

    
68
    const char *getAttribute(const char *attrName) const {
69
        StringMap::const_iterator it = attributes.find(attrName);
70
        return it==attributes.end() ? NULL : it->second.c_str();
71
    }
72

    
73
    /**
74
     * Returns the type of this result item (INT,DOUBLE,ENUM).
75
     * If neither "type" nor "enum" attribute is given it returns DOUBLE.
76
     */
77
    Type getType() const;
78

    
79
    /**
80
     * Returns a pointer to the enum type described by the "enum" attribute
81
     * or NULL if no "enum" attribute.
82
     */
83
    EnumType* getEnum() const;
84

    
85
    bool isComputed() const { return computation != NULL; }
86
};
87

    
88
/**
89
 * Represents an output scalar
90
 */
91
struct SCAVE_API ScalarResult : public ResultItem
92
{
93
    double value;
94
    bool isField;
95
};
96

    
97
/**
98
 * Represents an output vector. This is only the declaration,
99
 * actual vector data are not kept in memory.
100
 */
101
// TODO: there's a missing superclass between vectors and histograms that provides statistics, see the various sort<foo>By<bar> methods we need to make it work
102
struct SCAVE_API VectorResult : public ResultItem
103
{
104
    int vectorId;
105
    std::string columns;
106
    eventnumber_t startEventNum, endEventNum;
107
    simultime_t startTime, endTime;
108
    Statistics stat;
109

    
110
    VectorResult() : vectorId(-1), startEventNum(-1), endEventNum(-1), startTime(0.0), endTime(0.0) {}
111

    
112
    long getCount()      const { return stat.getCount(); }
113
    double getMin()      const { return stat.getMin(); }
114
    double getMax()      const { return stat.getMax(); }
115
    double getMean()     const { return stat.getMean(); }
116
    double getVariance() const { return stat.getVariance(); }
117
    double getStddev()   const { return stat.getStddev(); }
118

    
119
    /**
120
     * Returns the value of the "interpolation-mode" attribute as an InterpolationMode,
121
     * defaults to UNSPECIFIED.
122
     */
123
    InterpolationMode getInterpolationMode() const;
124
};
125

    
126
/**
127
 * Represents a histogram.
128
 */
129
// TODO: there's a missing superclass between vectors and histograms that provides statistics, see the various sort<foo>By<bar> methods we need to make it work
130
struct SCAVE_API HistogramResult : public ResultItem
131
{
132
    Statistics stat;
133
    std::vector<double> bins;
134
    std::vector<double> values;
135

    
136
    long getCount()      const { return stat.getCount(); }
137
    double getMin()      const { return stat.getMin(); }
138
    double getMax()      const { return stat.getMax(); }
139
    double getMean()     const { return stat.getMean(); }
140
    double getVariance() const { return stat.getVariance(); }
141
    double getStddev()   const { return stat.getStddev(); }
142

    
143
    void addBin(double lower_bound, double value);
144
};
145

    
146
typedef std::vector<ScalarResult> ScalarResults;
147
typedef std::vector<VectorResult> VectorResults;
148
typedef std::vector<HistogramResult> HistogramResults;
149

    
150
typedef std::vector<Run*> RunList;
151
typedef std::vector<ResultFile*> ResultFileList;
152
typedef std::vector<FileRun *> FileRunList;
153

    
154
/**
155
 * Represents a loaded scalar or vector file.
156
 */
157
struct SCAVE_API ResultFile
158
{
159
    int id;  // position in fileList
160
    ResultFileManager *resultFileManager; // backref to containing ResultFileManager
161
    std::string fileSystemFilePath; // directory+fileName of underlying file (for fopen())
162
    std::string directory; // directory's location in the Eclipse workspace
163
    std::string fileName; // file name
164
    std::string filePath; // workspace directory + fileName
165
    bool computed;
166
    ScalarResults scalarResults;
167
    VectorResults vectorResults;
168
    HistogramResults histogramResults;
169
    int numLines;
170
    int numUnrecognizedLines;
171
};
172

    
173
/**
174
 * Represents a run. If several scalar or vector files contain
175
 * the same run (i.e. runName is the same), they will share
176
 * the same Run object.
177
 */
178
struct SCAVE_API Run
179
{
180
    std::string runName; // unique identifier for the run, "runId"
181
    ResultFileManager *resultFileManager; // backref to containing ResultFileManager
182

    
183
    // various attributes of the run are stored in a string map.
184
    // keys include: runNumber, networkName, datetime, experiment, measurement, replication
185
    StringMap attributes;
186
    int runNumber; // this is stored separately as well, for convenience
187

    
188
    // module parameters: maps wildcard pattern to value
189
    StringMap moduleParams;
190

    
191
    // utility methods to non-disruptive access to the maps (note that evaluating
192
    // attributes["nonexisting-key"] would create a blank entry with that key!)
193
    const char *getAttribute(const char *attrName) const {
194
        StringMap::const_iterator it = attributes.find(attrName);
195
        return it==attributes.end() ? NULL : it->second.c_str();
196
    }
197
    const char *getModuleParam(const char *paramName) const {
198
        StringMap::const_iterator it = moduleParams.find(paramName);
199
        return it==moduleParams.end() ? NULL : it->second.c_str();
200
    }
201
};
202

    
203

    
204
/**
205
 * Represents a run in a result file. Such item is needed because
206
 * result files and runs are in many-to-many relationship: a result file
207
 * may contain more than one runs (.sca file), and during a simulation run
208
 * (represented by struct Run) more than one result files are written into
209
 * (namely, a .vec and a .sca file, or sometimes many of them).
210
 * And ResultItems refer to a FileRun instead of a ResultFile and a Run,
211
 * to conserve memory.
212
 */
213
struct SCAVE_API FileRun
214
{
215
    ResultFile *fileRef;
216
    Run *runRef;
217
};
218

    
219
typedef std::set<std::string> StringSet;
220
typedef std::vector<std::string> StringVector;
221

    
222
typedef std::map<std::pair<ComputationID, ID> , ID> ComputedIDCache;
223

    
224
class CmpBase;
225

    
226
/**
227
 * Loads and efficiently stores OMNeT++ output scalar files and output
228
 * vector files. (Actual vector contents in vector files are not read
229
 * into memory though, only the list of vectors.)
230
 *
231
 * Gives access to the list of module names and scalar data labels used in
232
 * the file as well as the list of all data.
233
 *
234
 * Data can be extracted, filtered, etc by external functions.
235
 */
236
class SCAVE_API ResultFileManager
237
{
238
    friend class IDList;  // _type()
239
    friend class CmpBase; // uncheckedGet...()
240
  private:
241
    // List of files loaded. This vector can have holes (NULLs) in it due to
242
    // unloaded files. The "id" field of ResultFile is the index into this vector.
243
    // It is not allowed to move elements, because IDs contain the file's index in them.
244
    ResultFileList fileList;
245

    
246
    // List of unique runs in the files. If several files contain the same runName,
247
    // it will generate only one Run entry here.
248
    RunList runList;
249

    
250
    // ResultFiles and Runs have many-to-many relationship. This is where we store
251
    // their relations (instead of having a collection in both). ResultItems also
252
    // contain a FileRun pointer instead of separate ResultFile and Run pointers.
253
    FileRunList fileRunList;
254

    
255
    // module names and variable names are stringpooled to conserve space
256
    StringPool moduleNames;
257
    StringPool names;
258
    StringPool classNames; // currently not used
259

    
260
    ComputedIDCache computedIDCache;
261
#ifdef THREADED
262
    ReentrantReadWriteLock lock;
263
#endif
264

    
265
    struct sParseContext
266
    {
267
        ResultFile *fileRef; /*in*/
268
        const char *fileName; /*in*/
269
        int64 lineNo; /*inout*/
270
        FileRun *fileRunRef; /*inout*/
271
        // references to the result items which attributes should be added to
272
        int lastResultItemType; /*inout*/
273
        int lastResultItemIndex; /*inout*/
274
        // collected fields of the histogram to be created when the
275
        // first 'bin' is parsed
276
        std::string moduleName;
277
        std::string statisticName;
278
        long count;
279
        double min, max, sum, sumSqr;
280

    
281
        sParseContext(ResultFile *fileRef)
282
            : fileRef(fileRef), fileName(fileRef->filePath.c_str()), lineNo(0),
283
              fileRunRef(NULL), lastResultItemType(0), lastResultItemIndex(-1),
284
              count(0), min(POSITIVE_INFINITY), max(NEGATIVE_INFINITY), sum(0.0), sumSqr(0.0) {}
285

    
286
        void clearHistogram()
287
        {
288
            moduleName.clear();
289
            statisticName.clear();
290
            count = 0;
291
            min = POSITIVE_INFINITY;
292
            max = NEGATIVE_INFINITY;
293
            sum = 0.0;
294
            sumSqr = 0.0;
295
        }
296
    };
297

    
298
  public:
299
    enum {SCALAR=1, VECTOR=2, HISTOGRAM=4}; // must be 1,2,4,8 etc, because of IDList::getItemTypes()
300

    
301
  private:
302
    // ID: 1 bit computed, 1 bit field, 6 bit type, 24 bit fileid, 32 bit pos
303
    static bool _computed(ID id) { return (id >> 63) != 0; }
304
    static bool _field(ID id) { return (id >> 62) != 0; }
305
    static int _type(ID id)   {return (id >> 56) & 0x3fUL;}
306
    static int _fileid(ID id) {return (id >> 32) & 0x00fffffful;}
307
    static int _pos(ID id)    {return id & 0xffffffffUL;}
308
    static ID _mkID(bool computed, bool field, int type, int fileid, int pos) {
309
        assert((type>>7)==0 && (fileid>>24)==0 && (pos>>31)<=1);
310
        return ((computed ? (ID)1 << 63 : (ID)0) | (field ? (ID)1 << 62 : (ID)0) | (ID)type << 56) | ((ID)fileid << 32) | (ID)pos;
311
    }
312

    
313
    // utility functions called while loading a result file
314
    ResultFile *addFile(const char *fileName, const char *fileSystemFileName, bool computed);
315
    Run *addRun();
316
    FileRun *addFileRun(ResultFile *file, Run *run);  // associates a ResultFile with a Run
317

    
318
    void processLine(char **vec, int numTokens, sParseContext &ctx);
319
    int addScalar(FileRun *fileRunRef, const char *moduleName, const char *scalarName, double value, bool isField);
320
    int addVector(FileRun *fileRunRef, int vectorId, const char *moduleName, const char *vectorName, const char *columns);
321
    int addHistogram(FileRun *fileRunRef, const char *moduleName, const char *histogramName, Statistics stat, const StringMap &attrs);
322

    
323
    ResultFile *getFileForID(ID id) const; // checks for NULL
324
    void loadVectorsFromIndex(const char *filename, ResultFile *fileRef);
325

    
326
    template <class T>
327
    void collectIDs(IDList &result, std::vector<T> ResultFile::* vec, int type, bool includeComputed = false, bool includeFields = true) const;
328

    
329
    // unchecked getters are only for internal use by CmpBase in idlist.cc
330
    const ResultItem& uncheckedGetItem(ID id) const;
331
    const ScalarResult& uncheckedGetScalar(ID id) const;
332
    const VectorResult& uncheckedGetVector(ID id) const;
333
    const HistogramResult& uncheckedGetHistogram(ID id) const;
334

    
335
  public:
336
    ResultFileManager();
337
    ~ResultFileManager();
338

    
339
#ifdef THREADED
340
    ILock& getReadLock() { return lock.readLock(); }
341
    ILock& getWriteLock() { return lock.writeLock(); }
342
    ILock& getReadLock() const { return const_cast<ResultFileManager*>(this)->lock.readLock(); }
343
    ILock& getWriteLock() const { return const_cast<ResultFileManager*>(this)->lock.writeLock(); }
344
#endif
345

    
346

    
347
    // navigation
348
    ResultFileList getFiles() const; // filters out NULLs
349
    const RunList& getRuns() const {return runList;}
350
    RunList getRunsInFile(ResultFile *file) const;
351
    ResultFileList getFilesForRun(Run *run) const;
352

    
353
    const ResultItem& getItem(ID id) const;
354
    const ScalarResult& getScalar(ID id) const;
355
    const VectorResult& getVector(ID id) const;
356
    const HistogramResult& getHistogram(ID id) const;
357
    static int getTypeOf(ID id) {return _type(id);} // SCALAR/VECTOR/HISTOGRAM
358

    
359
    bool isStaleID(ID id) const;
360
    bool hasStaleID(const IDList& ids) const;
361

    
362
    // the following are needed for filter combos
363
    // Note: their return value is allocated with new and callers should delete them
364
    ResultFileList *getUniqueFiles(const IDList& ids) const; //XXX why returns pointer?
365
    RunList *getUniqueRuns(const IDList& ids) const;
366
    FileRunList *getUniqueFileRuns(const IDList& ids) const;
367
    StringSet *getUniqueModuleNames(const IDList& ids) const;
368
    StringSet *getUniqueNames(const IDList& ids) const;
369
    StringSet *getUniqueAttributeNames(const IDList &ids) const;
370
    StringSet *getUniqueRunAttributeNames(const RunList *runList) const;
371
    StringSet *getUniqueModuleParamNames(const RunList *runList) const;
372
    StringSet *getUniqueAttributeValues(const IDList &ids, const char *attrName) const;
373
    StringSet *getUniqueRunAttributeValues(const RunList& runList, const char *attrName) const;
374
    StringSet *getUniqueModuleParamValues(const RunList& runList, const char *paramName) const;
375

    
376
    // getting lists of data items
377
    IDList getAllScalars(bool includeComputed = false, bool includeFields = true) const;
378
    IDList getAllVectors(bool includeComputed = false) const;
379
    IDList getAllHistograms(bool includeComputed = false) const;
380
    IDList getAllItems(bool includeComputed = false, bool includeFields = true) const;
381
    IDList getScalarsInFileRun(FileRun *fileRun) const;
382
    IDList getVectorsInFileRun(FileRun *fileRun) const;
383
    IDList getHistogramsInFileRun(FileRun *fileRun) const;
384

    
385
    /**
386
     * Get a filtered subset of the input set (of scalars or vectors).
387
     * All three filter parameters may be null, or (the textual ones)
388
     * may contain wildcards (*,?).
389
     * Uses full string match (substrings need asterisks (*) both ends).
390
     */
391
    IDList filterIDList(const IDList& idlist,
392
                        const FileRunList *fileRunFilter,
393
                        const char *moduleFilter,
394
                        const char *nameFilter) const;
395

    
396
    IDList filterIDList(const IDList &idlist, const char *pattern) const;
397

    
398
    /**
399
     * Checks that the given pattern is syntactically correct.
400
     * If not, an exception is thrown, with a (more-or-less useful)
401
     * message.
402
     */
403
    static void checkPattern(const char *pattern);
404

    
405
    // computed data
406
    ID getComputedID(ComputationID computationID, ID inputID) const;
407
    ID addComputedVector(int vectorId, const char *name, const char *file, const StringMap &attributes, ComputationID computationID, ID inputID, ComputationNode node);
408

    
409
    /**
410
     * loading files. fileName is the file path in the Eclipse workspace;
411
     * the file is actually read from fileSystemFileName
412
     */
413
    ResultFile *loadFile(const char *fileName, const char *fileSystemFileName=NULL, bool reload=false);
414
    void unloadFile(ResultFile *file);
415

    
416

    
417
    bool isFileLoaded(const char *fileName) const;
418
    ResultFile *getFile(const char *fileName) const;
419
    Run *getRunByName(const char *runName) const;
420
    FileRun *getFileRun(ResultFile *file, Run *run) const;
421
    ID getItemByName(FileRun *fileRun, const char *module, const char *name) const;
422

    
423
    // filtering files and runs
424
    RunList filterRunList(const RunList& runList, const char *runNameFilter,
425
                                                  const StringMap& attrFilter) const;
426
    ResultFileList filterFileList(const ResultFileList& fileList, const char *filePathPattern) const;
427

    
428
    // select FileRuns which are both in file list and run list (both can be NULL, meaning '*')
429
    FileRunList getFileRuns(const ResultFileList *fileList, const RunList *runList) const;
430

    
431
    // utility
432
    //void dump(ResultFile *fileRef, std::ostream& out) const;
433

    
434
    StringVector *getFileAndRunNumberFilterHints(const IDList& idlist) const;
435
    StringVector *getFilePathFilterHints(const ResultFileList &fileList) const;
436
    StringVector *getRunNameFilterHints(const RunList &runList) const;
437
    StringVector *getModuleFilterHints(const IDList& idlist) const;
438
    StringVector *getNameFilterHints(const IDList& idlist)const;
439
    StringVector *getResultItemAttributeFilterHints(const IDList &idlist, const char *attrName) const;
440
    StringVector *getRunAttributeFilterHints(const RunList &runList, const char *attrName) const;
441
    StringVector *getModuleParamFilterHints(const RunList &runList, const char * paramName) const;
442

    
443
    const char *getRunAttribute(ID id, const char *attribute) const;
444
};
445

    
446
inline const ResultItem& ResultFileManager::uncheckedGetItem(ID id) const
447
{
448
    switch (_type(id))
449
    {
450
        case SCALAR: return fileList[_fileid(id)]->scalarResults[_pos(id)];
451
        case VECTOR: return fileList[_fileid(id)]->vectorResults[_pos(id)];
452
        case HISTOGRAM: return fileList[_fileid(id)]->histogramResults[_pos(id)];
453
        default: throw opp_runtime_error("ResultFileManager: invalid ID: wrong type");
454
    }
455
}
456

    
457
inline const ScalarResult& ResultFileManager::uncheckedGetScalar(ID id) const
458
{
459
    return fileList[_fileid(id)]->scalarResults[_pos(id)];
460
}
461

    
462
inline const VectorResult& ResultFileManager::uncheckedGetVector(ID id) const
463
{
464
    return fileList[_fileid(id)]->vectorResults[_pos(id)];
465
}
466

    
467
inline const HistogramResult& ResultFileManager::uncheckedGetHistogram(ID id) const
468
{
469
    return fileList[_fileid(id)]->histogramResults[_pos(id)];
470
}
471

    
472
inline ResultFile *ResultFileManager::getFileForID(ID id) const
473
{
474
    ResultFile *fileRef = fileList.at(_fileid(id));
475
    if (fileRef==NULL)
476
        throw opp_runtime_error("ResultFileManager: stale ID: its file has already been unloaded");
477
    return fileRef;
478
}
479

    
480
inline bool ResultFileManager::isStaleID(ID id) const
481
{
482
    return fileList.at(_fileid(id)) == NULL;
483
}
484

    
485
NAMESPACE_END
486

    
487

    
488
#endif
489

    
490