Project

General

Profile

Statistics
| Branch: | Revision:

root / src / scave / resultfilemanager.cc @ a3be1d55

History | View | Annotate | Download (49.5 KB)

1
//=========================================================================
2
//  RESULTFILEMANAGER.CC - part of
3
//                  OMNeT++/OMNEST
4
//           Discrete System Simulation in C++
5
//
6
//  Author: Andras Varga, Tamas Borbely
7
//
8
//=========================================================================
9

    
10
/*--------------------------------------------------------------*
11
  Copyright (C) 1992-2008 Andras Varga
12
  Copyright (C) 2006-2008 OpenSim Ltd.
13

14
  This file is distributed WITHOUT ANY WARRANTY. See the file
15
  `license' for details on this and other legal matters.
16
*--------------------------------------------------------------*/
17

    
18
#include <stdlib.h>
19
#include <fstream>
20
#include <sstream>
21
#include <iostream>
22
#include <memory>
23
#include <algorithm>
24
#include <utility>
25
#include <functional>
26
#include "opp_ctype.h"
27
#include "platmisc.h"
28
#include "matchexpression.h"
29
#include "patternmatcher.h"
30
#include "filereader.h"
31
#include "linetokenizer.h"
32
#include "stringtokenizer.h"
33
#include "filereader.h"
34
#include "indexfile.h"
35
#include "scaveutils.h"
36
#include "scaveexception.h"
37
#include "resultfilemanager.h"
38
#include "fileutil.h"
39
#include "commonutil.h"
40
#include "stringutil.h"
41

    
42
#ifdef THREADED
43
#define READER_MUTEX Mutex __reader_mutex_(getReadLock());
44
#define WRITER_MUTEX Mutex __writer_mutex_(getWriteLock());
45
#else
46
#define READER_MUTEX
47
#define WRITER_MUTEX
48
#endif
49

    
50
USING_NAMESPACE
51

    
52
ResultItem::Type ResultItem::getType() const
53
{
54
    StringMap::const_iterator it = attributes.find("type");
55
    if (it == attributes.end())
56
    {
57
        if (attributes.find("enum") != attributes.end())
58
            return TYPE_ENUM;
59
        else
60
            return TYPE_DOUBLE;
61
    }
62
    else
63
    {
64
        const std::string &type = it->second;
65
        if (type == "int")
66
            return TYPE_INT;
67
        else if (type == "double")
68
            return TYPE_DOUBLE;
69
        else if (type == "enum")
70
            return TYPE_ENUM;
71
        else
72
            throw opp_runtime_error("Unknown type: %s", type.c_str());
73
    }
74
}
75

    
76
EnumType* ResultItem::getEnum() const
77
{
78
    StringMap::const_iterator it = attributes.find("enum");
79
    if (it != attributes.end())
80
    {
81
        EnumType *enumPtr = new EnumType();
82
        enumPtr->parseFromString(it->second.c_str());
83
        return enumPtr;
84
    }
85
    else
86
    {
87
        return NULL;
88
    }
89
}
90

    
91
InterpolationMode VectorResult::getInterpolationMode() const
92
{
93
    StringMap::const_iterator it = attributes.find("interpolationmode");
94
    if (it != attributes.end())
95
    {
96
        const std::string &mode = it->second;
97
        if (mode == "none")
98
            return NONE;
99
        else if (mode == "sample-hold")
100
            return SAMPLE_HOLD;
101
        else if (mode == "backward-sample-hold")
102
            return BACKWARD_SAMPLE_HOLD;
103
        else if (mode == "linear")
104
            return LINEAR;
105
        else
106
            throw opp_runtime_error("Unknown interpolation mode: %s", mode.c_str());
107
    }
108
    else
109
    {
110
        return UNSPECIFIED;
111
    }
112
}
113

    
114
void HistogramResult::addBin(double lower_bound, double value)
115
{
116
    if (!bins.empty() && bins.back() >= lower_bound)
117
        throw opp_runtime_error("histogram bin bounds must be specified in increasing order");
118
    bins.push_back(lower_bound);
119
    values.push_back(value);
120
}
121

    
122
ResultFileManager::ResultFileManager()
123
{
124
}
125

    
126
ResultFileManager::~ResultFileManager()
127
{
128
    for (int i=0; i<(int)fileRunList.size(); i++)
129
        delete fileRunList[i];
130

    
131
    for (int i=0; i<(int)runList.size(); i++)
132
        delete runList[i];
133

    
134
    for (int i=0; i<(int)fileList.size(); i++)
135
        delete fileList[i];
136

    
137
    fileRunList.clear();
138
    runList.clear();
139
    fileList.clear();
140

    
141
    moduleNames.clear();
142
    names.clear();
143
    classNames.clear();
144
}
145

    
146
ResultFileList ResultFileManager::getFiles() const
147
{
148
    READER_MUTEX
149
    ResultFileList out;
150
    for (int i=0; i<(int)fileList.size(); i++)
151
        if (fileList[i] && !fileList[i]->computed)
152
            out.push_back(fileList[i]);
153
    return out;
154
}
155

    
156
RunList ResultFileManager::getRunsInFile(ResultFile *file) const
157
{
158
    READER_MUTEX
159
    RunList out;
160
    for (int i=0; i<(int)fileRunList.size(); i++)
161
        if (fileRunList[i]->fileRef == file)
162
            out.push_back(fileRunList[i]->runRef);
163
    return out;
164
}
165

    
166
ResultFileList ResultFileManager::getFilesForRun(Run *run) const
167
{
168
    READER_MUTEX
169
    ResultFileList out;
170
    for (int i=0; i<(int)fileRunList.size(); i++)
171
        if (fileRunList[i]->runRef == run && !fileRunList[i]->fileRef->computed)
172
            out.push_back(fileRunList[i]->fileRef);
173
    return out;
174
}
175

    
176
const ResultItem& ResultFileManager::getItem(ID id) const
177
{
178
    READER_MUTEX
179
    try
180
    {
181
        switch (_type(id))
182
        {
183
            case SCALAR: return getFileForID(id)->scalarResults.at(_pos(id));
184
            case VECTOR: return getFileForID(id)->vectorResults.at(_pos(id));
185
            case HISTOGRAM: return getFileForID(id)->histogramResults.at(_pos(id));
186
            default: throw opp_runtime_error("ResultFileManager: invalid ID: wrong type");
187
        }
188
    }
189
    catch (std::out_of_range e)
190
    {
191
        throw opp_runtime_error("ResultFileManager::getItem(id): invalid ID");
192
    }
193
}
194

    
195
ResultFileList *ResultFileManager::getUniqueFiles(const IDList& ids) const
196
{
197
    READER_MUTEX
198
    // collect unique runs in this dataset
199
    std::set<ResultFile*> set;
200
    for (int i=0; i<ids.size(); i++)
201
        set.insert(getItem(ids.get(i)).fileRunRef->fileRef);
202

    
203
    // convert to list for easier handling at recipient
204
    return new ResultFileList(set.begin(), set.end());
205
}
206

    
207
RunList *ResultFileManager::getUniqueRuns(const IDList& ids) const
208
{
209
    READER_MUTEX
210
    // collect unique runs in this dataset
211
    std::set<Run*> set;
212
    for (int i=0; i<ids.size(); i++)
213
        set.insert(getItem(ids.get(i)).fileRunRef->runRef);
214

    
215
    // convert to list for easier handling at recipient
216
    return new RunList(set.begin(), set.end());
217
}
218

    
219
FileRunList *ResultFileManager::getUniqueFileRuns(const IDList& ids) const
220
{
221
    READER_MUTEX
222
    // collect unique FileRuns in this dataset
223
    std::set<FileRun*> set;
224
    for (int i=0; i<ids.size(); i++)
225
        set.insert(getItem(ids.get(i)).fileRunRef);
226

    
227
    // convert to list for easier handling at recipient
228
    return new FileRunList(set.begin(), set.end());
229
}
230

    
231
StringSet *ResultFileManager::getUniqueModuleNames(const IDList& ids) const
232
{
233
    READER_MUTEX
234
    // collect unique module names in this dataset
235
    StringSet *set = new StringSet();
236
    for (int i=0; i<ids.size(); i++)
237
        set->insert(*getItem(ids.get(i)).moduleNameRef);
238
    return set;
239
}
240

    
241
StringSet *ResultFileManager::getUniqueNames(const IDList& ids) const
242
{
243
    READER_MUTEX
244
    // collect unique scalar/vector names in this dataset
245
    StringSet *set = new StringSet();
246
    for (int i=0; i<ids.size(); i++)
247
        set->insert(*getItem(ids.get(i)).nameRef);
248
    return set;
249
}
250

    
251
StringSet *ResultFileManager::getUniqueAttributeNames(const IDList &ids) const
252
{
253
    READER_MUTEX
254
    StringSet *set = new StringSet;
255
    for (int i=0; i<ids.size(); i++)
256
    {
257
        const StringMap &attributes = getItem(ids.get(i)).attributes;
258
        for(StringMap::const_iterator it = attributes.begin(); it != attributes.end(); ++it)
259
            set->insert(it->first);
260
    }
261
    return set;
262
}
263

    
264
StringSet *ResultFileManager::getUniqueRunAttributeNames(const RunList *runList) const
265
{
266
    READER_MUTEX
267
    StringSet *set = new StringSet;
268
    for (RunList::const_iterator runRef = runList->begin(); runRef != runList->end(); ++runRef)
269
    {
270
        const StringMap &attributes = (*runRef)->attributes;
271
        for(StringMap::const_iterator it = attributes.begin(); it != attributes.end(); ++it)
272
            set->insert(it->first);
273
    }
274
    return set;
275
}
276

    
277
StringSet *ResultFileManager::getUniqueModuleParamNames(const RunList *runList) const
278
{
279
    READER_MUTEX
280
    StringSet *set = new StringSet;
281
    for (RunList::const_iterator runRef = runList->begin(); runRef != runList->end(); ++runRef)
282
    {
283
        const StringMap &params = (*runRef)->moduleParams;
284
        for(StringMap::const_iterator it = params.begin(); it != params.end(); ++it)
285
            set->insert(it->first);
286
    }
287
    return set;
288
}
289

    
290
StringSet *ResultFileManager::getUniqueAttributeValues(const IDList &ids, const char *attrName) const
291
{
292
    READER_MUTEX
293
    StringSet *values = new StringSet;
294
    for (int i = 0; i < ids.size(); ++i)
295
    {
296
        const char *value = getItem(ids.get(i)).getAttribute(attrName);
297
        if (value != NULL)
298
            values->insert(value);
299
    }
300
    return values;
301
}
302

    
303
StringSet *ResultFileManager::getUniqueRunAttributeValues(const RunList& runList, const char *attrName) const
304
{
305
    READER_MUTEX
306
    StringSet *values = new StringSet;
307
    for (RunList::const_iterator runRef = runList.begin(); runRef != runList.end(); ++runRef)
308
    {
309
        const char *value = (*runRef)->getAttribute(attrName);
310
        if (value!=NULL)
311
            values->insert(value);
312
    }
313

    
314
    return values;
315
}
316

    
317
StringSet *ResultFileManager::getUniqueModuleParamValues(const RunList& runList, const char *paramName) const
318
{
319
    READER_MUTEX
320
    StringSet *values = new StringSet;
321
    for (RunList::const_iterator runRef = runList.begin(); runRef != runList.end(); ++runRef)
322
    {
323
        const char *value = (*runRef)->getModuleParam(paramName);
324
        if (value != NULL)
325
            values->insert(value);
326
    }
327

    
328
    return values;
329
}
330

    
331
const ScalarResult& ResultFileManager::getScalar(ID id) const
332
{
333
    READER_MUTEX
334
    if (_type(id)!=SCALAR)
335
        throw opp_runtime_error("ResultFileManager::getScalar(id): this item is not a scalar");
336
    return getFileForID(id)->scalarResults.at(_pos(id));
337
}
338

    
339
const VectorResult& ResultFileManager::getVector(ID id) const
340
{
341
    READER_MUTEX
342
    if (_type(id)!=VECTOR)
343
        throw opp_runtime_error("ResultFileManager::getVector(id): this item is not a vector");
344
    return getFileForID(id)->vectorResults.at(_pos(id));
345
}
346

    
347
const HistogramResult& ResultFileManager::getHistogram(ID id) const
348
{
349
    READER_MUTEX
350
    if (_type(id)!=HISTOGRAM)
351
        throw opp_runtime_error("ResultFileManager::getHistogram(id): this item is not a histogram");
352
    return getFileForID(id)->histogramResults.at(_pos(id));
353
}
354

    
355
template <class T>
356
void ResultFileManager::collectIDs(IDList &out, std::vector<T> ResultFile::* vec, int type, bool includeComputed, bool includeFields) const
357
{
358
    for (int k=0; k<(int)fileList.size(); k++)
359
    {
360
        if (fileList[k]!=NULL)
361
        {
362
            std::vector<T>& v = fileList[k]->*vec;
363
            for (int i=0; i<(int)v.size(); i++) {
364
                bool isComputed = v[i].isComputed();
365
                bool isField = type == SCALAR ? ((ScalarResult&)v[i]).isField : false;
366

    
367
                if ((!isField || includeFields) && (!isComputed || includeComputed))
368
                    out.uncheckedAdd(_mkID(false, isField, type, k, i));
369
            }
370
        }
371
    }
372
}
373

    
374
IDList ResultFileManager::getAllItems(bool includeComputed, bool includeFields) const
375
{
376
    READER_MUTEX
377
    IDList out;
378
    collectIDs(out, &ResultFile::scalarResults, SCALAR, includeComputed, includeFields);
379
    collectIDs(out, &ResultFile::vectorResults, VECTOR, includeComputed, includeFields);
380
    collectIDs(out, &ResultFile::histogramResults, HISTOGRAM, includeComputed, includeFields);
381
    return out;
382
}
383

    
384
IDList ResultFileManager::getAllScalars(bool includeComputed, bool includeFields) const
385
{
386
    READER_MUTEX
387
    IDList out;
388
    collectIDs(out, &ResultFile::scalarResults, SCALAR, includeComputed, includeFields);
389
    return out;
390
}
391

    
392
IDList ResultFileManager::getAllVectors(bool includeComputed) const
393
{
394
    READER_MUTEX
395
    IDList out;
396
    collectIDs(out, &ResultFile::vectorResults, VECTOR, includeComputed);
397
    return out;
398
}
399

    
400
IDList ResultFileManager::getAllHistograms(bool includeComputed) const
401
{
402
    READER_MUTEX
403
    IDList out;
404
    collectIDs(out, &ResultFile::histogramResults, HISTOGRAM, includeComputed);
405
    return out;
406
}
407

    
408
IDList ResultFileManager::getScalarsInFileRun(FileRun *fileRun) const
409
{
410
    READER_MUTEX
411
    IDList out;
412
    int fileId = fileRun->fileRef->id;
413
    ScalarResults& v = fileRun->fileRef->scalarResults;
414
    for (int i=0; i<(int)v.size(); i++)
415
        if (v[i].fileRunRef==fileRun && !v[i].isComputed())
416
            out.uncheckedAdd(_mkID(false,v[i].isField,SCALAR,fileId,i));
417
    return out;
418
}
419

    
420
IDList ResultFileManager::getVectorsInFileRun(FileRun *fileRun) const
421
{
422
    READER_MUTEX
423
    IDList out;
424
    int fileId = fileRun->fileRef->id;
425
    VectorResults& v = fileRun->fileRef->vectorResults;
426
    for (int i=0; i<(int)v.size(); i++)
427
        if (v[i].fileRunRef==fileRun && !v[i].isComputed())
428
            out.uncheckedAdd(_mkID(false,false,VECTOR,fileId,i));
429
    return out;
430
}
431

    
432
IDList ResultFileManager::getHistogramsInFileRun(FileRun *fileRun) const
433
{
434
    READER_MUTEX
435
    IDList out;
436
    int fileId = fileRun->fileRef->id;
437
    HistogramResults& v = fileRun->fileRef->histogramResults;
438
    for (int i=0; i<(int)v.size(); i++)
439
        if (v[i].fileRunRef==fileRun && !v[i].isComputed())
440
            out.uncheckedAdd(_mkID(false,false,HISTOGRAM,fileId,i));
441
    return out;
442
}
443

    
444
bool ResultFileManager::isFileLoaded(const char *fileName) const
445
{
446
    return getFile(fileName)!=NULL;
447
}
448

    
449
ResultFile *ResultFileManager::getFile(const char *fileName) const
450
{
451
    if (!fileName)
452
        return NULL;
453

    
454
    READER_MUTEX
455
    for (int i=0; i<(int)fileList.size(); i++)
456
        if (fileList[i]!=NULL && fileList[i]->filePath==fileName)
457
            return fileList[i];
458
    return NULL;
459
}
460

    
461
Run *ResultFileManager::getRunByName(const char *runName) const
462
{
463
    if (!runName)
464
        return NULL;
465

    
466
    READER_MUTEX
467
    for (int i=0; i<(int)runList.size(); i++)
468
        if (runList[i]->runName==runName)
469
            return runList[i];
470
    return NULL;
471
}
472

    
473
FileRun *ResultFileManager::getFileRun(ResultFile *file, Run *run) const
474
{
475
    READER_MUTEX
476
    for (int i=0; i<(int)fileRunList.size(); i++)
477
        if (fileRunList[i]->fileRef==file && fileRunList[i]->runRef==run)
478
            return fileRunList[i];
479
    return NULL;
480
}
481

    
482
// currently unused
483
ID ResultFileManager::getItemByName(FileRun *fileRunRef, const char *module, const char *name) const
484
{
485
    if (!fileRunRef || !module || !name)
486
        return 0;
487

    
488
    READER_MUTEX
489

    
490
    const std::string *moduleNameRef = moduleNames.find(module);
491
    if (!moduleNameRef)
492
        return 0;
493

    
494
    const std::string *nameRef = names.find(name);
495
    if (!nameRef)
496
        return 0;
497

    
498
    ScalarResults& scalarResults = fileRunRef->fileRef->scalarResults;
499
    for (int i=0; i<(int)scalarResults.size(); i++)
500
    {
501
        const ResultItem& d = scalarResults[i];
502
        if (d.moduleNameRef==moduleNameRef && d.nameRef==nameRef && d.fileRunRef==fileRunRef)
503
            return _mkID(d.isComputed(), scalarResults[i].isField, SCALAR, fileRunRef->fileRef->id, i);
504
    }
505

    
506
    VectorResults& vectorResults = fileRunRef->fileRef->vectorResults;
507
    for (int i=0; i<(int)vectorResults.size(); i++)
508
    {
509
        const ResultItem& d = vectorResults[i];
510
        if (d.moduleNameRef==moduleNameRef && d.nameRef==nameRef && d.fileRunRef==fileRunRef)
511
            return _mkID(d.isComputed(), false, VECTOR, fileRunRef->fileRef->id, i);
512
    }
513

    
514
    HistogramResults& histogramResults = fileRunRef->fileRef->histogramResults;
515
    for (int i=0; i<(int)histogramResults.size(); i++)
516
    {
517
        const ResultItem& d = histogramResults[i];
518
        if (d.moduleNameRef==moduleNameRef && d.nameRef==nameRef && d.fileRunRef==fileRunRef)
519
            return _mkID(d.isComputed(), false, HISTOGRAM, fileRunRef->fileRef->id, i);
520
    }
521
    return 0;
522
}
523

    
524
RunList ResultFileManager::filterRunList(const RunList& runList,
525
                                         const char *runNameFilter,
526
                                         const StringMap& attrFilter) const
527
{
528
    READER_MUTEX
529

    
530
    RunList out;
531

    
532
    // runName matcher
533
    PatternMatcher runNamePattern(runNameFilter, false, true, true);
534

    
535
    // copy attributes to vectors for performance (std::map iterator is
536
    // infamously slow, and we need PatternMatchers as well)
537
    StringVector attrNames;
538
    std::vector<PatternMatcher> attrValues;
539
    for (StringMap::const_iterator it = attrFilter.begin(); it!=attrFilter.end(); ++it)
540
    {
541
        attrNames.push_back(it->first);
542
        attrValues.push_back(PatternMatcher(it->second.c_str(), false, true, true));
543
    }
544

    
545
    // do it
546
    for (int i=0; i<(int)runList.size(); i++)
547
    {
548
        Run *run = runList[i];
549
        if (!runNamePattern.matches(run->runName.c_str()))
550
            continue;
551
        bool matches = true;
552
        for (int j=0; j<(int)attrNames.size() && matches; j++)
553
        {
554
            const char *attrValue = run->getAttribute(attrNames[j].c_str());
555
            if (attrValue == NULL)
556
                attrValue = "";
557

    
558
            if (!attrValues[j].matches(attrValue))
559
            {
560
                matches = false;
561
                break;
562
            }
563
        }
564
        if (matches)
565
            out.push_back(run);
566
    }
567
    return out;
568
}
569

    
570
ResultFileList ResultFileManager::filterFileList(const ResultFileList& fileList,
571
                                                 const char *filePathFilter) const
572
{
573
    ResultFileList out;
574

    
575
    // filePath matcher
576
    PatternMatcher filePathPattern(filePathFilter, false, true, true);
577

    
578
    READER_MUTEX
579
    for (int i=0; i<(int)fileList.size(); i++)
580
    {
581
        ResultFile *file = fileList[i];
582
        if (!filePathPattern.matches(file->filePath.c_str()))
583
            continue;
584
        out.push_back(file);
585
    }
586
    return out;
587
}
588

    
589
FileRunList ResultFileManager::getFileRuns(const ResultFileList *fileList, const RunList *runList) const
590
{
591
    READER_MUTEX
592
    FileRunList out;
593
    for (int i=0; i<(int)fileRunList.size(); i++)
594
    {
595
        FileRun *fileRun = fileRunList[i];
596
        if (fileList && std::find(fileList->begin(), fileList->end(), fileRun->fileRef)==fileList->end())
597
            continue;
598
        if (runList && std::find(runList->begin(), runList->end(), fileRun->runRef)==runList->end())
599
            continue;
600
        out.push_back(fileRun);
601
    }
602
    return out;
603
}
604

    
605
IDList ResultFileManager::filterIDList(const IDList& idlist,
606
                                       const FileRunList *fileRunFilter,
607
                                       const char *moduleFilter,
608
                                       const char *nameFilter) const
609
{
610
    READER_MUTEX
611

    
612
    // "*" means no filtering, so ignore it
613
    if (!strcmp(moduleFilter,"*")) moduleFilter = "";
614
    if (!strcmp(nameFilter,"*")) nameFilter = "";
615

    
616
    // prepare for wildcard matches
617
    bool patMatchModule = PatternMatcher::containsWildcards(moduleFilter);
618
    bool patMatchName = PatternMatcher::containsWildcards(nameFilter);
619

    
620
    PatternMatcher *modulePattern = NULL;
621
    if (patMatchModule)
622
    {
623
        modulePattern = new PatternMatcher(moduleFilter, false, true, true); // case-sensitive full-string match
624
    }
625
    PatternMatcher *namePattern = NULL;
626
    if (patMatchName)
627
    {
628
        namePattern = new PatternMatcher(nameFilter, false, true, true); // case-sensitive full-string match
629
    }
630

    
631
    // TODO: if there's no wildcard, find string pointers in the stringsets
632
    // in advance, then we don't have to do strcmp().
633

    
634
    // iterate over all values and add matching ones to "out".
635
    // we can exploit the fact that ResultFileManager contains the data in the order
636
    // they were read from file, i.e. grouped by runs
637
    IDList out;
638
    FileRun *lastFileRunRef = NULL;
639
    bool lastFileRunMatched = false;
640
    int sz = idlist.size();
641
    for (int i=0; i<sz; i++)
642
    {
643
        ID id = idlist.get(i);
644
        const ResultItem& d = getItem(id);
645

    
646
        if (fileRunFilter)
647
        {
648
            if (lastFileRunRef!=d.fileRunRef)
649
            {
650
                lastFileRunRef = d.fileRunRef;
651
                lastFileRunMatched = std::find(fileRunFilter->begin(), fileRunFilter->end(), d.fileRunRef) != fileRunFilter->end();
652
            }
653
            if (!lastFileRunMatched)
654
                continue;
655
        }
656

    
657
        if (moduleFilter && moduleFilter[0] &&
658
            (patMatchModule ? !modulePattern->matches(d.moduleNameRef->c_str())
659
                            : strcmp(d.moduleNameRef->c_str(), moduleFilter))
660
           )
661
            continue; // no match
662

    
663
        if (nameFilter && nameFilter[0] &&
664
            (patMatchName ? !namePattern->matches(d.nameRef->c_str())
665
                          : strcmp(d.nameRef->c_str(), nameFilter))
666
           )
667
            continue; // no match
668

    
669
        // everything matched, insert it.
670
        // (note: uncheckedAdd() is fine: if input IDList didn't contain duplicates,
671
        // the result won't either)
672
        out.uncheckedAdd(id);
673
    }
674
    return out;
675
}
676

    
677
class MatchableResultItem : public MatchExpression::Matchable
678
{
679
    const ResultItem& item;
680

    
681
    public:
682
        MatchableResultItem(const ResultItem& item) : item(item) {}
683
        virtual const char *getDefaultAttribute() const;
684
        virtual const char *getAttribute(const char *name) const;
685
    private:
686
        const char *getName() const { return item.nameRef->c_str(); }
687
        const char *getModuleName() const { return item.moduleNameRef->c_str(); }
688
        const char *getFileName() const { return item.fileRunRef->fileRef->filePath.c_str(); }
689
        const char *getRunName() const { return item.fileRunRef->runRef->runName.c_str(); }
690
        const char *getResultItemAttribute(const char *attrName) const { return item.getAttribute(attrName); }
691
        const char *getRunAttribute(const char *attrName) const { return item.fileRunRef->runRef->getAttribute(attrName); }
692
        const char *getModuleParam(const char *paramName) const { return item.fileRunRef->runRef->getModuleParam(paramName); }
693
};
694

    
695
const char *MatchableResultItem::getDefaultAttribute() const
696
{
697
    return getName();
698
}
699

    
700
const char *MatchableResultItem::getAttribute(const char *name) const
701
{
702
    if (strcasecmp("name", name) == 0)
703
        return getName();
704
    else if (strcasecmp("module", name) == 0)
705
        return getModuleName();
706
    else if (strcasecmp("file", name) == 0)
707
        return getFileName();
708
    else if (strcasecmp("run", name) == 0)
709
        return getRunName();
710
    else if (strncasecmp("attr:", name, 5) == 0)
711
        return getRunAttribute(name+5);
712
    else if (strncasecmp("param:", name, 6) == 0)
713
        return getModuleParam(name+6);
714
    else
715
        return getResultItemAttribute(name);
716
}
717

    
718
IDList ResultFileManager::filterIDList(const IDList &idlist, const char *pattern) const
719
{
720
    if (pattern == NULL || pattern[0] == '\0') // no filter
721
        pattern = "*";
722

    
723
    MatchExpression matchExpr(pattern, false /*dottedpath*/, true /*fullstring*/, true /*casesensitive*/);
724

    
725
    READER_MUTEX
726
    IDList out;
727
    int sz = idlist.size();
728
    for (int i=0; i<sz; ++i)
729
    {
730
        ID id = idlist.get(i);
731
        const ResultItem &item = getItem(id);
732
        MatchableResultItem matchable(item);
733
        if (matchExpr.matches(&matchable))
734
            out.uncheckedAdd(id);
735
    }
736
    return out;
737
}
738

    
739
void ResultFileManager::checkPattern(const char *pattern)
740
{
741
    if (pattern==NULL || pattern[0] == '\0') // no filter
742
        return;
743

    
744
    // parse it
745
    //XXX after successful parsing, we could also check that attribute names in it are valid
746
    MatchExpression matchExpr(pattern, false /*dottedpath*/, true /*fullstring*/, true /*casesensitive*/);
747
}
748

    
749
ResultFile *ResultFileManager::addFile(const char *fileName, const char *fileSystemFileName, bool computed)
750
{
751
    ResultFile *file = new ResultFile();
752
    file->id = fileList.size();
753
    fileList.push_back(file);
754
    file->resultFileManager = this;
755
    file->fileSystemFilePath = fileSystemFileName;
756
    file->filePath = fileNameToSlash(fileName);
757
    splitFileName(file->filePath.c_str(), file->directory, file->fileName);
758
    file->computed = computed;
759
    file->numLines = 0;
760
    file->numUnrecognizedLines = 0;
761
    return file;
762
}
763

    
764
Run *ResultFileManager::addRun()
765
{
766
    Run *run = new Run();
767
    run->resultFileManager = this;
768
    run->runNumber = 0;
769
    runList.push_back(run);
770
    return run;
771
}
772

    
773
FileRun *ResultFileManager::addFileRun(ResultFile *file, Run *run)
774
{
775
    FileRun *fileRun = new FileRun();
776
    fileRunList.push_back(fileRun);
777
    fileRun->fileRef = file;
778
    fileRun->runRef = run;
779
    return fileRun;
780
}
781

    
782
int ResultFileManager::addScalar(FileRun *fileRunRef, const char *moduleName,
783
                                  const char *scalarName, double value, bool isField)
784
{
785
    ScalarResult scalar;
786
    scalar.fileRunRef = fileRunRef;
787

    
788
    scalar.moduleNameRef = moduleNames.insert(moduleName);
789
    scalar.nameRef = names.insert(scalarName);
790
    scalar.value = value;
791
    scalar.isField = isField;
792

    
793
    ScalarResults &scalars = fileRunRef->fileRef->scalarResults;
794
    scalars.push_back(scalar);
795
    return scalars.size() - 1;
796
}
797

    
798
int ResultFileManager::addVector(FileRun *fileRunRef, int vectorId, const char *moduleName, const char *vectorName, const char *columns)
799
{
800
    VectorResult vector;
801
    vector.fileRunRef = fileRunRef;
802
    vector.vectorId = vectorId;
803
    vector.moduleNameRef = moduleNames.insert(moduleName);
804
    vector.nameRef = names.insert(vectorName);
805
    vector.columns = columns;
806
    vector.stat = Statistics(-1, NaN, NaN, NaN, NaN);
807
    VectorResults &vectors = fileRunRef->fileRef->vectorResults;
808
    vectors.push_back(vector);
809
    return vectors.size() - 1;
810
}
811

    
812
int ResultFileManager::addHistogram(FileRun *fileRunRef, const char *moduleName, const char *histogramName,
813
        Statistics stat, const StringMap &attrs)
814
{
815
    HistogramResult histogram;
816
    histogram.attributes = attrs;
817
    histogram.fileRunRef = fileRunRef;
818
    histogram.moduleNameRef = moduleNames.insert(moduleName);
819
    histogram.nameRef = names.insert(histogramName);
820
    histogram.stat = stat;
821
    HistogramResults &histograms = fileRunRef->fileRef->histogramResults;
822
    histograms.push_back(histogram);
823
    return histograms.size() - 1;
824
}
825

    
826

    
827
// create a file for each dataset?
828
ID ResultFileManager::addComputedVector(int vectorId, const char *name, const char *file,
829
        const StringMap &attributes, ComputationID computationID, ID input, ComputationNode computation)
830
{
831
    WRITER_MUTEX
832

    
833
    assert(getTypeOf(input) == VECTOR);
834

    
835
    const VectorResult& vector = getVector(input);
836

    
837
    ResultFile *fileRef = getFile(file);
838
    if (!fileRef)
839
        fileRef = addFile(file, file, true); // XXX
840
    Run *runRef = vector.fileRunRef->runRef;
841
    FileRun *fileRunRef = getFileRun(fileRef, runRef);
842
    if (!fileRunRef)
843
        fileRunRef = addFileRun(fileRef, runRef);
844

    
845
    VectorResult newVector = VectorResult();
846
    newVector.vectorId = vectorId;
847
    newVector.computation = computation;
848
    newVector.columns = vector.columns;
849
    newVector.moduleNameRef = vector.moduleNameRef;
850
    newVector.nameRef = names.insert(name);
851
    newVector.fileRunRef = fileRunRef;
852
    newVector.attributes = attributes;
853
    newVector.stat = Statistics(-1, NaN, NaN, NaN, NaN);
854
    fileRef->vectorResults.push_back(newVector);
855
    ID id = _mkID(true, false, VECTOR, fileRef->id, fileRef->vectorResults.size()-1);
856
    std::pair<ComputationID, ID> key = std::make_pair(computationID, input);
857
    computedIDCache[key] = id;
858
    return id;
859
}
860

    
861
ID ResultFileManager::getComputedID(ComputationID computationID, ID input) const
862
{
863
    READER_MUTEX
864
    std::pair<ComputationID, ID> key = std::make_pair(computationID, input);
865
    ComputedIDCache::const_iterator it = computedIDCache.find(key);
866
    if (it != computedIDCache.end())
867
      return it->second;
868
    else
869
        return -1;
870
}
871

    
872
/*
873
void ResultFileManager::dump(ResultFile *fileRef, std::ostream& out) const
874
{
875
    Run *prevRunRef = NULL;
876
    for (ScalarResults::const_iterator i = scalars.begin(); i!=scalars.end(); i++)
877
    {
878
        const ScalarResult& d = *i;
879
        if (d.runRef->fileRef==fileRef)
880
        {
881
            if (d.runRef!=prevRunRef)
882
            {
883
                out << "run " << d.runRef->runNumber << " " << d.runRef->networkName
884
                    << " \"" << d.runRef->date << "\"\n";
885
                prevRunRef = d.runRef;
886
            }
887
            out << "scalar \"" << *d.moduleNameRef << "\"\t\""
888
                << *d.nameRef << "\"\t" << d.value << "\n";   //FIXME use opp_quotestr() here
889
        }
890
    }
891
}
892
*/
893

    
894
#ifdef CHECK
895
#undef CHECK
896
#endif
897
#define CHECK(cond,msg) if (!(cond)) throw ResultFileFormatException(msg, ctx.fileName, ctx.lineNo);
898

    
899

    
900
void ResultFileManager::processLine(char **vec, int numTokens, sParseContext &ctx)
901
{
902
    ++ctx.lineNo;
903

    
904
    // ignore empty lines
905
    if (numTokens==0 || vec[0][0]=='#')
906
        return;
907

    
908
    // process "run" lines
909
    if (vec[0][0]=='r' && !strcmp(vec[0],"run"))
910
    {
911
        CHECK(numTokens==2, "incorrect 'run' line -- run <runID> expected");
912

    
913
        // "run" line, format: run <runName>
914
        // find out if we have that run already
915
        Run *runRef = getRunByName(vec[1]);
916
        if (!runRef)
917
        {
918
            // not yet: add it
919
            runRef = addRun();
920
            runRef->runName = vec[1];
921
        }
922
        // associate Run with this file
923
        CHECK(getFileRun(ctx.fileRef, runRef)==NULL, "non-unique runId in the file");
924
        ctx.fileRunRef = addFileRun(ctx.fileRef, runRef);
925

    
926
        ctx.lastResultItemType = 0;
927
        ctx.lastResultItemIndex = -1;
928
        ctx.clearHistogram();
929
        return;
930
    }
931
    else if (vec[0][0] == 'v' && strcmp(vec[0], "version") == 0)
932
    {
933
        int version;
934
        CHECK(numTokens==2, "incorrect 'version' line -- version <number> expected");
935
        CHECK(parseInt(vec[1], version), "version is not a number");
936
        CHECK(version==2, "expects version 2");
937
        return;
938
    }
939

    
940

    
941
    // if we haven't seen a "run" line yet (as with old vector files), add a default run
942
    if (ctx.fileRunRef==NULL)
943
    {
944
        // fake a new Run
945
        Run *runRef = addRun();
946
        ctx.fileRunRef = addFileRun(ctx.fileRef, runRef);
947
        runRef->runNumber = 0;
948

    
949
        /*
950
        // make up a unique runName
951
        std::stringstream os;
952
        static int counter=1000;
953
        os << fileRef->fileName << ":" << lineNum << "-#n/a-" <<++counter;
954
        runRef->runName = os.str();
955
        */
956
    }
957

    
958
    if (vec[0][0]=='s' && !strcmp(vec[0],"scalar"))
959
    {
960
        // syntax: "scalar <module> <scalarname> <value>"
961
        CHECK(numTokens==4, "incorrect 'scalar' line -- scalar <module> <scalarname> <value> expected");
962

    
963
        double value;
964
        CHECK(parseDouble(vec[3],value), "invalid value column");
965

    
966
        ctx.lastResultItemType = SCALAR;
967
        ctx.lastResultItemIndex = addScalar(ctx.fileRunRef, vec[1], vec[2], value, false);
968
        ctx.clearHistogram();
969
    }
970
    else if (vec[0][0]=='v' && !strcmp(vec[0],"vector"))
971
    {
972
        // syntax: "vector <id> <module> <vectorname> [<columns>]"
973
        CHECK(numTokens==4 || numTokens==5, "incorrect 'vector' line -- vector <id> <module> <vectorname> [<columns>] expected");
974
        int vectorId;
975
        CHECK(parseInt(vec[1], vectorId), "invalid vector id in vector definition");
976
        const char *columns = (numTokens < 5 || opp_isdigit(vec[4][0]) ? "TV" : vec[4]);
977

    
978
        ctx.lastResultItemType = VECTOR;
979
        ctx.lastResultItemIndex = addVector(ctx.fileRunRef, vectorId, vec[2], vec[3], columns);
980
        ctx.clearHistogram();
981
    }
982
    else if (vec[0][0]=='s' && !strcmp(vec[0],"statistic"))
983
    {
984
        // syntax: "statistic <module> <statisticname>"
985
        CHECK(numTokens==3, "incorrect 'statistic' line -- statistic <module> <statisticname> expected");
986

    
987
        ctx.clearHistogram();
988
        ctx.moduleName = vec[1];
989
        ctx.statisticName = vec[2];
990
        ctx.lastResultItemType = SCALAR; // add scalars first
991
        ctx.lastResultItemIndex = ctx.fileRef->scalarResults.size();
992

    
993
        CHECK(!ctx.moduleName.empty(), "missing module name");
994
        CHECK(!ctx.statisticName.empty(), "missing statistics name");
995
    }
996
    else if (vec[0][0]=='f' && !strcmp(vec[0],"field"))
997
    {
998
        // syntax: "field <name> <value>"
999
        CHECK(numTokens==3, "incorrect 'field' line -- field <name> <value> expected");
1000

    
1001
        std::string fieldName = vec[1];
1002
        double value;
1003
        CHECK(parseDouble(vec[2], value), "invalid scalar file: invalid field value");
1004

    
1005
        CHECK(!ctx.moduleName.empty() && !ctx.statisticName.empty(),
1006
                "invalid scalar file: missing statistics declaration");
1007
        std::string scalarName = ctx.statisticName + ":" + fieldName;
1008

    
1009
        // set statistics field in the current histogram
1010
        bool isField = true;
1011
        if (fieldName == "count")
1012
            ctx.count = (long)value;
1013
        else if (fieldName == "min")
1014
            ctx.min = value;
1015
        else if (fieldName == "max")
1016
            ctx.max = value;
1017
        else if (fieldName == "sum")
1018
            ctx.sum = value;
1019
        else if (fieldName == "sqrsum")
1020
            ctx.sumSqr = value;
1021
        else if (fieldName != "mean" && fieldName != "stddev")
1022
            isField = false;
1023

    
1024
        addScalar(ctx.fileRunRef, ctx.moduleName.c_str(), scalarName.c_str(), value, isField);
1025
    }
1026
    else if (vec[0][0]=='b' && !strcmp(vec[0],"bin"))
1027
    {
1028
        // syntax: "bin <lower_bound> <value>"
1029
        CHECK(numTokens==3, "incorrect 'bin' line -- bin <lowerBound> <value> expected");
1030
        double lower_bound, value;
1031
        CHECK(parseDouble(vec[1], lower_bound), "invalid lower bound");
1032
        CHECK(parseDouble(vec[2], value), "invalid bin value");
1033

    
1034
        if (ctx.lastResultItemType != HISTOGRAM)
1035
        {
1036
            CHECK(ctx.lastResultItemType == SCALAR && !ctx.moduleName.empty() && !ctx.statisticName.empty(),
1037
                  "stray 'bin' line (not preceded by a 'statistic' line)");
1038
            Statistics stat(ctx.count, ctx.min, ctx.max, ctx.sum, ctx.sumSqr);
1039
            const ScalarResults &scalars = ctx.fileRef->scalarResults;
1040
            const StringMap &attrs = ctx.lastResultItemIndex < (int)scalars.size() ?
1041
                                        scalars[ctx.lastResultItemIndex].attributes : StringMap();
1042
            ctx.lastResultItemType = HISTOGRAM;
1043
            ctx.lastResultItemIndex = addHistogram(ctx.fileRunRef, ctx.moduleName.c_str(), ctx.statisticName.c_str(), stat, attrs);
1044
        }
1045
        HistogramResult& histogram = ctx.fileRef->histogramResults[ctx.lastResultItemIndex];
1046
        try {
1047
            histogram.addBin(lower_bound, value);
1048
        } catch (std::exception& e) {
1049
            throw ResultFileFormatException(e.what(), ctx.fileName, ctx.lineNo);
1050
        }
1051
    }
1052
    else if (vec[0][0]=='a' && !strcmp(vec[0],"attr"))
1053
    {
1054
        // syntax: "attr <name> <value>"
1055
        CHECK(numTokens==3, "incorrect 'attr' line -- attr <name> <value> expected");
1056

    
1057
        std::string attrName = vec[1];
1058
        std::string attrValue = vec[2];
1059

    
1060
        if (ctx.lastResultItemType == 0) // run attribute
1061
        {
1062
            // store attribute
1063
            StringMap &attributes = ctx.fileRunRef->runRef->attributes;
1064
            StringMap::iterator oldPairRef = attributes.find(attrName);
1065
            CHECK(oldPairRef == attributes.end() || oldPairRef->second == attrValue,
1066
                  "Value of run attribute conflicts with previously loaded value");
1067
            attributes[attrName] = attrValue;
1068

    
1069
            // the "runNumber" attribute is also stored separately
1070
            if (attrName == "runNumber")
1071
                CHECK(parseInt(vec[2], ctx.fileRunRef->runRef->runNumber), "invalid result file: int value expected as runNumber");
1072
        }
1073
        else if (ctx.lastResultItemIndex >= 0) // resultItem attribute
1074
        {
1075
            if (ctx.lastResultItemType == SCALAR)
1076
                for (int i=ctx.lastResultItemIndex; i < (int)ctx.fileRef->scalarResults.size() ;++i)
1077
                {
1078
                    ctx.fileRef->scalarResults[i].attributes[attrName] = attrValue;
1079
                }
1080
            else if (ctx.lastResultItemType == VECTOR)
1081
                for (int i=ctx.lastResultItemIndex; i < (int)ctx.fileRef->vectorResults.size() ;++i)
1082
                {
1083
                    ctx.fileRef->vectorResults[i].attributes[attrName] = attrValue;
1084
                }
1085
            else if (ctx.lastResultItemType == HISTOGRAM)
1086
                for (int i=ctx.lastResultItemIndex; i < (int)ctx.fileRef->histogramResults.size() ;++i)
1087
                {
1088
                    ctx.fileRef->histogramResults[i].attributes[attrName] = attrValue;
1089
                }
1090
        }
1091
    }
1092
    else if (vec[0][0]=='p' && !strcmp(vec[0],"param"))
1093
    {
1094
        // syntax: "param <namePattern> <value>"
1095
        CHECK(numTokens==3, "incorrect 'param' line -- param <namePattern> <value> expected");
1096

    
1097
        // store module param
1098
        std::string paramName = vec[1];
1099
        std::string paramValue = vec[2];
1100
        StringMap &params = ctx.fileRunRef->runRef->moduleParams;
1101
        StringMap::iterator oldPairRef = params.find(paramName);
1102
        CHECK(oldPairRef == params.end() || oldPairRef->second == paramValue,
1103
              "Value of module parameter conflicts with previously loaded value");
1104
        params[paramName] = paramValue;
1105
    }
1106
    else if (opp_isdigit(vec[0][0]) && numTokens>=3)
1107
    {
1108
        // this looks like a vector data line, skip it this time
1109
        // syntax: "<vectorID> <2-or-3-columns>"
1110
        CHECK(numTokens==3 || numTokens==4, "incorrect vector data line -- 3 or 4 items expected");
1111
    }
1112
    else
1113
    {
1114
        // ignore unknown lines and vector data lines
1115
        //ctx.fileRef->numUnrecognizedLines++; -- counter not used any more
1116
        CHECK(false, "unrecognized line");
1117
    }
1118
}
1119

    
1120
static bool isFileReadable(const char *fileName)
1121
{
1122
    FILE *f = fopen(fileName, "r");
1123
    if (f!=NULL)
1124
    {
1125
        fclose(f);
1126
        return true;
1127
    }
1128
    else
1129
        return false;
1130
}
1131

    
1132
ResultFile *ResultFileManager::loadFile(const char *fileName, const char *fileSystemFileName, bool reload)
1133
{
1134
    WRITER_MUTEX
1135
    // tricky part: we store "fileName" which is the Eclipse pathname of the file
1136
    // (or some other "display name" of the file), but we actually load from
1137
    // fileSystemFileName which is the fileName suitable for fopen()
1138

    
1139
    // check if loaded
1140
    ResultFile *fileRef = getFile(fileName);
1141
    if (fileRef) {
1142
        if (reload) {
1143
            unloadFile(fileRef);
1144
        }
1145
        else
1146
            return fileRef;
1147
    }
1148

    
1149
    // try if file can be opened, before we add it to our database
1150
    if (fileSystemFileName==NULL)
1151
        fileSystemFileName = fileName;
1152
    if (!isFileReadable(fileSystemFileName))
1153
        throw opp_runtime_error("cannot open `%s' for read", fileSystemFileName);
1154

    
1155
    // add to fileList
1156
    fileRef = NULL;
1157

    
1158
    try
1159
    {
1160
        fileRef = addFile(fileName, fileSystemFileName, false);
1161

    
1162
        // if vector file and has index, load vectors from the index file
1163
        if (IndexFile::isVectorFile(fileSystemFileName) && IndexFile::isIndexFileUpToDate(fileSystemFileName))
1164
        {
1165
            std::string indexFileName = IndexFile::getIndexFileName(fileSystemFileName);
1166
            loadVectorsFromIndex(indexFileName.c_str(), fileRef);
1167
        }
1168
        else
1169
        {
1170
            // process lines in file
1171
            FileReader freader(fileSystemFileName);
1172
            char *line;
1173
            LineTokenizer tokenizer;
1174
            sParseContext ctx(fileRef);
1175
            while ((line=freader.getNextLineBufferPointer())!=NULL)
1176
            {
1177
                int len = freader.getCurrentLineLength();
1178
                int numTokens = tokenizer.tokenize(line, len);
1179
                char **tokens = tokenizer.tokens();
1180
                processLine(tokens, numTokens, ctx);
1181
            }
1182

    
1183
            fileRef->numLines = ctx.lineNo; // freader.getNumReadLines();
1184
        }
1185
    }
1186
    catch (std::exception&)
1187
    {
1188
        try
1189
        {
1190
            if (fileRef)
1191
                unloadFile(fileRef);
1192
        }
1193
        catch (...) {}
1194

    
1195
        throw;
1196
    }
1197

    
1198
    return fileRef;
1199
}
1200

    
1201
void ResultFileManager::loadVectorsFromIndex(const char *filename, ResultFile *fileRef)
1202
{
1203
    VectorFileIndex *index = IndexFileReader(filename).readAll();
1204
    int numOfVectors = index->getNumberOfVectors();
1205

    
1206
    if (numOfVectors == 0) {
1207
        delete index;
1208
        return;
1209
    }
1210

    
1211
    Run *runRef = getRunByName(index->run.runName.c_str());
1212
    if (!runRef)
1213
    {
1214
        runRef = addRun();
1215
        runRef->runName = index->run.runName;
1216
    }
1217
    runRef->runNumber = index->run.runNumber;
1218
    runRef->attributes = index->run.attributes;
1219
    runRef->moduleParams = index->run.moduleParams;
1220
    FileRun *fileRunRef = addFileRun(fileRef, runRef);
1221

    
1222
    for (int i = 0; i < numOfVectors; ++i)
1223
    {
1224
        const VectorData *vectorRef = index->getVectorAt(i);
1225
        assert(vectorRef);
1226

    
1227
        VectorResult vectorResult;
1228
        vectorResult.fileRunRef = fileRunRef;
1229
        vectorResult.attributes = StringMap(vectorRef->attributes);
1230
        vectorResult.vectorId = vectorRef->vectorId;
1231
        vectorResult.moduleNameRef = moduleNames.insert(vectorRef->moduleName);
1232
        vectorResult.nameRef = names.insert(vectorRef->name);
1233
        vectorResult.columns = vectorRef->columns;
1234
        vectorResult.startEventNum = vectorRef->startEventNum;
1235
        vectorResult.endEventNum = vectorRef->endEventNum;
1236
        vectorResult.startTime = vectorRef->startTime;
1237
        vectorResult.endTime = vectorRef->endTime;
1238
        vectorResult.stat = vectorRef->stat;
1239
        fileRef->vectorResults.push_back(vectorResult);
1240
    }
1241
    delete index;
1242
}
1243

    
1244
void ResultFileManager::unloadFile(ResultFile *file)
1245
{
1246
    WRITER_MUTEX
1247

    
1248
    // remove computed vector IDs
1249
    for (ComputedIDCache::iterator it = computedIDCache.begin(); it != computedIDCache.end();)
1250
    {
1251
        ID id = it->first.second;
1252
        if (_fileid(id) == file->id)
1253
        {
1254
            ComputedIDCache::iterator oldIt = it;
1255
            it++;
1256
            computedIDCache.erase(oldIt);
1257
        }
1258
        else
1259
            ++it;
1260
    }
1261

    
1262
    // remove FileRun entries
1263
    RunList runsPotentiallyToBeDeleted;
1264
    for (int i=0; i<(int)fileRunList.size(); i++)
1265
    {
1266
        if (fileRunList[i]->fileRef==file)
1267
        {
1268
            // remember Run; if this was the last reference, we need to delete it
1269
            runsPotentiallyToBeDeleted.push_back(fileRunList[i]->runRef);
1270

    
1271
            // delete fileRun
1272
            delete fileRunList[i];
1273
            fileRunList.erase(fileRunList.begin()+i);
1274
            i--;
1275
        }
1276
    }
1277

    
1278
    // delete ResultFile entry. Note that the fileList array will have a hole.
1279
    // It is not allowed to move another ResultFile into the hole, because
1280
    // that would change its "id", and invalidate existing IDs (IDLists)
1281
    // that use that file.
1282
    fileList[file->id] = NULL;
1283
    delete file;
1284

    
1285
    // remove Runs that don't appear in other loaded files
1286
    for (int i=0; i<(int)runsPotentiallyToBeDeleted.size(); i++)
1287
    {
1288
        Run *runRef = runsPotentiallyToBeDeleted[i];
1289
        if (getFilesForRun(runRef).empty())
1290
        {
1291
            // delete it.
1292
            RunList::iterator it = std::find(runList.begin(), runList.end(), runRef);
1293
            assert(it != runList.end());  // runs may occur only once in runsPotentiallyToBeDeleted, because runNames are not allowed to repeat in files
1294
            runList.erase(it);
1295
        }
1296
    }
1297
}
1298

    
1299
/*--------------------------------------------------------------------------
1300
 *                        compute filter hints
1301
 *--------------------------------------------------------------------------*/
1302

    
1303
/**
1304
 * Utility class: collects strings that occur at least twice.
1305
 */
1306
class DuplicateStringCollector
1307
{
1308
  private:
1309
    typedef std::map<std::string,int> StringCountMap;
1310
    StringCountMap map;
1311
    StringVector vec;
1312
  public:
1313
    void add(const std::string& str) {
1314
        StringCountMap::iterator mi = map.find(str);
1315
        if (mi==map.end())
1316
            map[str] = 1;  // 1st occurrence
1317
        else if (++(mi->second)==2)  // 2nd occurrence
1318
            vec.push_back(str);
1319
    }
1320
    StringVector& get() {
1321
        return vec;
1322
    }
1323
};
1324

    
1325
inline bool strdictLess(const std::string &first, const std::string &second)
1326
{
1327
    return strdictcmp(first.c_str(), second.c_str()) < 0;
1328
}
1329

    
1330
struct StrDictCompare
1331
{
1332
    bool operator()(const std::string &first, const std::string &second)
1333
    {
1334
        return strdictLess(first, second);
1335
    }
1336
};
1337

    
1338
typedef std::set<std::string, StrDictCompare> SortedStringSet;
1339

    
1340
static void replaceDigitsWithWildcard(std::string &str)
1341
{
1342
    std::string::iterator start;
1343
    while ((start=std::find_if(str.begin(), str.end(), opp_isdigit))!=str.end())
1344
    {
1345
        std::string::iterator end = std::find_if(start, str.end(), std::not1(std::ptr_fun(opp_isdigit)));
1346
        str.replace(start, end, 1, '*');
1347
    }
1348
}
1349

    
1350
StringVector *ResultFileManager::getFileAndRunNumberFilterHints(const IDList& idlist) const
1351
{
1352
    READER_MUTEX
1353
    FileRunList *fileRuns = getUniqueFileRuns(idlist);
1354

    
1355
    StringVector vec;
1356
    DuplicateStringCollector coll;
1357

    
1358
    for (int i=0; i<(int)fileRuns->size(); i++)
1359
    {
1360
        FileRun *fileRun = (*fileRuns)[i];
1361
        if (fileRun->runRef->runNumber==0)
1362
        {
1363
            vec.push_back(fileRun->fileRef->filePath);
1364
        }
1365
        else
1366
        {
1367
            char runNumberStr[32];
1368
            sprintf(runNumberStr, "%d", fileRun->runRef->runNumber);
1369
            vec.push_back(fileRun->fileRef->filePath+"#"+runNumberStr);
1370
            coll.add(fileRun->fileRef->filePath+"#*");
1371
        }
1372
    }
1373
    delete fileRuns;
1374

    
1375
    // sort and concatenate them, and return the result
1376
    StringVector *wildvec = new StringVector(coll.get());
1377
    std::sort(vec.begin(), vec.end(), strdictLess);
1378
    std::sort(wildvec->begin(), wildvec->end(), strdictLess);
1379
    wildvec->insert(wildvec->end(), vec.begin(), vec.end());
1380
    return wildvec;
1381
}
1382

    
1383
StringVector *ResultFileManager::getFilePathFilterHints(const ResultFileList &fileList) const
1384
{
1385
    READER_MUTEX
1386
    StringVector *filterHints = new StringVector;
1387
    for (ResultFileList::const_iterator it = fileList.begin(); it != fileList.end(); ++it)
1388
        filterHints->push_back((*it)->filePath);
1389
    std::sort(filterHints->begin(), filterHints->end(), strdictLess);
1390
    filterHints->insert(filterHints->begin(), "*");
1391
    return filterHints;
1392
}
1393

    
1394
StringVector *ResultFileManager::getRunNameFilterHints(const RunList &runList) const
1395
{
1396
    READER_MUTEX
1397
    StringVector *filterHints = new StringVector;
1398
    for (RunList::const_iterator it = runList.begin(); it != runList.end(); ++it)
1399
        if ((*it)->runName.size() > 0)
1400
            filterHints->push_back((*it)->runName);
1401
    std::sort(filterHints->begin(), filterHints->end(), strdictLess);
1402
    filterHints->insert(filterHints->begin(), "*");
1403
    return filterHints;
1404
}
1405

    
1406
StringVector *ResultFileManager::getModuleFilterHints(const IDList& idlist) const
1407
{
1408
    READER_MUTEX
1409
    StringSet& names = *getUniqueModuleNames(idlist);
1410

    
1411
    SortedStringSet nameHints;
1412
    DuplicateStringCollector coll;
1413

    
1414
    for (StringSet::iterator i=names.begin(); i!=names.end(); i++)
1415
    {
1416
        std::string a = (*i);
1417

    
1418
        // replace embedded numbers with "*"
1419
        if (names.size() > 100)
1420
            replaceDigitsWithWildcard(a);
1421
        nameHints.insert(a);
1422

    
1423
        // break it up along dots, and...
1424
        StringTokenizer tokenizer(a.c_str(), ".");
1425
        const char *prefix = "";
1426
        const char *suffix = ".*";
1427
        while (tokenizer.hasMoreTokens()) {
1428
            const char *token = tokenizer.nextToken();
1429
            if (!tokenizer.hasMoreTokens()) suffix = "";
1430
            coll.add(std::string(prefix)+token+suffix);
1431
            prefix = "*.";
1432
        }
1433
    }
1434
    delete &names;
1435

    
1436
    // sort and concatenate them, and return the result
1437
    StringVector *wildvec = new StringVector(coll.get());
1438
    wildvec->push_back(std::string("*"));
1439
    std::sort(wildvec->begin(), wildvec->end(), strdictLess);
1440
    wildvec->insert(wildvec->end(), nameHints.begin(), nameHints.end());
1441
    return wildvec;
1442
}
1443

    
1444
StringVector *ResultFileManager::getNameFilterHints(const IDList& idlist) const
1445
{
1446
    READER_MUTEX
1447
    StringSet& names = *getUniqueNames(idlist);
1448

    
1449
    StringVector vec;
1450
    DuplicateStringCollector coll;
1451

    
1452
    for (StringSet::iterator i=names.begin(); i!=names.end(); i++)
1453
    {
1454
        std::string a = (*i);
1455
        vec.push_back(a);
1456

    
1457
        // break it up along spaces, and...
1458
        StringTokenizer tokenizer(a.c_str());
1459
        const char *prefix = "";
1460
        const char *suffix = " *";
1461
        while (tokenizer.hasMoreTokens()) {
1462
            const char *token = tokenizer.nextToken();
1463
            if (!tokenizer.hasMoreTokens()) suffix = "";
1464
            coll.add(std::string(prefix)+token+suffix);
1465
            prefix = "* ";
1466
        }
1467
    }
1468
    delete &names;
1469

    
1470
    // sort and concatenate them, and return the result
1471
    StringVector *wildvec = new StringVector(coll.get());
1472
    wildvec->push_back(std::string("*"));
1473
    std::sort(vec.begin(), vec.end(), strdictLess);
1474
    std::sort(wildvec->begin(), wildvec->end(), strdictLess);
1475
    wildvec->insert(wildvec->end(), vec.begin(), vec.end());
1476
    return wildvec;
1477
}
1478

    
1479
StringVector *ResultFileManager::getResultItemAttributeFilterHints(const IDList &idlist, const char *attrName) const
1480
{
1481
    READER_MUTEX
1482
    StringSet *attrValues = getUniqueAttributeValues(idlist, attrName);
1483
    StringVector *filterHints = new StringVector(attrValues->begin(), attrValues->end());
1484
    std::sort(filterHints->begin(), filterHints->end(), strdictLess);
1485
    filterHints->insert(filterHints->begin(), "*");
1486
    delete attrValues;
1487
    return filterHints;
1488
}
1489

    
1490
StringVector *ResultFileManager::getRunAttributeFilterHints(const RunList &runList, const char *attrName) const
1491
{
1492
    READER_MUTEX
1493
    StringSet *attrValues = getUniqueRunAttributeValues(runList, attrName);
1494
    StringVector *filterHints = new StringVector(attrValues->begin(), attrValues->end());
1495
    std::sort(filterHints->begin(), filterHints->end(), strdictLess);
1496
    filterHints->insert(filterHints->begin(), "*");
1497
    delete attrValues;
1498
    return filterHints;
1499
}
1500

    
1501
StringVector *ResultFileManager::getModuleParamFilterHints(const RunList &runList, const char * paramName) const
1502
{
1503
    READER_MUTEX
1504
    StringSet *paramValues = getUniqueModuleParamValues(runList, paramName);
1505
    StringVector *filterHints = new StringVector(paramValues->begin(), paramValues->end());
1506
    std::sort(filterHints->begin(), filterHints->end(), strdictLess);
1507
    filterHints->insert(filterHints->begin(), "*");
1508
    delete paramValues;
1509
    return filterHints;
1510
}
1511

    
1512
bool ResultFileManager::hasStaleID(const IDList& ids) const
1513
{
1514
    READER_MUTEX
1515
    for (int i = 0; i < ids.size(); ++i)
1516
    {
1517
        ID id = ids.get(i);
1518
        if (isStaleID(id))
1519
            return true;
1520
    }
1521
    return false;
1522
}
1523

    
1524
const char *ResultFileManager::getRunAttribute(ID id, const char *attribute) const
1525
{
1526
    return getItem(id).fileRunRef->runRef->getAttribute(attribute);
1527
}