Project

General

Profile

Statistics
| Branch: | Revision:

root / src / scave / datasorter.cc @ c87b95b0

History | View | Annotate | Download (18.4 KB)

1
//==========================================================================
2
//  DATASORTER.CC - part of
3
//                     OMNeT++/OMNEST
4
//            Discrete System Simulation in C++
5
//
6
//  Author: Andras Varga, Tamas Borbely
7
//
8
//==========================================================================
9

    
10
/*--------------------------------------------------------------*
11
  Copyright (C) 1992-2008 Andras Varga
12
  Copyright (C) 2006-2008 OpenSim Ltd.
13

14
  This file is distributed WITHOUT ANY WARRANTY. See the file
15
  `license' for details on this and other legal matters.
16
*--------------------------------------------------------------*/
17

    
18
#include <algorithm>
19
#include <utility>
20
#include <stdio.h>
21
#include <stdlib.h>
22
#include <string.h>
23
#include <assert.h>
24
#include "resultfilemanager.h"
25
#include "commonutil.h"
26
#include "datasorter.h"
27
#include "stringutil.h"
28

    
29
USING_NAMESPACE
30

    
31
using namespace std;
32

    
33

    
34
ResultFileManager *DataSorter::tmpResultFileMgr;
35

    
36
/*----------------------------------------
37
 *              XYDataset
38
 *----------------------------------------*/
39
void XYDataset::add(const ScalarResult &d)
40
{
41
    int row, column;
42

    
43
    KeyToIndexMap::iterator rowRef = rowKeyToIndexMap.find(d);
44
    if (rowRef != rowKeyToIndexMap.end())
45
    {
46
        row = rowOrder[rowRef->second];
47
    }
48
    else // add new row
49
    {
50
        row = rowKeys.size();
51
        values.push_back(vector<Statistics>(columnKeys.size(), Statistics()));
52
        rowKeyToIndexMap[d] = row;
53
        rowKeys.push_back(d);
54
        rowOrder.push_back(row);
55
    }
56

    
57
    KeyToIndexMap::iterator columnRef = columnKeyToIndexMap.find(d);
58
    if (columnRef != columnKeyToIndexMap.end())
59
    {
60
        column = columnOrder[columnRef->second];
61
    }
62
    else // add new column
63
    {
64
        column = columnKeys.size();
65
        for (vector<Row>::iterator rowRef = values.begin(); rowRef != values.end(); ++rowRef)
66
            rowRef->push_back(Statistics());
67
        columnKeyToIndexMap[d] = column;
68
        columnKeys.push_back(d);
69
        columnOrder.push_back(column);
70
    }
71

    
72
    values.at(row).at(column).collect(d.value);
73
}
74

    
75
void XYDataset::swapRows(int row1, int row2)
76
{
77
    int temp = rowOrder[row1];
78
    rowOrder[row1] = rowOrder[row2];
79
    rowOrder[row2] = temp;
80
}
81

    
82
void XYDataset::sortRows()
83
{
84
    vector<int> rowOrder;
85
    for (KeyToIndexMap::iterator it = rowKeyToIndexMap.begin(); it != rowKeyToIndexMap.end(); ++it)
86
        rowOrder.push_back(this->rowOrder[it->second]);
87

    
88
    this->rowOrder = rowOrder;
89
}
90

    
91
void XYDataset::sortColumns()
92
{
93
    vector<int> columnOrder;
94
    for (KeyToIndexMap::iterator it = columnKeyToIndexMap.begin(); it != columnKeyToIndexMap.end(); ++it)
95
        columnOrder.push_back(this->columnOrder[it->second]);
96
    this->columnOrder = columnOrder;
97
}
98

    
99
struct ValueAndIndex
100
{
101
    double value;
102
    int index;
103
    ValueAndIndex(double value, int index) : value(value), index(index) {}
104
    bool operator<(const ValueAndIndex& other) const { return this->value < other.value; }
105
};
106

    
107
void XYDataset::sortColumnsAccordingToFirstRowMean()
108
{
109
    if (values.size() > 0)
110
    {
111
        vector<ValueAndIndex> vals;
112
        int firstRow = rowOrder[0];
113
        for (int i = 0; i < (int)values[firstRow].size(); ++i)
114
        {
115
            double mean = values[firstRow][i].getMean();
116
            if (!isNaN(mean))
117
                vals.push_back(ValueAndIndex(mean, i));
118
        }
119

    
120
        sort(vals.begin(), vals.end());
121

    
122
        columnOrder = vector<int>(vals.size());
123
        for (int i = 0; i < (int)vals.size(); ++i)
124
            columnOrder[i] = vals[i].index;
125
    }
126
}
127

    
128
/*
129
 * Grouping functions
130
 */
131
bool DataSorter::sameGroupFileRunScalar(const ResultItem& d1, const ResultItem& d2)
132
{
133
    return d1.fileRunRef==d2.fileRunRef && d1.nameRef==d2.nameRef;
134
}
135

    
136
bool DataSorter::sameGroupModuleScalar(const ResultItem& d1, const ResultItem& d2)
137
{
138
    return d1.moduleNameRef==d2.moduleNameRef && d1.nameRef==d2.nameRef;
139
}
140

    
141
bool DataSorter::sameGroupFileRunModule(const ResultItem& d1, const ResultItem& d2)
142
{
143
    return d1.fileRunRef==d2.fileRunRef && d1.moduleNameRef==d2.moduleNameRef;
144
}
145

    
146
/*
147
 * Compare functions
148
 */
149
bool DataSorter::lessByModuleRef(ID id1, ID id2)
150
{
151
    if (id1==-1 || id2==-1) return id2!=-1; // -1 is the smallest
152
    const ResultItem& d1 = tmpResultFileMgr->getItem(id1);
153
    const ResultItem& d2 = tmpResultFileMgr->getItem(id2);
154
    return strdictcmp(d1.moduleNameRef->c_str(), d2.moduleNameRef->c_str()) < 0;
155
}
156

    
157
bool DataSorter::equalByModuleRef(ID id1, ID id2)
158
{
159
    if (id1==-1 || id2==-1) return id1==id2;
160
    const ResultItem& d1 = tmpResultFileMgr->getItem(id1);
161
    const ResultItem& d2 = tmpResultFileMgr->getItem(id2);
162
    return d1.moduleNameRef == d2.moduleNameRef;
163
}
164

    
165
bool DataSorter::lessByFileAndRun(ID id1, ID id2)
166
{
167
    if (id1==-1 || id2==-1) return id2!=-1; // -1 is the smallest
168
    const ResultItem& d1 = tmpResultFileMgr->getItem(id1);
169
    const ResultItem& d2 = tmpResultFileMgr->getItem(id2);
170

    
171
    // compare first by file, then by run
172
    int cmpFile = strdictcmp(d1.fileRunRef->fileRef->filePath.c_str(), d2.fileRunRef->fileRef->filePath.c_str());
173
    if (cmpFile!=0)
174
        return cmpFile < 0;
175
    int cmpRun = strdictcmp(d1.fileRunRef->runRef->runName.c_str(), d2.fileRunRef->runRef->runName.c_str());
176
    return cmpRun < 0;
177
}
178

    
179
bool DataSorter::equalByFileAndRun(ID id1, ID id2)
180
{
181
    if (id1==-1 || id2==-1) return id1==id2;
182
    const ResultItem& d1 = tmpResultFileMgr->getItem(id1);
183
    const ResultItem& d2 = tmpResultFileMgr->getItem(id2);
184
    return d1.fileRunRef == d2.fileRunRef;
185
}
186

    
187
bool DataSorter::lessByName(ID id1, ID id2)
188
{
189
    if (id1==-1 || id2==-1) return id2!=-1; // -1 is the smallest
190
    const ResultItem& d1 = tmpResultFileMgr->getItem(id1);
191
    const ResultItem& d2 = tmpResultFileMgr->getItem(id2);
192
    return strdictcmp(d1.nameRef->c_str(), d2.nameRef->c_str()) < 0;
193
}
194

    
195
bool DataSorter::equalByName(ID id1, ID id2)
196
{
197
    if (id1==-1 || id2==-1) return id1==id2;
198
    const ResultItem& d1 = tmpResultFileMgr->getItem(id1);
199
    const ResultItem& d2 = tmpResultFileMgr->getItem(id2);
200
    return d1.nameRef == d2.nameRef;
201
}
202

    
203
bool DataSorter::lessByValue(ID id1, ID id2)
204
{
205
    if (id1==-1 || id2==-1) return id2!=-1; // -1 is the smallest
206
    const ScalarResult& d1 = tmpResultFileMgr->getScalar(id1);
207
    const ScalarResult& d2 = tmpResultFileMgr->getScalar(id2);
208
    return d1.value < d2.value;
209
}
210

    
211
/*----------------------------------------
212
 *             DataSorter
213
 *----------------------------------------*/
214

    
215
template<class GroupingFn>
216
IDVectorVector DataSorter::doGrouping(const IDList& idlist, GroupingFn sameGroup)
217
{
218
    tmpResultFileMgr = resultFileMgr;
219

    
220
    // parse idlist and do grouping as well, on the fly
221
    IDVectorVector vv;
222
    int sz = idlist.size();
223
    for (int ii = 0; ii < sz; ii++)
224
    {
225
        ID id = idlist.get(ii);
226

    
227
        // check of this id shares fileRef, runNumber & scalarName with one of the
228
        // IDVectors already in vv
229
        const ResultItem& d = resultFileMgr->getItem(id);
230
        IDVectorVector::iterator i;
231
        for (i=vv.begin(); i!=vv.end(); ++i)
232
        {
233
            ID vvid = (*i)[0];  // first element in IDVector selected by i
234
            const ResultItem& vvd = resultFileMgr->getItem(vvid);
235
            if (sameGroup(d,vvd))
236
                break;
237
        }
238
        if (i==vv.end())
239
        {
240
            // not found -- new one has to be added
241
            vv.push_back(IDVector());
242
            i = vv.end()-1;
243
        }
244

    
245
        // insert
246
        i->push_back(id);
247
    }
248
    return vv;
249
}
250

    
251
template <class LessFn, class EqualFn>
252
void DataSorter::sortAndAlign(IDVectorVector& vv, LessFn less, EqualFn equal)
253
{
254
    tmpResultFileMgr = resultFileMgr;
255

    
256
    // order each group
257
    for (IDVectorVector::iterator i=vv.begin(); i!=vv.end(); ++i)
258
        std::sort(i->begin(), i->end(), less);
259

    
260
    // now insert "null" elements (id=-1) so that every group is of same length,
261
    // and same indices are "equal()"
262
    for (int pos=0; ; pos++)
263
    {
264
        // determine "smallest" element in all vectors, on position "pos"
265
        ID minId = -1;
266
        IDVectorVector::iterator i;
267
        for (i=vv.begin(); i!=vv.end(); ++i)
268
            if ((int)i->size()>pos)
269
                minId = (minId==-1) ? (*i)[pos] : less((*i)[pos],minId) ? (*i)[pos] : minId;
270

    
271
        // if pos is past the end of all vectors, we're done
272
        if (minId==-1)
273
            break;
274

    
275
        // if a vector has something different on this position, add a "null" element here
276
        for (i=vv.begin(); i!=vv.end(); ++i)
277
            if ((int)i->size()<=pos || !equal((*i)[pos],minId))
278
                i->insert(i->begin()+pos,-1);
279
    }
280
}
281

    
282
IDVectorVector DataSorter::groupByFields(const IDList& idlist, ResultItemFields fields)
283
{
284
    return doGrouping(idlist, ResultItemFieldsEqual(fields));
285
}
286

    
287

    
288
IDVectorVector DataSorter::groupAndAlign(const IDList& idlist, ResultItemFields fields)
289
{
290
    IDVectorVector vv = doGrouping(idlist, ResultItemFieldsEqual(fields));
291

    
292
    sortAndAlign(vv,
293
        IDFieldsLess(fields.complement(), resultFileMgr),
294
        IDFieldsEqual(fields.complement(), resultFileMgr));
295
    return vv;
296
}
297

    
298
/*
299
 * Returns true iff the jth column is missing
300
 * i.e. X value (i=0) is missing or each Y value (i>0) is missing.
301
 */
302
static bool isMissing(const IDVectorVector &vv, int j)
303
{
304
    if (vv[0][j]==-1)
305
        return true;
306
    bool foundY = false;
307
    for (int i=1; i<(int)vv.size();++i)
308
        if (vv[i][j]!=-1)
309
        {
310
            foundY = true;
311
            break;
312
        }
313
    return !foundY;
314
}
315

    
316
XYDataset DataSorter::groupAndAggregate(const IDList& scalars, ResultItemFields rowFields, ResultItemFields columnFields)
317
{
318
    Assert(scalars.areAllScalars());
319

    
320
    XYDataset dataset(rowFields, columnFields);
321

    
322
    int sz = scalars.size();
323
    for (int i = 0; i < sz; i++)
324
    {
325
        ID id = scalars.get(i);
326
        const ScalarResult& d = resultFileMgr->getScalar(id);
327
        dataset.add(d);
328
    }
329
    return dataset;
330
}
331

    
332

    
333
IDVectorVector DataSorter::prepareScatterPlot(const IDList& scalars, const char *moduleName, const char *scalarName)
334
{
335
    Assert(scalars.areAllScalars());
336

    
337
    // form groups (IDVectors) by moduleName+scalarName
338
    IDVectorVector vv = doGrouping(scalars, sameGroupModuleScalar);
339
    if (vv.size()==0)
340
        return vv;
341

    
342
    // order each group by fileRef+runNumber, and insert "null" elements (id=-1) so that
343
    // every group is of same length, and same indices contain same fileRef+runNumber
344
    sortAndAlign(vv, lessByFileAndRun, equalByFileAndRun);
345

    
346
    // find series for X axis (modulename, scalarname)...
347
    int xpos = -1;
348
    for (IDVectorVector::iterator i=vv.begin(); i!=vv.end(); ++i)
349
    {
350
        ID id = -1;
351
        for (IDVector::iterator j=i->begin(); j!=i->end(); ++j)
352
            if (*j!=-1)
353
                {id = *j;break;}
354
        if (id==-1)
355
            continue;
356
        const ScalarResult& d = resultFileMgr->getScalar(id);
357
        if (*d.moduleNameRef==moduleName && *d.nameRef==scalarName)
358
            {xpos = i-vv.begin();break;}
359
    }
360
    if (xpos==-1)
361
        throw opp_runtime_error("data for X axis not found");
362

    
363
    // ... and bring X series to 1st place
364
    if (xpos!=0)
365
        std::swap(vv[0], vv[xpos]);
366

    
367
    // sort x axis, moving elements in all other vectors as well.
368
    // Strategy: we'll construct the result in vv2. First we sort X axis, then
369
    // move elements of the other vectors to the same positions where the
370
    // X values went.
371

    
372
    IDVectorVector vv2;
373
    vv2.resize(vv.size());
374
    vv2[0] = vv[0];
375

    
376
    // step one: remove values where X value is missing or each Y value is missing (id=-1)
377
    IDVector::iterator it = vv2[0].begin();
378
    int sz = vv[0].size();
379
    for (int j=0; j<sz; ++j)
380
    {
381
        if(isMissing(vv,j))
382
            vv2[0].erase(it);
383
        else
384
            ++it;
385
    }
386

    
387
    // step two: sort X axis
388
    std::sort(vv2[0].begin(), vv2[0].end(), lessByValue);
389

    
390
    // step three: allocate all other vectors in vv2 to be the same length
391
    // (necessary because we'll fill them in via assignment, NOT push_back() or insert())
392
    for (int k=1; k<(int)vv2.size(); k++)
393
        vv2[k].resize(vv2[0].size());
394

    
395
    // step four: copy over elements
396
    for (int pos=0; pos<(int)vv[0].size(); pos++)
397
    {
398
        ID id = vv[0][pos];
399
        if (id==-1) continue;
400
        IDVector::iterator dest = std::find(vv2[0].begin(),vv2[0].end(),id);
401
        if (dest==vv2[0].end()) continue;
402
        int destpos = dest - vv2[0].begin();
403
        for (int k=1; k<(int)vv.size(); k++)
404
            vv2[k][destpos] = vv[k][pos];
405
    }
406

    
407
    return vv2;
408
}
409

    
410
XYDataset DataSorter::prepareScatterPlot2(const IDList& scalars, const char *moduleName, const char *scalarName,
411
            ResultItemFields rowFields, ResultItemFields columnFields)
412
{
413
    Assert(scalars.areAllScalars());
414

    
415
    XYDataset dataset = groupAndAggregate(scalars, rowFields, columnFields);
416

    
417
    // find row of x values and move it to row 0
418
    int row;
419
    for (row = 0; row < dataset.getRowCount(); ++row)
420
    {
421
        std::string moduleName = dataset.getRowField(row, ResultItemField(ResultItemField::MODULE));
422
        std::string dataName = dataset.getRowField(row, ResultItemField(ResultItemField::NAME));
423
        if (dataset.getRowField(row, ResultItemField(ResultItemField::MODULE)) == moduleName &&
424
            dataset.getRowField(row, ResultItemField(ResultItemField::NAME)) == scalarName)
425
            break;
426
    }
427
    if (row < dataset.getRowCount())
428
    {
429
        if (row > 0)
430
            dataset.swapRows(0, row);
431
    }
432
    else
433
        throw opp_runtime_error("Data for X axis not found.");
434

    
435
    // sort columns so that X values are in ascending order
436
    dataset.sortColumnsAccordingToFirstRowMean();
437

    
438
    return dataset;
439
}
440

    
441
struct IsoGroupingFn : public std::binary_function<ResultItem, ResultItem, bool>
442
{
443
    typedef map<Run*, vector<double> > RunIsoValueMap;
444
    typedef map<pair<string, string>, int> IsoAttrIndexMap;
445
    RunIsoValueMap isoMap;
446
    ResultItemFields fields;
447

    
448
    IsoGroupingFn() {}
449
    IDList init(const IDList &idlist, const StringVector &moduleNames, const StringVector &scalarNames,
450
                ResultItemFields fields, ResultFileManager *manager);
451
    bool operator()(const ResultItem &d1, const ResultItem &d2) const;
452
};
453

    
454
IDList IsoGroupingFn::init(const IDList &idlist, const StringVector &moduleNames, const StringVector &scalarNames,
455
                            ResultItemFields fields, ResultFileManager *manager)
456
{
457
    this->fields = fields;
458

    
459
    //assert(moduleNames.size() == scalarNames.size());
460
    int numOfIsoValues = scalarNames.size();
461
    IDList result;
462

    
463
    // build iso (module,scalar) -> index map
464
    IsoAttrIndexMap indexMap;
465
    for (int i = 0; i < numOfIsoValues; ++i)
466
    {
467
        pair<string,string> key = make_pair(moduleNames[i], scalarNames[i]);
468
        indexMap[key] = i;
469
    }
470

    
471
    // build run -> iso values map
472
    int sz = idlist.size();
473
    for (int i = 0; i < sz; ++i)
474
    {
475
        ID id = idlist.get(i);
476
        const ScalarResult &scalar = manager->getScalar(id);
477
        pair<string,string> key = make_pair(*scalar.moduleNameRef, *scalar.nameRef);
478
        IsoAttrIndexMap::iterator it = indexMap.find(key);
479
        if (it != indexMap.end())
480
        {
481
            int index = it->second;
482
            Run *run = scalar.fileRunRef->runRef;
483
            RunIsoValueMap::iterator it2 = isoMap.find(run);
484
            if (it2 == isoMap.end())
485
            {
486
                it2 = isoMap.insert(make_pair(run, vector<double>(numOfIsoValues, NaN))).first;
487
            }
488
            it2->second[index] = scalar.value;
489
        }
490
        else
491
            result.add(id);
492
    }
493

    
494
    return result;
495
}
496

    
497
bool IsoGroupingFn::operator()(const ResultItem &d1, const ResultItem &d2) const
498
{
499
    if (d1.fileRunRef->runRef == d2.fileRunRef->runRef)
500
        return true;
501

    
502
    if (!fields.equal(d1,d2))
503
        return false;
504

    
505
    if (isoMap.empty())
506
        return true;
507

    
508
    RunIsoValueMap::const_iterator it1 = isoMap.find(d1.fileRunRef->runRef);
509
    RunIsoValueMap::const_iterator it2 = isoMap.find(d2.fileRunRef->runRef);
510
    if (it1 == isoMap.end() || it2 == isoMap.end())
511
        return false;
512

    
513
    const vector<double> &v1 = it1->second;
514
    const vector<double> &v2 = it2->second;
515

    
516
    assert(v1.size() == v2.size());
517

    
518
    for (int i = 0; i < (int)v1.size(); ++i)
519
        if(v1[i] != v2[i])
520
            return false;
521

    
522
    return true;
523
}
524

    
525
XYDatasetVector DataSorter::prepareScatterPlot3(const IDList& scalars, const char *moduleName, const char *scalarName,
526
        ResultItemFields rowFields, ResultItemFields columnFields,
527
        const StringVector &isoModuleNames, const StringVector &isoScalarNames, ResultItemFields isoFields)
528
{
529
    Assert(scalars.areAllScalars());
530

    
531
    // group data according to iso fields
532
    IsoGroupingFn grouping;
533
    IDList nonIsoScalars = grouping.init(scalars, isoModuleNames, isoScalarNames, isoFields, resultFileMgr);
534
    IDVectorVector groupedScalars = doGrouping(nonIsoScalars, grouping);
535

    
536
    XYDatasetVector datasets;
537
    for (IDVectorVector::iterator group = groupedScalars.begin(); group != groupedScalars.end(); ++group)
538
    {
539
        IDList groupAsIDList;
540
        for (IDVector::iterator id = group->begin(); id != group->end(); id++)
541
            groupAsIDList.add(*id);
542

    
543
        try
544
        {
545
            XYDataset dataset = prepareScatterPlot2(groupAsIDList, moduleName, scalarName, rowFields, columnFields);
546
            datasets.push_back(dataset);
547
        }
548
        catch (opp_runtime_error &e) {
549
            // no X data for some iso values -> omit from the result
550
        }
551
    }
552

    
553
    return datasets;
554
}
555

    
556
IDList DataSorter::getModuleAndNamePairs(const IDList& idlist, int maxcount)
557
{
558
    IDList out;
559

    
560
    // go through idlist and pick ids that represent a new (module, name pair)
561
    for (int ii = 0; ii < (int)idlist.size(); ii++)
562
    {
563
        ID id = idlist.get(ii);
564

    
565
        // check if module and name of this id is already in out[]
566
        const ResultItem& d = resultFileMgr->getItem(id);
567
        int i;
568
        int outSize = out.size();
569
        for (i=0; i<outSize; i++)
570
        {
571
            const ResultItem& vd = resultFileMgr->getItem(out.get(i));
572
            if (d.moduleNameRef==vd.moduleNameRef && d.nameRef==vd.nameRef)
573
                break;
574
        }
575

    
576
        // not yet -- then add it
577
        if (i==outSize)
578
        {
579
            out.add(id);
580
            if (outSize>maxcount)
581
                break; // enough is enough
582
        }
583
    }
584

    
585
    return out;
586
}
587

    
588
IDVectorVector DataSorter::prepareCopyToClipboard(const IDList& idlist)
589
{
590
    // form groups (IDVectors) by fileRef+runNumber+moduleNameRef
591
    IDVectorVector vv = doGrouping(idlist, sameGroupFileRunModule);
592

    
593
    // order each group by scalar name, and insert "null" elements (id=-1) so that
594
    // every group is of same length, and same indices contain same scalarNameRefs
595
    sortAndAlign(vv, lessByName, equalByName);
596

    
597
    return vv;
598
}
599