Skip to content

Commit e646bd3

Browse files
friedenhefriedenhe
authored
Merge pull request #10 from DAFoam/fix_procFvPatch
Fixed the MPI communication in processorFvPatchField
2 parents c2222dc + e66ae8e commit e646bd3

File tree

8 files changed

+308
-295
lines changed

8 files changed

+308
-295
lines changed

‎src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C

Lines changed: 0 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -148,133 +148,4 @@ void Foam::PstreamBuffers::clear()
148148
}
149149

150150

151-
void Foam::PstreamBuffers::calcOneToOneCommList
152-
(
153-
List<DynamicList<label>> neighbProcList,
154-
List<List<label>>& commList
155-
)
156-
{
157-
/*
158-
Description:
159-
This function computes the one-to-one communication list. Essentially,
160-
if we get a listList of neighbour processor indices, we want to compute
161-
the commList such that, for each subList, the commList will have one
162-
processor communicating with only one other processor. This function is
163-
needed when exchanging information between processors. This is
164-
because, for some reason, when we do nonBlocking comm, each processor can not
165-
read/write data from/to more than one neighbour processor at a time.
166-
The message communication somehow mess up and we will get mixed (error) data.
167-
To fix this, we have to determine a so-called one-to-one communication
168-
list, such that, in this list, each subList contain the one-to-one
169-
processor communication index. Then we have to do commList.size() communication
170-
between processors, each time, we allow one processor to read/write data
171-
from only one neighbour and repeat this for all coupled patches.
172-
173-
Example:
174-
If the neighbour processor list is like this:
175-
neighbProcList={
176-
{1,2,4}, // processor0's neighbour is 1, 2, and 4
177-
{0,3}, // processor1's neighbour is 0, and 3
178-
{0,3},
179-
{1,2,4},
180-
{0,3}
181-
};
182-
183-
Then, the computed commList is (commList can't have repeated indices for each row)
184-
commList={
185-
{0,1,2,3}, // 1st round, we exchange data between procs 0<->1 and procs 2<->3
186-
{0,2,1,3}, // 2nd round, we exchange data between procs 0<->2 and procs 1<->3
187-
{0,4},
188-
{3,4}
189-
}
190-
*/
191-
192-
label maxIters = 1000;
193-
// main iteration
194-
for(label iter=0;iter<maxIters;iter++)
195-
{
196-
if (iter==maxIters-1)
197-
{
198-
FatalErrorIn("calcOneToOneCommList")<<"maxIters reach! "<< exit(FatalError);
199-
}
200-
201-
label finished = 1;
202-
List<label> empty;
203-
// append an empty list to commList for this iter
204-
commList.append(empty);
205-
//Info<<" commList "<<commList<<endl;
206-
// now loop over all processors to see if the procI is already in commList
207-
for(label procI=0; procI<Pstream::nProcs(); procI++)
208-
{
209-
if (commList[iter].found(procI))
210-
{
211-
// procI is already in commList, we need to check which neighbour procI is attached to this procI,
212-
// and we need to remove the neighbour procI from neighbProcList (it has been added in commList)
213-
// There are two possibilities:
214-
// 1. procI is an even number, which means it appears as neibProcI<->procI in commList
215-
// 2. procI is an odd number, which means it appears as procI<->neibProcI in commList
216-
// For case 1, we need to remove the index before procI, for case 2, we need to remove
217-
// the index after procI. This is because neibProcI<->procI always appear as a pair
218-
// See the following line in "else"
219-
// commList[iter].append(procI);
220-
// commList[iter].append(neighbProcI);
221-
label foundIndx = commList[iter].find(procI);
222-
label foundIndxRemainder = foundIndx % 2;
223-
label indxToRemove = -9999;
224-
if (foundIndxRemainder == 0)
225-
{
226-
indxToRemove = foundIndx + 1;
227-
}
228-
else
229-
{
230-
indxToRemove = foundIndx - 1;
231-
}
232-
label valToRemove = commList[iter][indxToRemove];
233-
forAll(neighbProcList[procI], idxI)
234-
{
235-
const label& neiVal = neighbProcList[procI][idxI];
236-
if (neiVal == valToRemove)
237-
{
238-
neighbProcList[procI].remove(idxI);
239-
break;
240-
}
241-
}
242-
}
243-
else
244-
{
245-
// procI is NOT in commList, we need to add procI into commList
246-
// we also need to add the neighbour procI that is associated with procI
247-
// into commList, if neighbour procI is not there yet.
248-
// Therefore, procI and neighbourProcI are always a pair in commList
249-
// After we add neighbour procI into commList, we need to remove
250-
// it from neighbProcList[procI]
251-
forAll(neighbProcList[procI], idxI)
252-
{
253-
const label& neighbProcI = neighbProcList[procI][idxI];
254-
if(!commList[iter].found(neighbProcI))
255-
{
256-
commList[iter].append(procI);
257-
commList[iter].append(neighbProcI);
258-
neighbProcList[procI].remove(idxI);
259-
//Info<<" neighbProcList "<<neighbProcList<<endl;
260-
//Info<<" commList "<<commList<<endl;
261-
break;
262-
}
263-
}
264-
265-
}
266-
if (neighbProcList[procI].size()!=0)
267-
{
268-
finished=0;
269-
}
270-
}
271-
if(finished)
272-
{
273-
break;
274-
}
275-
}
276-
277-
}
278-
279-
280151
// ************************************************************************* //

‎src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -180,26 +180,6 @@ public:
180180
//- Clear storage and reset
181181
void clear();
182182

183-
void setOneToOneList(const List<label>& oneToOneList)
184-
{
185-
oneToOneList_.clear();
186-
oneToOneList_.setSize(oneToOneList.size());
187-
forAll(oneToOneList_, idxI)
188-
{
189-
oneToOneList_[idxI] = oneToOneList[idxI];
190-
}
191-
}
192-
193-
const List<label>& getOneToOneList()
194-
{
195-
return oneToOneList_;
196-
}
197-
198-
void calcOneToOneCommList
199-
(
200-
List<DynamicList<label>> neighbProcList,
201-
List<List<label>>& commList
202-
);
203183
};
204184

205185

‎src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -418,6 +418,140 @@ Foam::UPstream::commsTypes Foam::UPstream::defaultCommsType
418418
)
419419
);
420420

421+
// CoDiPack4OpenFOAM. The following variables are created to enable parallel AD
422+
int Foam::UPstream::procOneToOneCommListIndex = -9999;
423+
424+
Foam::List< Foam::List<int> > Foam::UPstream::procOneToOneCommList;
425+
426+
427+
void Foam::UPstream::calcProcOneToOneCommList
428+
(
429+
List<DynamicList<label>> neighbProcList,
430+
List<List<label>>& commList
431+
)
432+
{
433+
/*
434+
Description:
435+
This function computes the processor one-to-one communication list.
436+
If we get a listList of neighbour processor indices, we want to compute
437+
the commList such that, for each subList, the commList will have one
438+
processor communicating with only one other processor. This function is
439+
needed when exchanging information between processors. This is
440+
because, for some reason, when we do nonBlocking comm, each processor can not
441+
read/write data from/to more than one neighbour processor at a time.
442+
The message communication somehow mess up and we will get mixed (error) data.
443+
To fix this, we have to determine a so-called one-to-one communication
444+
list, such that, in this list, each subList contain the one-to-one
445+
processor communication index. Then we have to do commList.size() communication
446+
between processors, each time, we allow one processor to read/write data
447+
from only one neighbour and repeat this for all coupled patches.
448+
449+
Example:
450+
If the neighbour processor list is like this:
451+
neighbProcList={
452+
{1,2,4}, // processor0's neighbour is 1, 2, and 4
453+
{0,3}, // processor1's neighbour is 0, and 3
454+
{0,3},
455+
{1,2,4},
456+
{0,3}
457+
};
458+
459+
Then, the computed commList is (commList can't have repeated indices for each row)
460+
commList={
461+
{0,1,2,3}, // 1st round, we exchange data between procs 0<->1 and procs 2<->3
462+
{0,2,1,3}, // 2nd round, we exchange data between procs 0<->2 and procs 1<->3
463+
{0,4},
464+
{3,4}
465+
}
466+
*/
467+
468+
label maxIters = 1000;
469+
// main iteration
470+
for(label iter=0;iter<maxIters;iter++)
471+
{
472+
if (iter==maxIters-1)
473+
{
474+
FatalErrorIn("calcOneToOneCommList")<<"maxIters reach! "<< Foam::exit(FatalError);
475+
}
476+
477+
label finished = 1;
478+
List<label> empty;
479+
// append an empty list to commList for this iter
480+
commList.append(empty);
481+
//Info<<" commList "<<commList<<endl;
482+
// now loop over all processors to see if the procI is already in commList
483+
for(label procI=0; procI<Pstream::nProcs(); procI++)
484+
{
485+
if (commList[iter].found(procI))
486+
{
487+
// procI is already in commList, we need to check which neighbour procI is attached to this procI,
488+
// and we need to remove the neighbour procI from neighbProcList (it has been added in commList)
489+
// There are two possibilities:
490+
// 1. procI is an even number, which means it appears as neibProcI<->procI in commList
491+
// 2. procI is an odd number, which means it appears as procI<->neibProcI in commList
492+
// For case 1, we need to remove the index before procI, for case 2, we need to remove
493+
// the index after procI. This is because neibProcI<->procI always appear as a pair
494+
// See the following line in "else"
495+
// commList[iter].append(procI);
496+
// commList[iter].append(neighbProcI);
497+
label foundIndx = commList[iter].find(procI);
498+
label foundIndxRemainder = foundIndx % 2;
499+
label indxToRemove = -9999;
500+
if (foundIndxRemainder == 0)
501+
{
502+
indxToRemove = foundIndx + 1;
503+
}
504+
else
505+
{
506+
indxToRemove = foundIndx - 1;
507+
}
508+
label valToRemove = commList[iter][indxToRemove];
509+
forAll(neighbProcList[procI], idxI)
510+
{
511+
const label& neiVal = neighbProcList[procI][idxI];
512+
if (neiVal == valToRemove)
513+
{
514+
neighbProcList[procI].remove(idxI);
515+
break;
516+
}
517+
}
518+
}
519+
else
520+
{
521+
// procI is NOT in commList, we need to add procI into commList
522+
// we also need to add the neighbour procI that is associated with procI
523+
// into commList, if neighbour procI is not there yet.
524+
// Therefore, procI and neighbourProcI are always a pair in commList
525+
// After we add neighbour procI into commList, we need to remove
526+
// it from neighbProcList[procI]
527+
forAll(neighbProcList[procI], idxI)
528+
{
529+
const label& neighbProcI = neighbProcList[procI][idxI];
530+
if(!commList[iter].found(neighbProcI))
531+
{
532+
commList[iter].append(procI);
533+
commList[iter].append(neighbProcI);
534+
neighbProcList[procI].remove(idxI);
535+
//Info<<" neighbProcList "<<neighbProcList<<endl;
536+
//Info<<" commList "<<commList<<endl;
537+
break;
538+
}
539+
}
540+
541+
}
542+
if (neighbProcList[procI].size()!=0)
543+
{
544+
finished=0;
545+
}
546+
}
547+
if(finished)
548+
{
549+
break;
550+
}
551+
}
552+
553+
}
554+
421555
namespace Foam
422556
{
423557
// Register re-reader

‎src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,19 @@ public:
290290
//- Debugging: warn for use of any communicator differing from warnComm
291291
static label warnComm;
292292

293+
// CoDiPack4OpenFOAM. The following variables are created to enable parallel AD
294+
//- Which procOneToOneList to use
295+
static label procOneToOneCommListIndex;
296+
297+
//- a list to store the one-to-one communication in parallel cases
298+
static List< List<int> > procOneToOneCommList;
299+
300+
//- calculate the procOneToOneCommList
301+
static void calcProcOneToOneCommList
302+
(
303+
List<DynamicList<label>> neighbProcList,
304+
List<List<label>>& commList
305+
);
293306

294307
// Constructors
295308

0 commit comments

Comments
 (0)