Actual source code: ex8.c
1: static char help[] = "Demonstrates BuildTwoSided functions.\n";
3: #include <petscsys.h>
5: typedef struct {
6: PetscInt rank;
7: PetscScalar value;
8: char ok[3];
9: } Unit;
11: static PetscErrorCode MakeDatatype(MPI_Datatype *dtype)
12: {
13: MPI_Datatype dtypes[3], tmptype;
14: PetscMPIInt lengths[3];
15: MPI_Aint displs[3];
16: Unit dummy;
18: PetscFunctionBegin;
19: dtypes[0] = MPIU_INT;
20: dtypes[1] = MPIU_SCALAR;
21: dtypes[2] = MPI_CHAR;
22: lengths[0] = 1;
23: lengths[1] = 1;
24: lengths[2] = 3;
25: /* Curse the evil beings that made std::complex a non-POD type. */
26: displs[0] = (char *)&dummy.rank - (char *)&dummy; /* offsetof(Unit,rank); */
27: displs[1] = (char *)&dummy.value - (char *)&dummy; /* offsetof(Unit,value); */
28: displs[2] = (char *)&dummy.ok - (char *)&dummy; /* offsetof(Unit,ok); */
29: PetscCallMPI(MPI_Type_create_struct(3, lengths, displs, dtypes, &tmptype));
30: PetscCallMPI(MPI_Type_commit(&tmptype));
31: PetscCallMPI(MPI_Type_create_resized(tmptype, 0, sizeof(Unit), dtype));
32: PetscCallMPI(MPI_Type_commit(dtype));
33: PetscCallMPI(MPI_Type_free(&tmptype));
34: {
35: MPI_Aint lb, extent;
36: PetscCallMPI(MPI_Type_get_extent(*dtype, &lb, &extent));
37: PetscCheck(extent == sizeof(Unit), PETSC_COMM_WORLD, PETSC_ERR_LIB, "New type has extent %d != sizeof(Unit) %d", (int)extent, (int)sizeof(Unit));
38: }
39: PetscFunctionReturn(PETSC_SUCCESS);
40: }
42: struct FCtx {
43: PetscMPIInt rank;
44: PetscMPIInt nto;
45: PetscMPIInt *toranks;
46: Unit *todata;
47: PetscSegBuffer seg;
48: };
50: static PetscErrorCode FSend(MPI_Comm comm, const PetscMPIInt tag[], PetscMPIInt tonum, PetscMPIInt rank, void *todata, MPI_Request req[], void *ctx)
51: {
52: struct FCtx *fctx = (struct FCtx *)ctx;
54: PetscFunctionBegin;
55: PetscCheck(rank == fctx->toranks[tonum], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Rank %d does not match toranks[%d] %d", rank, tonum, fctx->toranks[tonum]);
56: PetscCheck(fctx->rank == *(PetscMPIInt *)todata, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Todata %d does not match rank %d", *(PetscMPIInt *)todata, fctx->rank);
57: PetscCallMPI(MPI_Isend(&fctx->todata[tonum].rank, 1, MPIU_INT, rank, tag[0], comm, &req[0]));
58: PetscCallMPI(MPI_Isend(&fctx->todata[tonum].value, 1, MPIU_SCALAR, rank, tag[1], comm, &req[1]));
59: PetscFunctionReturn(PETSC_SUCCESS);
60: }
62: static PetscErrorCode FRecv(MPI_Comm comm, const PetscMPIInt tag[], PetscMPIInt rank, void *fromdata, MPI_Request req[], void *ctx)
63: {
64: struct FCtx *fctx = (struct FCtx *)ctx;
65: Unit *buf;
67: PetscFunctionBegin;
68: PetscCheck(*(PetscMPIInt *)fromdata == rank, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Dummy data %d from rank %d corrupt", *(PetscMPIInt *)fromdata, rank);
69: PetscCall(PetscSegBufferGet(fctx->seg, 1, &buf));
70: PetscCallMPI(MPI_Irecv(&buf->rank, 1, MPIU_INT, rank, tag[0], comm, &req[0]));
71: PetscCallMPI(MPI_Irecv(&buf->value, 1, MPIU_SCALAR, rank, tag[1], comm, &req[1]));
72: buf->ok[0] = 'o';
73: buf->ok[1] = 'k';
74: buf->ok[2] = 0;
75: PetscFunctionReturn(PETSC_SUCCESS);
76: }
78: int main(int argc, char **argv)
79: {
80: PetscMPIInt rank, size, *toranks, *fromranks, nto, nfrom;
81: PetscInt i, n;
82: PetscBool verbose, build_twosided_f;
83: Unit *todata, *fromdata;
84: MPI_Datatype dtype;
86: PetscFunctionBeginUser;
87: PetscCall(PetscInitialize(&argc, &argv, (char *)0, help));
88: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
89: PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
91: verbose = PETSC_FALSE;
92: PetscCall(PetscOptionsGetBool(NULL, NULL, "-verbose", &verbose, NULL));
93: build_twosided_f = PETSC_FALSE;
94: PetscCall(PetscOptionsGetBool(NULL, NULL, "-build_twosided_f", &build_twosided_f, NULL));
96: for (i = 1, nto = 0; i < size; i *= 2) nto++;
97: PetscCall(PetscMalloc2(nto, &todata, nto, &toranks));
98: for (n = 0, i = 1; i < size; n++, i *= 2) {
99: toranks[n] = (rank + i) % size;
100: todata[n].rank = (rank + i) % size;
101: todata[n].value = (PetscScalar)rank;
102: todata[n].ok[0] = 'o';
103: todata[n].ok[1] = 'k';
104: todata[n].ok[2] = 0;
105: }
106: if (verbose) {
107: for (i = 0; i < nto; i++) PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d] TO %d: {%" PetscInt_FMT ", %g, \"%s\"}\n", rank, toranks[i], todata[i].rank, (double)PetscRealPart(todata[i].value), todata[i].ok));
108: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
109: }
111: PetscCall(MakeDatatype(&dtype));
113: if (build_twosided_f) {
114: struct FCtx fctx;
115: PetscMPIInt *todummy, *fromdummy;
116: fctx.rank = rank;
117: fctx.nto = nto;
118: fctx.toranks = toranks;
119: fctx.todata = todata;
120: PetscCall(PetscSegBufferCreate(sizeof(Unit), 1, &fctx.seg));
121: PetscCall(PetscMalloc1(nto, &todummy));
122: for (i = 0; i < nto; i++) todummy[i] = rank;
123: PetscCall(PetscCommBuildTwoSidedF(PETSC_COMM_WORLD, 1, MPI_INT, nto, toranks, todummy, &nfrom, &fromranks, &fromdummy, 2, FSend, FRecv, &fctx));
124: PetscCall(PetscFree(todummy));
125: PetscCall(PetscFree(fromdummy));
126: PetscCall(PetscSegBufferExtractAlloc(fctx.seg, &fromdata));
127: PetscCall(PetscSegBufferDestroy(&fctx.seg));
128: } else {
129: PetscCall(PetscCommBuildTwoSided(PETSC_COMM_WORLD, 1, dtype, nto, toranks, todata, &nfrom, &fromranks, &fromdata));
130: }
131: PetscCallMPI(MPI_Type_free(&dtype));
133: if (verbose) {
134: PetscInt *iranks, *iperm;
135: PetscCall(PetscMalloc2(nfrom, &iranks, nfrom, &iperm));
136: for (i = 0; i < nfrom; i++) {
137: iranks[i] = fromranks[i];
138: iperm[i] = i;
139: }
140: /* Receive ordering is non-deterministic in general, so sort to make verbose output deterministic. */
141: PetscCall(PetscSortIntWithPermutation(nfrom, iranks, iperm));
142: for (i = 0; i < nfrom; i++) {
143: PetscInt ip = iperm[i];
144: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d] FROM %d: {%" PetscInt_FMT ", %g, \"%s\"}\n", rank, fromranks[ip], fromdata[ip].rank, (double)PetscRealPart(fromdata[ip].value), fromdata[ip].ok));
145: }
146: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
147: PetscCall(PetscFree2(iranks, iperm));
148: }
150: PetscCheck(nto == nfrom, PETSC_COMM_SELF, PETSC_ERR_PLIB, "[%d] From ranks %d does not match To ranks %d", rank, nto, nfrom);
151: for (i = 1; i < size; i *= 2) {
152: PetscMPIInt expected_rank = (rank - i + size) % size;
153: PetscBool flg;
154: for (n = 0; n < nfrom; n++) {
155: if (expected_rank == fromranks[n]) goto found;
156: }
157: SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_PLIB, "[%d] Could not find expected from rank %d", rank, expected_rank);
158: found:
159: PetscCheck(PetscRealPart(fromdata[n].value) == expected_rank, PETSC_COMM_SELF, PETSC_ERR_PLIB, "[%d] Got data %g from rank %d", rank, (double)PetscRealPart(fromdata[n].value), expected_rank);
160: PetscCall(PetscStrcmp(fromdata[n].ok, "ok", &flg));
161: PetscCheck(flg, PETSC_COMM_SELF, PETSC_ERR_PLIB, "[%d] Got string %s from rank %d", rank, fromdata[n].ok, expected_rank);
162: }
163: PetscCall(PetscFree2(todata, toranks));
164: PetscCall(PetscFree(fromdata));
165: PetscCall(PetscFree(fromranks));
166: PetscCall(PetscFinalize());
167: return 0;
168: }
170: /*TEST
172: test:
173: nsize: 4
174: args: -verbose -build_twosided allreduce
176: test:
177: suffix: f
178: nsize: 4
179: args: -verbose -build_twosided_f -build_twosided allreduce
180: output_file: output/ex8_1.out
182: test:
183: suffix: f_ibarrier
184: nsize: 4
185: args: -verbose -build_twosided_f -build_twosided ibarrier
186: output_file: output/ex8_1.out
187: requires: defined(PETSC_HAVE_MPI_NONBLOCKING_COLLECTIVES)
189: test:
190: suffix: ibarrier
191: nsize: 4
192: args: -verbose -build_twosided ibarrier
193: output_file: output/ex8_1.out
194: requires: defined(PETSC_HAVE_MPI_NONBLOCKING_COLLECTIVES)
196: test:
197: suffix: redscatter
198: requires: mpi_reduce_scatter_block
199: nsize: 4
200: args: -verbose -build_twosided redscatter
201: output_file: output/ex8_1.out
203: TEST*/