159 {
160
161 char ok = stop_logging ? 0 : 1;
162#ifdef USE_MPI
163 MPI_Allreduce(MPI_IN_PLACE, &ok, 1, MPI_CHAR, MPI_MIN, MPI_COMM_WORLD);
164#endif
165 if (ok == 0) {
167 stop_logging = true;
168
170 std::cout << "Warning: The memory log has stopped working "
171 << " due to max number of allocation having been reached\n";
172 return;
173 }
174
175
177 double min_sys_vmemory = sysmem.first;
178 double max_sys_vmemory = sysmem.first;
179 double mean_sys_vmemory = sysmem.first;
180 double min_sys_rssmemory = sysmem.second;
181 double max_sys_rssmemory = sysmem.second;
182 double mean_sys_rssmemory = sysmem.second;
183
184#ifdef USE_MPI
185 MPI_Allreduce(MPI_IN_PLACE, &min_sys_vmemory, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
186 MPI_Allreduce(MPI_IN_PLACE, &max_sys_vmemory, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
187 MPI_Allreduce(MPI_IN_PLACE, &mean_sys_vmemory, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
188 MPI_Allreduce(MPI_IN_PLACE, &min_sys_rssmemory, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);
189 MPI_Allreduce(MPI_IN_PLACE, &max_sys_rssmemory, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
190 MPI_Allreduce(MPI_IN_PLACE, &mean_sys_rssmemory, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
191#endif
192 mean_sys_vmemory /=
NTasks;
193 mean_sys_rssmemory /=
NTasks;
195 std::cout << "\n#=====================================================\n";
196 std::cout << "# MemoryLogging\n";
197 std::cout << "#=====================================================\n\n";
198 std::cout << "Info about resident set size:\n";
199 std::cout << "[Current resident set size]:\n";
200 std::cout << "Min over tasks: " << std::setw(15) << min_sys_vmemory / 1.0e6 << " MB\n";
201 std::cout << "Mean over tasks: " << std::setw(15) << mean_sys_vmemory / 1.0e6 << " MB\n";
202 std::cout << "Max over tasks: " << std::setw(15) << max_sys_vmemory / 1.0e6 << " MB\n";
203 std::cout << "[Peak resident set size]:\n";
204 std::cout << "Min over tasks: " << std::setw(15) << min_sys_rssmemory / 1.0e6 << " MB\n";
205 std::cout << "Mean over tasks: " << std::setw(15) << mean_sys_rssmemory / 1.0e6 << " MB\n";
206 std::cout << "Max over tasks: " << std::setw(15) << max_sys_rssmemory / 1.0e6 << " MB\n";
207 }
208
210 std::cout << "\n";
211 std::cout << "For info below we are only tracking allocation of standard\n";
212 std::cout << "container (Vector) and only allocations larger than " << min_bytes_to_log << " bytes\n\n";
213 }
214
215 long long int min_memory = memory_in_use;
216 long long int max_memory = memory_in_use;
217 long long int mean_memory = memory_in_use;
218 long long int peak_memory = peak_memory_use;
219#ifdef USE_MPI
220 MPI_Allreduce(MPI_IN_PLACE, &min_memory, 1, MPI_LONG_LONG, MPI_MIN, MPI_COMM_WORLD);
221 MPI_Allreduce(MPI_IN_PLACE, &max_memory, 1, MPI_LONG_LONG, MPI_MAX, MPI_COMM_WORLD);
222 MPI_Allreduce(MPI_IN_PLACE, &mean_memory, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD);
223 MPI_Allreduce(MPI_IN_PLACE, &peak_memory, 1, MPI_LONG_LONG, MPI_MAX, MPI_COMM_WORLD);
224#endif
227 std::cout << "Memory in use (Task 0): " << std::setw(15) << double(memory_in_use) / 1.0e6 << " MB\n";
228 std::cout << "Min over tasks: " << std::setw(15) << double(min_memory) / 1.0e6 << " MB\n";
229 std::cout << "Mean over tasks: " << std::setw(15) << double(mean_memory) / 1.0e6 << " MB\n";
230 std::cout << "Max over tasks: " << std::setw(15) << double(max_memory) / 1.0e6 << " MB\n";
231 std::cout << "\n";
232 std::cout << "Peak memory use (Task 0): " << std::setw(15) << double(peak_memory_use) / 1.0e6
233 << " MB\n";
234 std::cout << "Max over tasks: " << std::setw(15) << double(peak_memory) / 1.0e6 << " MB\n";
235 std::cout << "\n";
236 }
237 if (not allocations.empty()) {
239 std::cout << "\nWe have the following things allocated on task 0: \n";
240 }
241 for (auto && a : allocations) {
243 std::string name = "";
244 if (labels.find(a.first) != labels.end())
245 name = labels[a.first];
246 std::string bytelabel = " (MB)";
247 double factor = 1e6;
248 std::cout << "Address: " << a.first << " Size: " << double(a.second) / factor << bytelabel
249 << " Label: " << name << "\n";
250 }
251 }
253 std::cout << "\n";
254 }
255 if (not memory_vs_time.empty()) {
257 std::cout << "Total memory as function of time:\n";
258 for (auto && m : memory_vs_time) {
259 double time_in_sec =
260 std::chrono::duration_cast<std::chrono::duration<double>>(m.first - time_start).count();
262 std::cout << " Time (sec): " << std::setw(13) << time_in_sec;
263 std::string bytelabel = " (MB)";
264 double factor = 1e6;
265 std::cout << " Memory: " << std::setw(13) << double(m.second) / factor << bytelabel << "\n";
266 }
267 }
268 }
270 std::cout << "\n#=====================================================\n";
271 std::cout << std::flush;
272 }
273 }
std::pair< double, double > get_system_memory_use()
Fetch the resident set size currently and the peak value so far of it.
int ThisTask
The MPI task number.
int NTasks
Total number of MPI tasks.