Add new CommDiagnostics and GpuDiagnostics managers

modules/standard/CommDiagnostics.chpl

@@ -151,8 +151,7 @@
   module present and once without it.
  */
 @unstable("The CommDiagnostics module is unstable and may change in the future")
-module CommDiagnostics
-{
+module CommDiagnostics {
   /*
     Print out stack traces for comm events printed after startVerboseComm
    */
@@ -253,32 +252,34 @@ module CommDiagnostics
      */
     var cache_put_misses: uint(64);
     /*
-      Number of prefetches issued to the remote cache at the granularity of cache pages.
-      This counter is specifically triggered via calls to chpl_comm_remote_prefetch
+      Number of prefetches issued to the remote cache at the granularity of
+      cache pages. This counter is specifically triggered via calls to
+      chpl_comm_remote_prefetch
     */
     var cache_num_prefetches : uint(64);
     /*
-      Number of readaheads issued to the remote cache at the granularity of cache pages.
+      Number of readaheads issued to the remote cache at the granularity of
+      cache pages.
     */
     var cache_num_page_readaheads : uint(64);
     /*
-      Number of cache pages that were prefetched but evicted from the cache before being accessed
-      (i.e., the prefetches were too early).
+      Number of cache pages that were prefetched but evicted from the cache
+      before being accessed (i.e., the prefetches were too early).
     */
     var cache_prefetch_unused : uint(64);
     /*
-      Number of cache pages that were prefetched but did not arrive in the cache before being accessed
-      (i.e., the prefetches were too late).
+      Number of cache pages that were prefetched but did not arrive in the
+      cache before being accessed (i.e., the prefetches were too late).
     */
     var cache_prefetch_waited : uint(64);
     /*
-      Number of cache pages that were read ahead but evicted from the cache before being accessed
-      (i.e., the readaheads were too early).
+      Number of cache pages that were read ahead but evicted from the cache
+      before being accessed (i.e., the readaheads were too early).
     */
     var cache_readahead_unused : uint(64);
     /*
-      Number of cache pages that were read ahead but did not arrive in the cache before being accessed
-      (i.e., the readaheads were too late).
+      Number of cache pages that were read ahead but did not arrive in the
+      cache before being accessed (i.e., the readaheads were too late).
     */
     var cache_readahead_waited : uint(64);
 
@@ -308,6 +309,7 @@ module CommDiagnostics
    */
   type commDiagnostics = chpl_commDiagnostics;
 
+  use ChapelIOSerialize;
   commDiagnostics implements writeSerializable;
 
   pragma "insert line file info"
@@ -433,25 +435,53 @@ module CommDiagnostics
     return cd;
   }
 
+  // TODO: can we have a manager for comm diagnostics in general?
+  // see test/library/standard/CommDiagnostics/manager.chpl for an example
+  // putting this in CommDiagnostics causes resolution issues
+  // since this standard module is used by internal modules and the context
+  // manager interface aggressively resolves functions, so we end up with
+  // use-before-def on things like LocaleSpace and Locales.
+
 
   /*
-    Print the current communication counts in a markdown table using a
-    row per locale and a column per operation.  By default, operations
+    A context manager that turns on verbose communication reporting for
+    the duration of the context.  See :proc:`startVerboseComm` and
+    :proc:`stopVerboseComm` for more information.
+  */
+  record verboseCommManager: contextManager {
+    @chpldoc.nodoc
+    proc enterContext() {
+      startVerboseComm();
+    }
+    @chpldoc.nodoc
+    proc exitContext(in e: owned Error?) {
+      stopVerboseComm();
+    }
+  }
+
+
+  /*
+    Print the communication counts in a markdown table using a
+    row per locale and a column per operation. By default, operations
     for which all locales have a count of zero are not displayed in
     the table, though an argument can be used to reverse that
     behavior.
 
+    :arg diagnostics: The diagnostics to print. This should be an array of
+                      :type:`commDiagnostics`, where each element of the array
+                      corresponds to a locale. If not provided,
+                      the diagnostics returned by
+                      :proc:`getCommDiagnostics` will be printed.
+    :type diagnostics: `[] commDiagnostics`
+
     :arg printEmptyColumns: Indicates whether empty columns should be printed (defaults to ``false``)
     :type printEmptyColumns: `bool`
   */
-  proc printCommDiagnosticsTable(printEmptyColumns=false) {
+  proc printCommDiagnosticsTable(diagnostics=getCommDiagnostics(), printEmptyColumns=false) {
     use Reflection, Math;
 
     param unstable = "unstable";
 
-    // grab all comm diagnostics
-    var CommDiags = getCommDiagnostics();
-
     // cache number of fields and store vector of whether field is active
     param nFields = getNumFields(chpl_commDiagnostics);
 
@@ -473,7 +503,7 @@ module CommDiagnostics
 
       var maxval = 0;
       for locID in LocaleSpace do
-        maxval = max(maxval, getField(CommDiags[locID], fieldID).safeCast(int));
+        maxval = max(maxval, getField(diagnostics[locID], fieldID).safeCast(int));
 
       if printEmptyColumns || maxval != 0 {
         const width = if commDiagsPrintUnstable == false && name == "amo"
@@ -501,7 +531,7 @@ module CommDiagnostics
       for param fieldID in 0..<nFields {
         var width = fieldWidth[fieldID];
         const count = if width < 0 then unstable
-                                   else getField(CommDiags[locID],
+                                   else getField(diagnostics[locID],
                                                  fieldID):string;
         if width != 0 then
           writef("| %*s ", abs(width), count);
@@ -527,5 +557,4 @@ module CommDiagnostics
     :proc:`getCommDiagnostics` for more information.
    */
   config param printInitCommCounts = false;
-
 }

modules/standard/GpuDiagnostics.chpl

@@ -234,4 +234,50 @@ module GpuDiagnostics
   private inline proc resetGpuDiagnosticsHere() {
     chpl_gpu_resetDiagnosticsHere();
   }
+
+  record gpuDiagnosticsManager: contextManager {
+    var autoPrint: bool;
+    @chpldoc.nodoc
+    var diagnostics: [LocaleSpace] gpuDiagnostics;
+
+    proc init(autoPrint: bool = false) {
+      this.autoPrint = autoPrint;
+    }
+
+    proc get() {
+      return diagnostics;
+    }
+
+    proc enterContext() {
+      resetGpuDiagnostics();
+      startGpuDiagnostics();
+    }
+    proc ref exitContext(in e: owned Error?) {
+      stopGpuDiagnostics();
+      diagnostics = getGpuDiagnostics();
+
+      if autoPrint {
+        writeln(diagnostics);
+      }
+    }
+  }
+
+
+  /*
+    A context manager that turns on verbose GPU communication reporting for
+    the duration of the context.  See :proc:`startVerboseGpu` and
+    :proc:`stopVerboseGpu` for more information.
+  */
+  record verboseGpuManager: contextManager {
+    @chpldoc.nodoc
+    proc enterContext() {
+      startVerboseGpu();
+    }
+    @chpldoc.nodoc
+    proc exitContext(in e: owned Error?) {
+      stopVerboseGpu();
+    }
+  }
+
+
 }

test/gpu/native/diagnostics.chpl

@@ -0,0 +1,32 @@
+use GpuDiagnostics;
+
+proc doKernel(numtrials, numnodes, numgpus) {
+  for 1..numtrials do
+    coforall locIdx in 0..<min(numnodes, numLocales) do on Locales[locIdx] {
+      coforall gpuId in 0..<min(numgpus, here.gpus.size) do
+        on here.gpus[gpuId] {
+          var A = [1, 2, 3, 4, 5];
+          A += 1;
+        }
+    }
+}
+
+manage new gpuDiagnosticsManager(autoPrint=true) {
+  doKernel(1, 2, 1);
+}
+writeln("finished\n");
+
+
+var m = new gpuDiagnosticsManager();
+writeln(m.get());
+manage m {
+  doKernel(1, 2, 1);
+}
+writeln("\nafter context");
+writeln(m.get());
+
+writeln("\nverbose");
+
+manage new verboseGpuManager() {
+  doKernel(1, 2, 1);
+}

test/gpu/native/diagnostics.good

@@ -0,0 +1,11 @@
+(kernel_launch = 1, host_to_device = 0, device_to_host = 0, device_to_device = 0) (kernel_launch = 1, host_to_device = 0, device_to_host = 0, device_to_device = 0)
+finished
+
+(kernel_launch = 0, host_to_device = 0, device_to_host = 0, device_to_device = 0) (kernel_launch = 0, host_to_device = 0, device_to_host = 0, device_to_device = 0)
+
+after context
+(kernel_launch = 1, host_to_device = 0, device_to_host = 0, device_to_device = 0) (kernel_launch = 1, host_to_device = 0, device_to_host = 0, device_to_device = 0)
+
+verbose
+0 (gpu 0): diagnostics.chpl:9: kernel launch (block size: 512x1x1)
+1 (gpu 0): diagnostics.chpl:9: kernel launch (block size: 512x1x1)

test/gpu/native/diagnostics.nomm-none.good

@@ -0,0 +1,10 @@
+(kernel_launch = 1, host_to_device = 0, device_to_host = 0, device_to_device = 0)
+finished
+
+(kernel_launch = 0, host_to_device = 0, device_to_host = 0, device_to_device = 0)
+
+after context
+(kernel_launch = 1, host_to_device = 0, device_to_host = 0, device_to_device = 0)
+
+verbose
+0 (gpu 0): diagnostics.chpl:9: kernel launch (block size: 512x1x1)

test/gpu/native/diagnostics.numlocales

@@ -0,0 +1 @@
+2

test/library/standard/CommDiagnostics/commDiagsTable.chpl

@@ -10,7 +10,7 @@ proc runtrial(numtrials, printEmpty) {
       on loc do
         x = 1;
   stopCommDiagnostics();
-  printCommDiagnosticsTable(printEmpty);
+  printCommDiagnosticsTable(printEmptyColumns=printEmpty);
   writeln();
 }
 

test/library/standard/CommDiagnostics/manager.chpl

@@ -0,0 +1,52 @@
+use CommDiagnostics;
+
+record commDiagnosticsManager: contextManager {
+  var autoPrint: bool;
+  @chpldoc.nodoc
+  var diagnostics: [LocaleSpace] commDiagnostics;
+
+  proc init(autoPrint: bool = false) {
+    this.autoPrint = autoPrint;
+  }
+
+  proc get() {
+    return diagnostics;
+  }
+
+  proc enterContext() {
+    resetCommDiagnostics();
+    startCommDiagnostics();
+  }
+  proc ref exitContext(in e: owned Error?) {
+    stopCommDiagnostics();
+    diagnostics = getCommDiagnostics();
+
+    if autoPrint {
+      printCommDiagnosticsTable(diagnostics=diagnostics);
+    }
+  }
+}
+
+
+proc doComm(numtrials) {
+  var x: int;
+  for 1..numtrials do
+    coforall loc in Locales with (ref x) do
+      on loc do
+        x = 1;
+}
+
+manage new commDiagnosticsManager(autoPrint=true) {
+  doComm(1);
+}
+writeln("finished\n");
+
+
+
+var m = new commDiagnosticsManager();
+printCommDiagnosticsTable(m.get());
+manage m {
+  doComm(1);
+}
+writeln("\nafter context");
+printCommDiagnosticsTable(m.get());

test/library/standard/CommDiagnostics/manager.comm-none.good

@@ -0,0 +1,13 @@
+| locale |
+| -----: |
+|      0 |
+finished
+
+| locale |
+| -----: |
+|      0 |
+
+after context
+| locale |
+| -----: |
+|      0 |

test/library/standard/CommDiagnostics/manager.compopts

@@ -0,0 +1 @@
+--no-cache-remote

test/library/standard/CommDiagnostics/manager.good

@@ -0,0 +1,22 @@
+| locale | put | execute_on_nb |
+| -----: | --: | ------------: |
+|      0 |   0 |             3 |
+|      1 |   1 |             0 |
+|      2 |   1 |             0 |
+|      3 |   1 |             0 |
+finished
+
+| locale |
+| -----: |
+|      0 |
+|      1 |
+|      2 |
+|      3 |
+
+after context
+| locale | put | execute_on_nb |
+| -----: | --: | ------------: |
+|      0 |   0 |             3 |
+|      1 |   1 |             0 |
+|      2 |   1 |             0 |
+|      3 |   1 |             0 |

test/library/standard/CommDiagnostics/manager.numlocales

@@ -0,0 +1 @@
+4

test/library/standard/CommDiagnostics/verboseManager.chpl

@@ -0,0 +1,14 @@
+use CommDiagnostics;
+
+proc doComm(numtrials) {
+  var x: int;
+  for 1..numtrials do
+    coforall loc in Locales with (ref x) do
+      on loc do
+        x = 1;
+}
+
+manage new verboseCommManager() {
+  doComm(1);
+}
+

test/library/standard/CommDiagnostics/verboseManager.compopts

@@ -0,0 +1 @@
+--no-cache-remote

test/library/standard/CommDiagnostics/verboseManager.good

@@ -0,0 +1,3 @@
+1: verboseManager.chpl:8: remote put, node 0, 8 bytes, commid 4
+2: verboseManager.chpl:8: remote put, node 0, 8 bytes, commid 4
+3: verboseManager.chpl:8: remote put, node 0, 8 bytes, commid 4

test/library/standard/CommDiagnostics/verboseManager.numlocales

@@ -0,0 +1 @@
+4

test/library/standard/CommDiagnostics/verboseManager.prediff

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+grep 'remote put' $2 | sort > $2.tmp
+mv $2.tmp $2