<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang="">
<head>
  <meta charset="utf-8" />
  <meta name="generator" content="pandoc" />
  <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
  <meta name="author" content="Jared Hoberock and Michael Garland" />
  <meta name="dcterms.date" content="2020-06-15" />
  <title>Correcting the Design of Bulk Execution</title>
  <style>
    code{white-space: pre-wrap;}
    span.smallcaps{font-variant: small-caps;}
    span.underline{text-decoration: underline;}
    div.column{display: inline-block; vertical-align: top; width: 50%;}
    div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
    ul.task-list{list-style: none;}
    pre > code.sourceCode { white-space: pre; position: relative; }
    pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
    pre > code.sourceCode > span:empty { height: 1.2em; }
    code.sourceCode > span { color: inherit; text-decoration: inherit; }
    div.sourceCode { margin: 1em 0; }
    pre.sourceCode { margin: 0; }
    @media screen {
    div.sourceCode { overflow: auto; }
    }
    @media print {
    pre > code.sourceCode { white-space: pre-wrap; }
    pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
    }
    pre.numberSource code
      { counter-reset: source-line 0; }
    pre.numberSource code > span
      { position: relative; left: -4em; counter-increment: source-line; }
    pre.numberSource code > span > a:first-child::before
      { content: counter(source-line);
        position: relative; left: -1em; text-align: right; vertical-align: baseline;
        border: none; display: inline-block;
        -webkit-touch-callout: none; -webkit-user-select: none;
        -khtml-user-select: none; -moz-user-select: none;
        -ms-user-select: none; user-select: none;
        padding: 0 4px; width: 4em;
        background-color: #ffffff;
        color: #a0a0a0;
      }
    pre.numberSource { margin-left: 3em; border-left: 1px solid #a0a0a0;  padding-left: 4px; }
    div.sourceCode
      { color: #1f1c1b; background-color: #ffffff; }
    @media screen {
    pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
    }
    code span. { color: #1f1c1b; } /* Normal */
    code span.al { color: #bf0303; background-color: #f7e6e6; font-weight: bold; } /* Alert */
    code span.an { color: #ca60ca; } /* Annotation */
    code span.at { color: #0057ae; } /* Attribute */
    code span.bn { color: #b08000; } /* BaseN */
    code span.bu { color: #644a9b; font-weight: bold; } /* BuiltIn */
    code span.cf { color: #1f1c1b; font-weight: bold; } /* ControlFlow */
    code span.ch { color: #924c9d; } /* Char */
    code span.cn { color: #aa5500; } /* Constant */
    code span.co { color: #898887; } /* Comment */
    code span.cv { color: #0095ff; } /* CommentVar */
    code span.do { color: #607880; } /* Documentation */
    code span.dt { color: #0057ae; } /* DataType */
    code span.dv { color: #b08000; } /* DecVal */
    code span.er { color: #bf0303; text-decoration: underline; } /* Error */
    code span.ex { color: #0095ff; font-weight: bold; } /* Extension */
    code span.fl { color: #b08000; } /* Float */
    code span.fu { color: #644a9b; } /* Function */
    code span.im { color: #ff5500; } /* Import */
    code span.in { color: #b08000; } /* Information */
    code span.kw { color: #1f1c1b; font-weight: bold; } /* Keyword */
    code span.op { color: #1f1c1b; } /* Operator */
    code span.ot { color: #006e28; } /* Other */
    code span.pp { color: #006e28; } /* Preprocessor */
    code span.re { color: #0057ae; background-color: #e0e9f8; } /* RegionMarker */
    code span.sc { color: #3daee9; } /* SpecialChar */
    code span.ss { color: #ff5500; } /* SpecialString */
    code span.st { color: #bf0303; } /* String */
    code span.va { color: #0057ae; } /* Variable */
    code span.vs { color: #bf0303; } /* VerbatimString */
    code span.wa { color: #bf0303; } /* Warning */
  </style>
  <!--[if lt IE 9]>
    <script src="//cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv-printshiv.min.js"></script>
  <![endif]-->
  <style>
  /* ==========================================================================
     Basic formatting of the whole article
     ========================================================================== */

  @media screen
  {
      html { font-size: 11pt; }
  }

  @media print
  {
      html { font-size: 10pt; }
  }


  body {
      width: 45em;
      margin-top: 2em;
      margin-left: auto;
      margin-right: auto;
      
      font-family: "Times", "Times New Roman", serif;
      font-style: normal;
      font-variant: normal;

      background-color: white;
      color: black;
  }

  /* Adjustments for printing */
  @page {
      size: portrait;
      margin-top:    12%;
      margin-left:   12%;
      margin-top:    12%;
      margin-bottom: 18%;
      orphans: 3;
      widows:  3;
  }


  /* ==========================================================================
     Text elements
     ========================================================================== */

  p { text-align: justify; }
  li p { text-align: left; }

  h1,h2,h3,h4,h5,h6 {
      font-family: "Helvetica Neue", "Helvetica", "Arial", sans-serif; 
      page-break-after: avoid;
  }

  h1 { font-size: 1.73em; }
  h2 { font-size: 1.44em; }
  h3 { font-size: 1.20em; }
  h4 { font-size: 1.00em; }

  /* Pandoc hard-codes the section number in a span */
  span.header-section-number {
      margin-right: 0.5em;
  }

  span.header-section-number:after {
      content: ".";
  }

  /* Setup the header information at the top of the document */
  header   { text-align: center; margin-bottom: 4em; }
  h1.title { font-family: sans-serif; font-size: 2.0em; }
  p.subtitle { font-family: sans-serif; font-size: 1.4em; }
  p.author { font-size: 1.2em; }
  p.date   { font-size: 1.2em; }
  header p { text-align: center; }

  /* Special rules for code block formatting */
  pre
  {
      padding-left:   1em;
      padding-top:    1ex;
      padding-bottom: 1ex;

      font-family: "Courier New", "Courier", monospace;

      page-break-inside: avoid;

      overflow: visible;
  }

  blockquote
  {
      border-left: solid 2pt #ddd;
      padding-left: 1em;
  }

  /* ==========================================================================
     Figures, equations, and other numbered elements
     ========================================================================== */


  /* Figures and figure captions */
  body { counter-reset: figure; }

  figcaption:before
  {
      counter-increment: figure;
      content: "Figure " counter(figure) ": ";
  }

  figcaption { margin-top: 0.5em; }

  figure
  {
      margin-top: 1.5em;
      margin-bottom: 3em;
  }

  /* Patch default Pandoc style settings */
  @media screen {
      div.sourceCode { overflow: visible; }
  }
  </style>
</head>
<body>
<header id="title-block-header">
<h1 class="title">Correcting the Design of Bulk Execution</h1>
<p class="subtitle">P2181r0</p>
<p class="author">Jared Hoberock and Michael Garland</p>
<p class="date">June 15, 2020</p>
</header>
<section id="introduction" data-number="1">
<h1 data-number="1"><span class="header-section-number">1</span> Introduction</h1>
<p>A bulk execution interface was introduced as a fundamental operation supported by executors in <a href="http://wg21.link/N4406">N4406</a> (“Parallel algorithms need executors”) and adopted in <a href="http://wg21.link/p0443r0">P0443r0</a>, the first unified executor proposal, in the form of a <code>bulk_execute</code> interface. This interface has been present in P0443 from the beginning because a properly designed <code>bulk_execute</code> interface accomplishes two goals of fundamental importance. It provides the basis for exploiting platforms that support efficient mechanisms for creating many execution agents simultaneously, and it encapsulates the (potentially platform-specific) means of doing so.</p>
<p>The design of P0443 has evolved significantly since its initial revision, most notably to adopt the sender/receiver approach for lazy execution. The design of <code>bulk_execute</code> has lagged behind these changes, and is presented with inconsistent signatures in <a href="http://wg21.link/p0443r13">P0443r13</a>. The lack of a consistently defined interface for bulk execution must be resolved before P0443 can be adopted.</p>
<p>In this paper, we propose a design for bulk execution that corrects this defect in P0443r13. Our proposal:</p>
<ul>
<li><p>Defines <code>bulk_execute</code> as an interface for eager work submission, mirroring the semantics of <code>execute</code> in P0443r13.</p></li>
<li><p>Introduces a new <code>bulk_schedule</code> that provides a basis for lazy work submission, mirroring the semantics of <code>schedule</code> in P0443r13.</p></li>
<li><p>Introduces a <code>many_receiver_of</code> concept used to codify the requirements of senders returned by <code>bulk_schedule</code>, which must arrange for <code>set_value</code> to be called repeatedly.</p></li>
</ul>
<p>Adopting these proposals requires only minor changes to P0443. They do not change any of the concepts or mechanisms in P0443 aside from the defective definition of <code>bulk_execute</code>. They also make bulk execution more useful by providing for both eager and lazy submission, rather than eager submission alone.</p>
</section>
<section id="background" data-number="2">
<h1 data-number="2"><span class="header-section-number">2</span> Background</h1>
<p>Every revision of P0443 has included <code>bulk_execute</code> as the lowest level primitive operation for creating work in bulk through an executor. Both P0443 and the interface of <code>bulk_execute</code> have evolved since its first revision, but the intended functionality of <code>bulk_execute</code> has remained unchanged: it is the basis for creating a group of function invocations in bulk in a single operation.</p>
<p>The design sketched in <a href="http://wg21.link/p1660r0">P1660r0</a> (“A compromise executor design sketch”) is the basis for the current specification in <a href="http://wg21.link/p0443r13">P0443r13</a>. While reaffirming the importance of bulk execution, it proposed only to:</p>
<blockquote>
<p>Introduce a customizable bulk execution API whose specific shape is left as future work.</p>
</blockquote>
<p>Section 5.3 of that paper provided some “highly speculative” suggestions, but no definitive design was given. P0443r13 also attempts to incorporate the proposal of <a href="http://wg21.link/p1993r1">P1993r1</a> (“Restore shared state to <code>bulk_execute</code>”) to return a sender result so that dependent work may be chained with a bulk task.</p>
<p>This results in the intended interface of <code>bulk_execute</code> in P0443r13:</p>
<div class="sourceCode" id="cb1"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb1-1"><a href="#cb1-1"></a>sender_of&lt;<span class="dt">void</span>&gt; <span class="kw">auto</span> bulk_execute(executor <span class="kw">auto</span> ex,</span>
<span id="cb1-2"><a href="#cb1-2"></a>                                  invocable <span class="kw">auto</span> f,</span>
<span id="cb1-3"><a href="#cb1-3"></a>                                  <span class="dt">executor_shape_t</span>&lt;<span class="kw">decltype</span>(ex)&gt; shape);</span></code></pre></div>
<p>This formulation creates <code>shape</code> invocations of function <code>f</code> on execution agents created by executor <code>ex</code>. A sender of <code>void</code> corresponding to the completion of these invocations is the result.</p>
<section id="inconsistent-definitions-in-p0443" data-number="2.1">
<h2 data-number="2.1"><span class="header-section-number">2.1</span> Inconsistent definitions in P0443</h2>
<p>Despite this intent, the material addressing bulk execution in <a href="http://wg21.link/p0443r13">P0443r13</a> is not self-consistent. This inconsistency is particularly apparent in the envisioned return type of <code>bulk_execute</code>.</p>
<ul>
<li>Section 1.3 includes an example use of <code>bulk_execute</code> that returns a sender:</li>
</ul>
<div class="sourceCode" id="cb2"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb2-1"><a href="#cb2-1"></a>    sender <span class="kw">auto</span> s = execution::bulk_execute(ex, ...);</span></code></pre></div>
<ul>
<li><p>Section 2.2.3.9 specifies the customization point <code>execution::bulk_execute</code>, yet remains silent on its return type.</p></li>
<li><p>Section 2.5.5.5 specifies that the interface of <code>static_thread_pool</code> includes a <code>bulk_execute</code> method returning <code>void</code>:</p></li>
</ul>
<div class="sourceCode" id="cb3"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb3-1"><a href="#cb3-1"></a>    <span class="kw">template</span>&lt;<span class="kw">class</span> Function&gt;</span>
<span id="cb3-2"><a href="#cb3-2"></a>    <span class="dt">void</span> bulk_execute(Function&amp;&amp; f, <span class="dt">size_t</span> n) <span class="at">const</span>;</span></code></pre></div>
<p>Our proposal eliminates this inconsistency with a single, clearly defined interface for <code>bulk_execute</code>.</p>
</section>
<section id="shared-state-and-dependent-tasks" data-number="2.2">
<h2 data-number="2.2"><span class="header-section-number">2.2</span> Shared state and dependent tasks</h2>
<p>Programs need to chain dependent tasks together, in both the singular and bulk cases. Furthermore, it is particularly important to provide a means for delivering shared state (e.g., barrier objects or shared output arrays) to all the constituent invocations of a bulk operation.</p>
<p>SG1 considered this issue at its February 2020 meeting in Prague, and decided that:</p>
<blockquote>
<p>Poll: We should add a sender argument and sender result to bulk execution functions (providing an opportunity to build shared state, established dependencies in/out)</p>
<pre><code>SF  F  N  A  SA
17  7  0  0  0</code></pre>
</blockquote>
<p>Our proposal fulfills this requirement with a new <code>bulk_schedule</code> interface.</p>
</section>
</section>
<section id="corrected-bulk-interface" data-number="3">
<h1 data-number="3"><span class="header-section-number">3</span> Corrected Bulk Interface</h1>
<p>The inconsistent interfaces for bulk execution in <a href="http://wg21.link/p0443r13">P0443r13</a> arise from uncertainty about the means for integrating senders into the <code>bulk_execute</code> interface. The design for singular execution in P0443r13 avoids this confusion by providing <em>two</em> interfaces (<code>execute</code> and <code>schedule</code>) that disentangle the concerns of eager submission and lazy scheduling. The defects in the interface for bulk execution in P0443r13 are readily corrected by adopting a similar approach.</p>
<p>The <code>bulk_execute</code> operation should be the mechanism for eager submission of work in bulk, a role analogous to <code>execute</code>. Its interface should have the following form:</p>
<div class="sourceCode" id="cb5"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb5-1"><a href="#cb5-1"></a>    <span class="dt">void</span> bulk_execute(executor <span class="kw">auto</span> ex,</span>
<span id="cb5-2"><a href="#cb5-2"></a>                      invocable&lt;<span class="dt">executor_index_t</span>&lt;<span class="kw">decltype</span>(ex)&gt; <span class="kw">auto</span> f,</span>
<span id="cb5-3"><a href="#cb5-3"></a>                      <span class="dt">executor_shape_t</span>&lt;<span class="kw">decltype</span>(ex)&gt; shape);</span></code></pre></div>
<p>The invocable <code>f</code> has been submitted for execution in a group of the given shape before <code>bulk_execute</code> returns, but the point at which actual execution occurs is implementation defined. Thus, in the following example, some additional means of synchronization would be required before the vector <code>ints</code> can be used in another computation.</p>
<div class="sourceCode" id="cb6"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb6-1"><a href="#cb6-1"></a><span class="kw">auto</span> executor = ...</span>
<span id="cb6-2"><a href="#cb6-2"></a><span class="bu">std::</span>vector&lt;<span class="dt">int</span>&gt; ints = ...</span>
<span id="cb6-3"><a href="#cb6-3"></a></span>
<span id="cb6-4"><a href="#cb6-4"></a><span class="co">// launch work to mutate a vector of integers</span></span>
<span id="cb6-5"><a href="#cb6-5"></a>bulk_execute(executor,</span>
<span id="cb6-6"><a href="#cb6-6"></a>             [&amp;](<span class="dt">size_t</span> idx) { ints[i] += <span class="dv">1</span>; },</span>
<span id="cb6-7"><a href="#cb6-7"></a>             vec.size());</span></code></pre></div>
<p>A new interface is required for scheduling work for later submission. This interface should use senders as the means of composition. This is the role of <code>schedule</code> for singular execution; therefore, we propose the addition of an analogous bulk operation. This new <code>bulk_schedule</code> operation should have an interface of the following form:</p>
<div class="sourceCode" id="cb7"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb7-1"><a href="#cb7-1"></a>    sender <span class="kw">auto</span> bulk_schedule(executor <span class="kw">auto</span> ex,</span>
<span id="cb7-2"><a href="#cb7-2"></a>                              <span class="dt">executor_shape_t</span>&lt;<span class="kw">decltype</span>(ex)&gt; shape,</span>
<span id="cb7-3"><a href="#cb7-3"></a>                              sender <span class="kw">auto</span> prologue);</span></code></pre></div>
<p>A receiver connected to the sender returned by <code>bulk_schedule</code> will be submitted for execution in a group of the given shape upon a subsequent call to <code>start</code>.</p>
<p>The “prologue” sender provided to <code>bulk_schedule</code> is intended to deliver state that should be shared across the group of execution agents created upon execution. Each agent is identified by an index sent via <code>set_value</code> along with the shared state (if any) delivered by the prologue. The following example illustrates the use of <code>bulk_schedule</code>, along with functionality proposed in <a href="http://wg21.link/p1897r3">P1897r3</a>, to share a collection of integers across a group of execution agents and mutate each element individually.</p>
<div class="sourceCode" id="cb8"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb8-1"><a href="#cb8-1"></a><span class="kw">auto</span> executor = ...</span>
<span id="cb8-2"><a href="#cb8-2"></a><span class="bu">std::</span>vector&lt;<span class="dt">int</span>&gt; ints = ...</span>
<span id="cb8-3"><a href="#cb8-3"></a></span>
<span id="cb8-4"><a href="#cb8-4"></a><span class="co">// assemble a computation to mutate a vector of integers</span></span>
<span id="cb8-5"><a href="#cb8-5"></a><span class="kw">auto</span> increment =</span>
<span id="cb8-6"><a href="#cb8-6"></a>    bulk_schedule(executor, vec.size(), just(ints)) |</span>
<span id="cb8-7"><a href="#cb8-7"></a>    transform([](<span class="dt">size_t</span> idx, <span class="bu">std::</span>vector&lt;<span class="dt">int</span>&gt;&amp; ints)</span>
<span id="cb8-8"><a href="#cb8-8"></a>    {</span>
<span id="cb8-9"><a href="#cb8-9"></a>        ints[i] += <span class="dv">1</span>;</span>
<span id="cb8-10"><a href="#cb8-10"></a>    });</span>
<span id="cb8-11"><a href="#cb8-11"></a></span>
<span id="cb8-12"><a href="#cb8-12"></a><span class="co">// perform the computation</span></span>
<span id="cb8-13"><a href="#cb8-13"></a>execution::submit(increment, null_receiver{});</span></code></pre></div>
<p>We specify the action of the sender returned from <code>bulk_schedule</code> in terms of a call to <code>bulk_execute</code>, and the <a href="#appendix">Appendix</a> contains a reference implementation illustrating how this can be done. This design decision has two fundamental advantages: it encapsulates details of work submission in one place and guarantees semantic equivalence between eager and lazy mechanisms for work submission. Thus, assuming that our two examples use the same executor, the author of this code can be assured that both examples have the same semantics.</p>
<section id="specification-of-bulk_execute" data-number="3.1">
<h2 data-number="3.1"><span class="header-section-number">3.1</span> Specification of <code>bulk_execute</code></h2>
<p>[<em>Editorial note:</em> Replace Section 2.2.3.9 (<code>execution::bulk_execute</code>) in P0443r13 with the material in this section. –<em>end editorial note</em>]</p>
<p>The name <code>execution::bulk_execute</code> denotes a customization point object. If <code>is_convertible_v&lt;decltype(S), execution::executor_shape_t&lt;decltype(remove_cvref_t&lt;E&gt;)&gt;&gt;</code> is true, then the expression <code>execution::bulk_execute(E, F, S)</code> for some subexpressions <code>E</code>, <code>F</code>, and <code>S</code> is expression-equivalent to:</p>
<ul>
<li><p><code>E.bulk_execute(F, S)</code>, if that expression is valid. If the function selected does not execute <code>F</code> in an <code>S</code>-shaped group of execution agents with forward progress <code>query(E, execution::bulk_guarantee)</code> on executor <code>E</code>, the program is ill-formed with no diagnostic required.</p></li>
<li><p>Otherwise, <code>bulk_execute(E, F, S)</code>, if that expression is valid, with overload resolution performed in a context that includes the declaration</p>
<pre><code>  void bulk_execute();</code></pre>
<p>and that does not include a declaration of <code>execution::bulk_execute</code>.</p>
<p>If the function selected does not bulk execute <code>F</code> with shape <code>S</code> on executor <code>E</code>, the program is ill-formed with no diagnostic required.</p></li>
<li><p>Otherwise, if the type of <code>E</code> models <code>executor</code>, and the type of <code>F</code> and <code>executor_index_t&lt;remove_cvref_t&lt;E&gt;&gt;</code> model <code>invocable</code>, and if <code>query(E, execution::bulk_guarantee)</code> equals <code>execution::bulk_guarantee.unsequenced)</code></p>
<ul>
<li><p>If the type of <code>F</code> models <code>copy_constructible</code>, then equivalent to <code>execution::execute(E, [f=DECAY_COPY(F)]{ invoke(f, idx); })</code> for each <code>idx</code> spanned by <code>S</code>.</p></li>
<li><p>Otherwise, equivalent to <code>execution::execute(E, [&amp;]{ invoke(F, idx); })</code> for each <code>idx</code> spanned by <code>S</code>.</p></li>
</ul></li>
<li><p>Otherwise, <code>execution::bulk_execute(E, F, S)</code> is ill-formed.</p></li>
</ul>
</section>
<section id="specification-of-bulk_schedule" data-number="3.2">
<h2 data-number="3.2"><span class="header-section-number">3.2</span> Specification of <code>bulk_schedule</code></h2>
<p>[<em>Editorial note:</em> Introduce a new Section 2.2.3.10 (<code>execution::bulk_schedule</code>) containing the material in this section. –<em>end editorial note</em>]</p>
<p>The name <code>execution::bulk_schedule</code> denotes a customization point object. For some subexpressions <code>executor</code>, <code>shape</code>, and <code>prologue</code>, let <code>E</code> be a type such that <code>decltype((executor))</code> is <code>E</code>, and let <code>S</code> be a type such that <code>decltype((shape))</code> is <code>S</code>, and let <code>P</code> be a type such that <code>decltype((prologue))</code> is <code>P</code>. The expression <code>execution::bulk_schedule(executor, shape, prologue)</code> is ill-formed if <code>typed_sender&lt;P&gt;</code> is not <code>true</code>.</p>
<p>Otherwise, let <code>many-receiver</code> be the exposition-only type</p>
<pre><code>struct many-receiver {
  template&lt;class E&gt; void set_error(E&amp;&amp;) &amp;&amp; noexcept;

  void set_done() &amp;&amp; noexcept;

  template&lt;class... Args&gt;
  void set_value(S, Args&amp;...) noexcept;
};</code></pre>
<p>The expression <code>execution::bulk_scheduler(executor, shape, prologue)</code> is expression-equivalent to:</p>
<ul>
<li><p><code>executor.bulk_schedule(shape, prologue)</code>, if that expression is valid and its type <code>R</code> satisfies <code>sender_to&lt;R, many-receiver&gt;</code>, and if <code>sender_traits&lt;R&gt;::value_types&lt;tuple, variant&gt;</code> is <code>variant&lt;tuple&lt;executor_index_t&lt;decltype(executor)&gt;, add_lvalue_reference_t&lt;Values&gt;...&gt;...&gt;</code> for all <code>Values...</code> parameter packs sent by <code>prologue</code>.</p></li>
<li><p>Otherwise, <code>bulk_schedule(executor, shape, prologue)</code>, if that expression is valid with overload resolution performed in a context that includes the declaration</p>
<pre><code>void bulk_schedule();</code></pre>
<p>and that does not include a declaration of <code>execution::bulk_schedule</code>, and if that expression’s type satisfies <code>sender_to&lt;R, many-receiver&gt;</code>, and if <code>sender_traits&lt;R&gt;::value_types&lt;tuple, variant&gt;</code> is <code>variant&lt;tuple&lt;executor_index_t&lt;decltype(executor)&gt;, add_lvalue_reference_t&lt;Values&gt;...&gt;...&gt;</code> for all <code>Values...</code> parameter packs sent by <code>prologue</code>.</p></li>
<li><p>Otherwise, if <code>executor&lt;E&gt;</code> is true and <code>executor_shape_t&lt;E&gt;</code> is <code>S</code>, returns a sender object <code>s</code> whose implementation-defined type <code>R</code> satisfies <code>sender_to&lt;R, many-receiver&gt;</code>. <code>execution::connect(s,r)</code> returns an object <code>o</code> whose implementation-defined type satisfies <code>operation_state</code>.</p>
<ul>
<li><p>Let <code>values...</code> be a parameter pack of values sent by <code>prologue</code>. <code>execution::start(o)</code> calls <code>execution::bulk_execute(executor, call-set-value, shape)</code>.</p>
<p>Where <code>call-set-value</code> is an implementation-defined function object whose call operator is equivalent to</p>
<pre><code>void operator()(executor_index_t&lt;E&gt; idx) {
  execution::set_value(r, idx, values...)`;
}</code></pre></li>
<li><p>Otherwise, let <code>error</code> be an error sent by <code>prologue</code>. <code>execution::start(o)</code> calls <code>execution::set_error(move(r), error)</code>.</p></li>
<li><p>Otherwise, <code>execution::start(o)</code> calls <code>execution::set_done(move(r))</code>.</p></li>
</ul></li>
<li><p>Otherwise, <code>execution::bulk_schedule(executor, shape, prologue)</code> is ill-formed.</p></li>
</ul>
</section>
</section>
<section id="supporting-definitions" data-number="4">
<h1 data-number="4"><span class="header-section-number">4</span> Supporting Definitions</h1>
<p>The <code>receiver</code> concept defined in P0443r13 (Section 2.2.4) specifies that:</p>
<blockquote>
<p>exactly one of the receiver’s completion-signal operations shall complete non-exceptionally before the receiver is destroyed.</p>
</blockquote>
<p>In the bulk case, <code>set_value</code> may be called and completed many times. Therefore, we suggest introducing a corresponding <code>many_receiver_of</code> concept that explicitly addresses the case where <code>set_value</code> is called many times. Introducing such a concept would help make the specification of the sender returned by <code>bulk_schedule</code> more precise.</p>
<section id="concept-many_receiver_of" data-number="4.1">
<h2 data-number="4.1"><span class="header-section-number">4.1</span> Concept <code>many_receiver_of</code></h2>
<p>A many receiver represents the continuation of possibly many asynchronous operations.</p>
<pre><code>template&lt;class R, class... Args&gt;
  concept many_receiver_of =
    receiver&lt;R&gt; &amp;&amp;
    requires(remove_cvref_t&lt;R&gt;&amp; r, Args... args) {
      execution::set_value(r, (Args) args...);
    };</code></pre>
<p>The many receiver’s signal operations have semantic requirements that are collectively known as the <em>many receiver contract</em>, described below:</p>
<ul>
<li>None of a many receiver’s signal operations shall be invoked before <code>execution::start</code> has been called on the operation state object that was returned by <code>execution::connect</code> to connect that many receiver to a sender.</li>
<li>Once <code>execution::start</code> has been called on the operation state object, either:
<ul>
<li>All calls to <code>execution::set_value</code> on that many receiver shall complete non-exceptionally before the many receiver is destroyed, or</li>
<li>Exactly one call to <code>execution::set_error</code> or <code>execution::set_done</code> on that receiver shall complete non-exceptionally before the many receiver is destroyed.</li>
</ul></li>
<li>If any call to <code>execution::set_value</code> exits with an exception, it is still valid to call <code>execution::set_error</code> or <code>execution::set_done</code> on the receiver.</li>
</ul>
</section>
<section id="definitions-of-execution" data-number="4.2">
<h2 data-number="4.2"><span class="header-section-number">4.2</span> Definitions of execution</h2>
<p>An editorial note in <a href="http://wg21.link/P0443r13#executionexecute">P0334r13, Section 2.2.3.4</a> says that:</p>
<blockquote>
<p>We should probably define what “execute the function object F on the executor E” means more carefully.</p>
</blockquote>
<p>We suggest the following definition:</p>
<p>An executor <em>executes</em> an expression by scheduling the creation of an execution agent on which the expression executes. Invocable expressions are invoked by that execution agent. Execution of expressions that are not invocable is executor-defined.</p>
<p>Furthermore, we suggest adding the analogous definitions for bulk execution:</p>
<p>A <em>group of execution agents</em> created in bulk has a <em>shape</em>. Execution agents within a group are identified by <em>indices</em>, whose unique values are the set of contiguous indices spanned by the group’s shape.</p>
<p>An executor <em>bulk executes</em> an expression by scheduling the creation of a group of execution agents on which the expression executes in bulk. Invocable expressions are invoked with each execution agent’s index. Bulk execution of expressions that are not invocables is executor-defined.</p>
</section>
</section>
<section id="discussion" data-number="5">
<h1 data-number="5"><span class="header-section-number">5</span> Discussion</h1>
<p>The preceding sections contain the entirety of our proposed corrections and additions to <a href="http://wg21.link/p0443r13">P0443r13</a>. This section provides some additional background explanation and highlights some additional proposals that others may wish to consider separately.</p>
<section id="design-of-the-bulk-interface" data-number="5.1">
<h2 data-number="5.1"><span class="header-section-number">5.1</span> Design of the bulk interface</h2>
<p>This proposal positions <code>bulk_execute</code> as the direct analogue of <code>execute</code>. Both are low-level interfaces for creating execution and are necessary to expose platform-level work creation interfaces, which may be implemented outside the standard library. Furthermore, individual executor types may provide important platform-provided forward progress guarantees, such as a guarantee of mutual concurrency among agents.</p>
<p>While the default implementation of the <code>bulk_execute</code> customization point decays to a loop around <code>execute</code> in the absence of an executor-provided method, the <code>bulk_execute</code> operation is semantically distinct from a loop. Every loop construct in the standard is either explicitly sequential or permitted to fall back to a sequential equivalent at the sole discretion of the implementation. In contrast, executors may be used with <code>bulk_execute</code> to guarantee execution semantics that have no lowering onto sequential execution. For example, an executor whose <code>bulk_execute</code> method guarantees that all its created agents are concurrent with each other has no sequential equivalent.</p>
</section>
<section id="execution-policies" data-number="5.2">
<h2 data-number="5.2"><span class="header-section-number">5.2</span> Execution policies</h2>
<p>As in all prior revisions of P0443, the <code>bulk_execute</code> interface we propose does not include an execution policy argument. The use of execution policies in <code>bulk_execute</code> would be fundamentally inconsistent with their use throughout the rest of the library.</p>
<p>Execution policies were designed as a mechanism for customizing the execution of algorithms in the standard library in a way that could support the broadest possible range of architectures (see <a href="http://wg21.link/N3554">N3554</a>). As designed, they are suitable for customizing operations that can optionally change execution semantics (e.g., parallel execution in multiple threads). They are not, however, suitable for customizing low-level interfaces such as <code>bulk_execute</code> where mandatory execution semantics have already been specified in the form of an executor.</p>
<p>For every invocation of an algorithm with an execution policy, it is valid to replace the policy specified in the call with <code>execution::seq</code> without changing the meaning of the program. Similarly, conforming implementations are granted the freedom to fall back to sequential execution, regardless of the policy specified. This cannot be done with <code>bulk_execute</code> if the executor provides guarantees (e.g., non-blocking execution or concurrent forward progress) inconsistent with sequential execution in the calling thread.</p>
<p>The use of execution policies in the library is also designed to support a variety of vendor-supplied execution policies. Providing such vendor-specific policies to <code>bulk_execute</code> would typically have no meaning unless the executor is also a vendor-specific executor specifically designed to recognize that policy. In this case, all information provided by the policy could have been provided via the executor itself, making the policy parameter unnecessary. Once the executor semantics have been customized via the property-based <code>require</code> mechanism, any semantics implied by a policy are at best redundant and at worst contradictory.</p>
</section>
<section id="default-implementation-of-bulk_execute" data-number="5.3">
<h2 data-number="5.3"><span class="header-section-number">5.3</span> Default implementation of <code>bulk_execute</code></h2>
<p>We follow the existing practice in P0443 and specify a default implementation for the <code>bulk_execute</code> customization point when the executor does not provide a corresponding method. This default implementation calls the <code>execute</code> customization point in a loop. We recommend this over the alternative of calling <code>execute</code> with an invocable containing a loop, since the latter never creates parallelism amongst the bulk agents and thus creates significant risk of latent data races that manifest only when a non-default implementation is used.</p>
<p>Both <code>execute</code>, and by extension <code>bulk_execute</code>, allow non-copyable invocable types. This manifests in the third bullet point of the specification of <code>bulk_execute</code>, which has two cases. The first case opportunistically creates copies of the user’s invocable when it is possible to do so. Each agent created by the executor receives one of these copies. Otherwise, if the invocable is not copyable, each agent receives a reference to the invocable instead of a copy. This policy was chosen to ensure that invocables containing non-copyable, non-moveable types (e.g., synchronization objects) are still usable with <code>bulk_execute</code>. The caller of <code>execute</code> and/or <code>bulk_execute</code> must ensure that a non-copyable, non-moveable invocable outlives the group of agents that invokes it and that overlapping invocations do not create data races.</p>
</section>
<section id="additional-convenience-overloads" data-number="5.4">
<h2 data-number="5.4"><span class="header-section-number">5.4</span> Additional convenience overloads</h2>
<p>The <code>bulk_schedule</code> interface may be marginally more convenient if an additional overload is provided without a prologue sender:</p>
<div class="sourceCode" id="cb14"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb14-1"><a href="#cb14-1"></a>    sender <span class="kw">auto</span> bulk_schedule(executor <span class="kw">auto</span> ex,</span>
<span id="cb14-2"><a href="#cb14-2"></a>                              <span class="dt">executor_shape_t</span>&lt;<span class="kw">decltype</span>(ex)&gt; shape);</span></code></pre></div>
<p>While an equivalent result can already be achieved by passing a suitable “empty” prologue sender through the interface we have proposed, this overload would be more convenient for the user of the interface.</p>
<p>It may also be worth considering adding an overload of <code>schedule</code> that accepts a prologue sender, mirroring the <code>bulk_schedule</code> interface we have proposed:</p>
<div class="sourceCode" id="cb15"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb15-1"><a href="#cb15-1"></a>    sender <span class="kw">auto</span> schedule(executor <span class="kw">auto</span> ex,</span>
<span id="cb15-2"><a href="#cb15-2"></a>                         sender <span class="kw">auto</span> prologue);</span></code></pre></div>
<p>Neither of these changes is essential, but adding these options to the existing overloads for <code>schedule</code> and <code>bulk_schedule</code> in P0443r13 and our proposal above, respectively, would make the scheduling interface more convenient and more predictable.</p>
</section>
<section id="delivering-submission-errors" data-number="5.5">
<h2 data-number="5.5"><span class="header-section-number">5.5</span> Delivering Submission Errors</h2>
<p>Our specification defines the behavior of <code>bulk_schedule</code> in terms of calls to <code>bulk_execute</code>. We believe this is a design decision of fundamental importance, since it encapsulates the details of submission in a single place. Moreover, it guarantees semantic equivalence between eager and lazy mechanisms for work submission. It also implies that errors that result in the process of work submission (e.g., in the implementation of <code>bulk_execute</code>) should be delivered through the usual mechanism of exceptions rather than via calls to <code>set_error</code>. This is true regardless of whether work is submitted via <code>bulk_execute</code> directly or scheduled for execution via <code>bulk_schedule</code>.</p>
<p>If the ability to deliver errors during submission via <code>set_error</code> is desired, it can be addressed separately from this proposal. For example, a candidate solution was provided in <a href="http://wg21.link/p1660r0">P1660, Section 5.2</a>. That paper recommended allowing the caller of <code>execute</code> or <code>bulk_execute</code> to control the error delivery channel by providing either an invocable—resulting in the use of exceptions—or a receiver—resulting in delivery via <code>set_error</code>.</p>
</section>
</section>
<section id="refs" class="unnumbered" data-number="">
<h1 class="unnumbered" data-number="">References</h1>
<div id="refs" class="references hanging-indent" role="doc-bibliography">
<div id="ref-P1993">
<p>Hoberock, Jared. 2020. “Restore Shared State to bulk_execute.” <a href="http://wg21.link/p1993r1">http://wg21.link/p1993r1</a>.</p>
</div>
<div id="ref-N4406">
<p>Hoberock, Jared, Michael Garland, and Olivier Girioux. 2015. “Parallel Algorithms Need Executors.” <a href="http://wg21.link/N4406">http://wg21.link/N4406</a>.</p>
</div>
<div id="ref-P0443">
<p>Hoberock, J., M. Garland, C. Kohlhoff, C. Mysen, C. Edwards, G. Brown, D. Hollman, et al. 2020. “A Unified Executors Proposal for C++.” <a href="http://wg21.link/p0443r13">http://wg21.link/p0443r13</a>.</p>
</div>
<div id="ref-P1660">
<p>Hoberock, J., M. Garland, B. Lelbach, M. Dominiak, E. Niebler, K. Shoop, L. Baker, L. Howes, D. Hollman, and G. Brown. 2019. “A Compromise Executor Design Sketch.” <a href="http://wg21.link/p1660r0">http://wg21.link/p1660r0</a>.</p>
</div>
<div id="ref-N3554">
<p>Hoberock, J., J. Marathe, M. Garland, O. Giroux, V. Grover, A. Laksberg, H. Sutter, and A. Robison. 2013. “A Parallel Algorithms Library.” <a href="http://wg21.link/N3554">http://wg21.link/N3554</a>.</p>
</div>
</div>
<hr />
</section>
<section id="appendix" class="unnumbered" data-number="">
<h1 class="unnumbered" data-number="">Appendix: Implementation of <code>bulk_schedule</code></h1>
<p>[<em>Editorial note:</em> Append this reference implementation for the default case of <code>bulk_schedule</code> to P0443 as Appendix 2.10. –<em>end editorial note</em>]</p>
<div class="sourceCode" id="cb16"><pre class="sourceCode cpp"><code class="sourceCode cpp"><span id="cb16-1"><a href="#cb16-1"></a><span class="kw">template</span>&lt;<span class="kw">class</span> S, <span class="kw">class</span> E, <span class="kw">class</span> P, <span class="kw">class</span> R&gt;</span>
<span id="cb16-2"><a href="#cb16-2"></a><span class="kw">struct</span> fan_out_receiver {</span>
<span id="cb16-3"><a href="#cb16-3"></a><span class="kw">private</span>:</span>
<span id="cb16-4"><a href="#cb16-4"></a>  <span class="kw">using</span> <span class="dt">variant_of_tuples_type</span> = <span class="kw">typename</span> sender_traits&lt;S&gt;::<span class="kw">template</span> value_types&lt;tuple,variant&gt;;</span>
<span id="cb16-5"><a href="#cb16-5"></a></span>
<span id="cb16-6"><a href="#cb16-6"></a>  optional&lt;<span class="dt">variant_of_tuples_type</span>&gt; <span class="va">maybe_variant_of_tuples_</span>;</span>
<span id="cb16-7"><a href="#cb16-7"></a>  E <span class="va">executor_</span>;</span>
<span id="cb16-8"><a href="#cb16-8"></a>  <span class="dt">executor_shape_t</span>&lt;E&gt; <span class="va">shape_</span>;</span>
<span id="cb16-9"><a href="#cb16-9"></a>  R <span class="va">receiver_</span>;</span>
<span id="cb16-10"><a href="#cb16-10"></a></span>
<span id="cb16-11"><a href="#cb16-11"></a>  <span class="kw">template</span>&lt;<span class="kw">class</span>... Args, <span class="dt">size_t</span>... I&gt;</span>
<span id="cb16-12"><a href="#cb16-12"></a>  <span class="dt">void</span> set_value_impl(Args&amp;&amp;... args, index_sequence&lt;I...&gt;) {</span>
<span id="cb16-13"><a href="#cb16-13"></a>    <span class="va">maybe_variant_of_tuples_</span>.emplace(make_tuple(forward&lt;Args&gt;(args)...));</span>
<span id="cb16-14"><a href="#cb16-14"></a></span>
<span id="cb16-15"><a href="#cb16-15"></a>    visit([&amp;<span class="va">executor_</span>, &amp;<span class="va">receiver_</span>](tuple&lt;Args...&gt;&amp; args) {</span>
<span id="cb16-16"><a href="#cb16-16"></a>      execution::bulk_execute(<span class="va">executor_</span>, [&amp;](<span class="dt">executor_index_t</span>&lt;E&gt; idx) {</span>
<span id="cb16-17"><a href="#cb16-17"></a>        execution::set_value(<span class="va">receiver_</span>, idx, get&lt;I&gt;(args)...);</span>
<span id="cb16-18"><a href="#cb16-18"></a>      };</span>
<span id="cb16-19"><a href="#cb16-19"></a>    },</span>
<span id="cb16-20"><a href="#cb16-20"></a>    *<span class="va">maybe_variant_of_tuples_</span>);</span>
<span id="cb16-21"><a href="#cb16-21"></a>  }</span>
<span id="cb16-22"><a href="#cb16-22"></a></span>
<span id="cb16-23"><a href="#cb16-23"></a><span class="kw">public</span>:</span>
<span id="cb16-24"><a href="#cb16-24"></a>  fan_out_receiver(<span class="at">const</span> E&amp; executor, <span class="dt">executor_shape_t</span>&lt;E&gt; shape, R&amp;&amp; receiver)</span>
<span id="cb16-25"><a href="#cb16-25"></a>    : <span class="va">maybe_variant_of_tuples_</span>{},</span>
<span id="cb16-26"><a href="#cb16-26"></a>      <span class="va">executor_</span>{ex},</span>
<span id="cb16-27"><a href="#cb16-27"></a>      <span class="va">shape_</span>{shape},</span>
<span id="cb16-28"><a href="#cb16-28"></a>      <span class="va">receiver_</span>{move(receiver)}</span>
<span id="cb16-29"><a href="#cb16-29"></a>  {}</span>
<span id="cb16-30"><a href="#cb16-30"></a></span>
<span id="cb16-31"><a href="#cb16-31"></a>  fan_out_receiver(fan_out_receiver&amp;&amp;) = <span class="cf">default</span>;</span>
<span id="cb16-32"><a href="#cb16-32"></a></span>
<span id="cb16-33"><a href="#cb16-33"></a>  <span class="kw">template</span>&lt;<span class="kw">class</span> E&gt;</span>
<span id="cb16-34"><a href="#cb16-34"></a>  <span class="dt">void</span> set_error(E&amp;&amp; e) &amp;&amp; {</span>
<span id="cb16-35"><a href="#cb16-35"></a>    execution::set_error(move(<span class="va">receiver_</span>), forward&lt;E&gt;(e));</span>
<span id="cb16-36"><a href="#cb16-36"></a>  }</span>
<span id="cb16-37"><a href="#cb16-37"></a></span>
<span id="cb16-38"><a href="#cb16-38"></a>  <span class="dt">void</span> set_done() &amp;&amp; {</span>
<span id="cb16-39"><a href="#cb16-39"></a>    execution::set_done(move(<span class="va">receiver_</span>));</span>
<span id="cb16-40"><a href="#cb16-40"></a>  }</span>
<span id="cb16-41"><a href="#cb16-41"></a></span>
<span id="cb16-42"><a href="#cb16-42"></a>  <span class="kw">template</span>&lt;<span class="kw">class</span>... Args&gt;</span>
<span id="cb16-43"><a href="#cb16-43"></a>    <span class="kw">requires</span> many_receiver_of&lt;R, <span class="dt">executor_shape_t</span>&lt;E&gt;, <span class="dt">remove_cvref_t</span>&lt;Args&gt;&amp;...&gt;</span>
<span id="cb16-44"><a href="#cb16-44"></a>  <span class="dt">void</span> set_value(Args&amp;&amp;... args) &amp;&amp; {</span>
<span id="cb16-45"><a href="#cb16-45"></a>    set_value_impl(forward&lt;Args&gt;(args)..., index_sequence_for&lt;Args...&gt;{});</span>
<span id="cb16-46"><a href="#cb16-46"></a>  }</span>
<span id="cb16-47"><a href="#cb16-47"></a>};</span>
<span id="cb16-48"><a href="#cb16-48"></a></span>
<span id="cb16-49"><a href="#cb16-49"></a><span class="kw">template</span>&lt;<span class="kw">class</span> E, <span class="kw">class</span> S, <span class="kw">class</span> P&gt;</span>
<span id="cb16-50"><a href="#cb16-50"></a><span class="kw">struct</span> as_bulk_sender {</span>
<span id="cb16-51"><a href="#cb16-51"></a><span class="kw">private</span>:</span>
<span id="cb16-52"><a href="#cb16-52"></a>  E <span class="va">ex_</span>;</span>
<span id="cb16-53"><a href="#cb16-53"></a>  S <span class="va">shape_</span>;</span>
<span id="cb16-54"><a href="#cb16-54"></a>  P <span class="va">prologue_</span>;</span>
<span id="cb16-55"><a href="#cb16-55"></a><span class="kw">public</span>:</span>
<span id="cb16-56"><a href="#cb16-56"></a>  <span class="kw">template</span>&lt;<span class="kw">template</span>&lt;<span class="kw">class</span>...&gt; <span class="kw">class</span> Tuple, <span class="kw">template</span>&lt;<span class="kw">class</span>...&gt; <span class="kw">class</span> Variant&gt;</span>
<span id="cb16-57"><a href="#cb16-57"></a>  <span class="kw">using</span> value_types = <span class="kw">typename</span> sender_traits&lt;S&gt;::<span class="kw">template</span> value_types&lt;Tuple, Variant&gt;;</span>
<span id="cb16-58"><a href="#cb16-58"></a></span>
<span id="cb16-59"><a href="#cb16-59"></a>  <span class="kw">template</span>&lt;<span class="kw">template</span>&lt;<span class="kw">class</span>...&gt; <span class="kw">class</span> Variant&gt;</span>
<span id="cb16-60"><a href="#cb16-60"></a>  <span class="kw">using</span> error_types = <span class="kw">typename</span> sender_traits&lt;S&gt;::<span class="kw">template</span> error_types&lt;Variant&gt;;</span>
<span id="cb16-61"><a href="#cb16-61"></a></span>
<span id="cb16-62"><a href="#cb16-62"></a>  <span class="at">static</span> <span class="kw">constexpr</span> <span class="dt">bool</span> sends_done = sender_traits&lt;S&gt;::sends_done;</span>
<span id="cb16-63"><a href="#cb16-63"></a></span>
<span id="cb16-64"><a href="#cb16-64"></a>  <span class="kw">template</span>&lt;<span class="kw">class</span> Sender&gt;</span>
<span id="cb16-65"><a href="#cb16-65"></a>  as_bulk_sender(<span class="at">const</span> Executor&amp; ex, Sender&amp;&amp; prologue, <span class="at">const</span> Shape&amp; shape)</span>
<span id="cb16-66"><a href="#cb16-66"></a>    : <span class="va">ex_</span>(ex),</span>
<span id="cb16-67"><a href="#cb16-67"></a>      <span class="va">prologue_</span>(forward&lt;Sender&gt;(prologue)),</span>
<span id="cb16-68"><a href="#cb16-68"></a>      <span class="va">shape_</span>(shape)</span>
<span id="cb16-69"><a href="#cb16-69"></a>  {}</span>
<span id="cb16-70"><a href="#cb16-70"></a></span>
<span id="cb16-71"><a href="#cb16-71"></a>  <span class="kw">template</span>&lt;<span class="kw">class</span> R&gt;</span>
<span id="cb16-72"><a href="#cb16-72"></a>    <span class="kw">requires</span> many_receiver_of&lt;R, <span class="dt">executor_index_t</span>&lt;E&gt;, ...&gt;</span>
<span id="cb16-73"><a href="#cb16-73"></a>  <span class="kw">auto</span> <span class="fu">connect</span>(R&amp;&amp; r) &amp;&amp;</span>
<span id="cb16-74"><a href="#cb16-74"></a>  {</span>
<span id="cb16-75"><a href="#cb16-75"></a>    <span class="cf">return</span> execution::<span class="fu">connect</span>(move(<span class="va">prologue_</span>), fan_out_receiver&lt;S&gt;{<span class="va">ex_</span>, <span class="va">shape_</span>, forward&lt;ManyReceiver&gt;(r)});</span>
<span id="cb16-76"><a href="#cb16-76"></a>  }</span>
<span id="cb16-77"><a href="#cb16-77"></a>};</span></code></pre></div>
</section>
</body>
</html>
