<!DOCTYPE html>

<html lang="en">

<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
    <meta name="apple-mobile-web-app-capable" content="yes">
    <meta name="apple-mobile-web-app-status-bar-style" content="black">
    <meta name="mobile-web-app-capable" content="yes">
    <title>
        barrier token-less split arrive/wait
    </title>
    <link rel="icon" type="image/ico" href="https://isocpp.org/favicon.ico">

    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha256-916EbMg70RQy9LHiGkXzG8hSg9EdNy97GazNG/aiY1w=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/ionicons/2.0.1/css/ionicons.min.css" integrity="sha256-3iu9jgsy9TpTwXKb7bNQzqWekRX7pPK+2OLj3R922fo=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/octicons/3.5.0/octicons.min.css" integrity="sha256-QiWfLIsCT02Sdwkogf6YMiQlj4NE84MKkzEMkZnMGdg=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.5.1/themes/prism.min.css" integrity="sha256-vtR0hSWRc3Tb26iuN2oZHt3KRUomwTufNIf5/4oeCyg=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@hackmd/emojify.js@2.1.0/dist/css/basic/emojify.min.css" integrity="sha256-UOrvMOsSDSrW6szVLe8ZDZezBxh5IoIfgTwdNDgTjiU=" crossorigin="anonymous" />
    <style>
        @import url(https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,500,500i|Source+Code+Pro:300,400,500|Source+Sans+Pro:300,300i,400,400i,600,600i|Source+Serif+Pro&subset=latin-ext);.hljs{display:block;background:#fff;padding:.5em;color:#333;overflow-x:auto}.hljs-comment,.hljs-meta{color:#969896}.hljs-emphasis,.hljs-quote,.hljs-string,.hljs-strong,.hljs-template-variable,.hljs-variable{color:#df5000}.hljs-keyword,.hljs-selector-tag,.hljs-type{color:#a71d5d}.hljs-attribute,.hljs-bullet,.hljs-literal,.hljs-number,.hljs-symbol{color:#0086b3}.hljs-built_in,.hljs-builtin-name{color:#005cc5}.hljs-name,.hljs-section{color:#63a35c}.hljs-tag{color:#333}.hljs-attr,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-id,.hljs-selector-pseudo,.hljs-title{color:#795da3}.hljs-addition{color:#55a532;background-color:#eaffea}.hljs-deletion{color:#bd2c00;background-color:#ffecec}.hljs-link{text-decoration:underline}.markdown-body{font-size:16px;line-height:1.5;word-wrap:break-word}.markdown-body:after,.markdown-body:before{display:table;content:""}.markdown-body:after{clear:both}.markdown-body>:first-child{margin-top:0!important}.markdown-body>:last-child{margin-bottom:0!important}.markdown-body a:not([href]){color:inherit;text-decoration:none}.markdown-body .absent{color:#c00}.markdown-body .anchor{float:left;padding-right:4px;margin-left:-20px;line-height:1}.markdown-body .anchor:focus{outline:none}.markdown-body blockquote,.markdown-body dl,.markdown-body ol,.markdown-body p,.markdown-body pre,.markdown-body table,.markdown-body ul{margin-top:0;margin-bottom:16px}.markdown-body hr{height:.25em;padding:0;margin:24px 0;background-color:#e7e7e7;border:0}.markdown-body blockquote{font-size:16px;padding:0 1em;color:#777;border-left:.25em solid #ddd}.markdown-body blockquote>:first-child{margin-top:0}.markdown-body blockquote>:last-child{margin-bottom:0}.markdown-body kbd,.popover kbd{display:inline-block;padding:3px 5px;font-size:11px;line-height:10px;color:#555;vertical-align:middle;background-color:#fcfcfc;border:1px solid #ccc;border-bottom-color:#bbb;border-radius:3px;box-shadow:inset 0 -1px 0 #bbb}.markdown-body .loweralpha{list-style-type:lower-alpha}.markdown-body h1,.markdown-body h2,.markdown-body h3,.markdown-body h4,.markdown-body h5,.markdown-body h6{margin-top:24px;margin-bottom:16px;font-weight:600;line-height:1.25}.markdown-body h1 .octicon-link,.markdown-body h2 .octicon-link,.markdown-body h3 .octicon-link,.markdown-body h4 .octicon-link,.markdown-body h5 .octicon-link,.markdown-body h6 .octicon-link{color:#000;vertical-align:middle;visibility:hidden}.markdown-body h1:hover .anchor,.markdown-body h2:hover .anchor,.markdown-body h3:hover .anchor,.markdown-body h4:hover .anchor,.markdown-body h5:hover .anchor,.markdown-body h6:hover .anchor{text-decoration:none}.markdown-body h1:hover .anchor .octicon-link,.markdown-body h2:hover .anchor .octicon-link,.markdown-body h3:hover .anchor .octicon-link,.markdown-body h4:hover .anchor .octicon-link,.markdown-body h5:hover .anchor .octicon-link,.markdown-body h6:hover .anchor .octicon-link{visibility:visible}.markdown-body h1 code,.markdown-body h1 tt,.markdown-body h2 code,.markdown-body h2 tt,.markdown-body h3 code,.markdown-body h3 tt,.markdown-body h4 code,.markdown-body h4 tt,.markdown-body h5 code,.markdown-body h5 tt,.markdown-body h6 code,.markdown-body h6 tt{font-size:inherit}.markdown-body h1{font-size:2em}.markdown-body h1,.markdown-body h2{padding-bottom:.3em;border-bottom:1px solid #eee}.markdown-body h2{font-size:1.5em}.markdown-body h3{font-size:1.25em}.markdown-body h4{font-size:1em}.markdown-body h5{font-size:.875em}.markdown-body h6{font-size:.85em;color:#777}.markdown-body ol,.markdown-body ul{padding-left:2em}.markdown-body ol.no-list,.markdown-body ul.no-list{padding:0;list-style-type:none}.markdown-body ol ol,.markdown-body ol ul,.markdown-body ul ol,.markdown-body ul ul{margin-top:0;margin-bottom:0}.markdown-body li>p{margin-top:16px}.markdown-body li+li{padding-top:.25em}.markdown-body dl{padding:0}.markdown-body dl dt{padding:0;margin-top:16px;font-size:1em;font-style:italic;font-weight:700}.markdown-body dl dd{padding:0 16px;margin-bottom:16px}.markdown-body table{display:block;width:100%;overflow:auto;word-break:normal;word-break:keep-all}.markdown-body table th{font-weight:700}.markdown-body table td,.markdown-body table th{padding:6px 13px;border:1px solid #ddd}.markdown-body table tr{background-color:#fff;border-top:1px solid #ccc}.markdown-body table tr:nth-child(2n){background-color:#f8f8f8}.markdown-body img{max-width:100%;box-sizing:content-box;background-color:#fff}.markdown-body img[align=right]{padding-left:20px}.markdown-body img[align=left]{padding-right:20px}.markdown-body .emoji{max-width:none;vertical-align:text-top;background-color:transparent}.markdown-body span.frame{display:block;overflow:hidden}.markdown-body span.frame>span{display:block;float:left;width:auto;padding:7px;margin:13px 0 0;overflow:hidden;border:1px solid #ddd}.markdown-body span.frame span img{display:block;float:left}.markdown-body span.frame span span{display:block;padding:5px 0 0;clear:both;color:#333}.markdown-body span.align-center{display:block;overflow:hidden;clear:both}.markdown-body span.align-center>span{display:block;margin:13px auto 0;overflow:hidden;text-align:center}.markdown-body span.align-center span img{margin:0 auto;text-align:center}.markdown-body span.align-right{display:block;overflow:hidden;clear:both}.markdown-body span.align-right>span{display:block;margin:13px 0 0;overflow:hidden;text-align:right}.markdown-body span.align-right span img{margin:0;text-align:right}.markdown-body span.float-left{display:block;float:left;margin-right:13px;overflow:hidden}.markdown-body span.float-left span{margin:13px 0 0}.markdown-body span.float-right{display:block;float:right;margin-left:13px;overflow:hidden}.markdown-body span.float-right>span{display:block;margin:13px auto 0;overflow:hidden;text-align:right}.markdown-body code,.markdown-body tt{padding:0;padding-top:.2em;padding-bottom:.2em;margin:0;font-size:85%;background-color:rgba(0,0,0,.04);border-radius:3px}.markdown-body code:after,.markdown-body code:before,.markdown-body tt:after,.markdown-body tt:before{letter-spacing:-.2em;content:"\00a0"}.markdown-body code br,.markdown-body tt br{display:none}.markdown-body del code{text-decoration:inherit}.markdown-body pre{word-wrap:normal}.markdown-body pre>code{padding:0;margin:0;font-size:100%;word-break:normal;white-space:pre;background:transparent;border:0}.markdown-body .highlight{margin-bottom:16px}.markdown-body .highlight pre{margin-bottom:0;word-break:normal}.markdown-body .highlight pre,.markdown-body pre{padding:16px;overflow:auto;font-size:85%;line-height:1.45;background-color:#f7f7f7;border-radius:3px}.markdown-body pre code,.markdown-body pre tt{display:inline;max-width:auto;padding:0;margin:0;overflow:visible;line-height:inherit;word-wrap:normal;background-color:transparent;border:0}.markdown-body pre code:after,.markdown-body pre code:before,.markdown-body pre tt:after,.markdown-body pre tt:before{content:normal}.markdown-body .csv-data td,.markdown-body .csv-data th{padding:5px;overflow:hidden;font-size:12px;line-height:1;text-align:left;white-space:nowrap}.markdown-body .csv-data .blob-line-num{padding:10px 8px 9px;text-align:right;background:#fff;border:0}.markdown-body .csv-data tr{border-top:0}.markdown-body .csv-data th{font-weight:700;background:#f8f8f8;border-top:0}.news .alert .markdown-body blockquote{padding:0 0 0 40px;border:0 none}.activity-tab .news .alert .commits,.activity-tab .news .markdown-body blockquote{padding-left:0}.task-list-item{list-style-type:none}.task-list-item label{font-weight:400}.task-list-item.enabled label{cursor:pointer}.task-list-item+.task-list-item{margin-top:3px}.task-list-item-checkbox{float:left;margin:.31em 0 .2em -1.3em!important;vertical-align:middle;cursor:default!important}.markdown-body{padding-top:40px;padding-bottom:40px;max-width:758px;overflow:visible!important;position:relative}.markdown-body .emoji{vertical-align:top}.markdown-body pre{border:inherit!important}.markdown-body code{color:inherit!important}.markdown-body pre code .wrapper{display:-moz-inline-flex;display:-ms-inline-flex;display:-o-inline-flex;display:inline-flex}.markdown-body pre code .gutter{float:left;overflow:hidden;-webkit-user-select:none;user-select:none}.markdown-body pre code .gutter.linenumber{text-align:right;position:relative;display:inline-block;cursor:default;z-index:4;padding:0 8px 0 0;min-width:20px;box-sizing:content-box;color:#afafaf!important;border-right:3px solid #6ce26c!important}.markdown-body pre code .gutter.linenumber>span:before{content:attr(data-linenumber)}.markdown-body pre code .code{float:left;margin:0 0 0 16px}.markdown-body .gist .line-numbers{border-left:none;border-top:none;border-bottom:none}.markdown-body .gist .line-data{border:none}.markdown-body .gist table{border-spacing:0;border-collapse:inherit!important}.markdown-body code[data-gist-id]{background:none;padding:0}.markdown-body code[data-gist-id]:after,.markdown-body code[data-gist-id]:before{content:""}.markdown-body code[data-gist-id] .blob-num{border:unset}.markdown-body code[data-gist-id] table{overflow:unset;margin-bottom:unset}.markdown-body code[data-gist-id] table tr{background:unset}.markdown-body[dir=rtl] pre{direction:ltr}.markdown-body[dir=rtl] code{direction:ltr;unicode-bidi:embed}.markdown-body .alert>p:last-child{margin-bottom:0}.markdown-body pre.abc,.markdown-body pre.flow-chart,.markdown-body pre.graphviz,.markdown-body pre.mermaid,.markdown-body pre.sequence-diagram,.markdown-body pre.vega{text-align:center;background-color:inherit;border-radius:0;white-space:inherit;overflow:visible}.markdown-body pre.abc>code,.markdown-body pre.flow-chart>code,.markdown-body pre.graphviz>code,.markdown-body pre.mermaid>code,.markdown-body pre.sequence-diagram>code,.markdown-body pre.vega>code{text-align:left}.markdown-body pre.abc>svg,.markdown-body pre.flow-chart>svg,.markdown-body pre.graphviz>svg,.markdown-body pre.mermaid>svg,.markdown-body pre.sequence-diagram>svg,.markdown-body pre.vega>svg{max-width:100%;height:100%}.markdown-body pre>code.wrap{white-space:pre-wrap;white-space:-moz-pre-wrap;white-space:-pre-wrap;white-space:-o-pre-wrap;word-wrap:break-word}.markdown-body .alert>p:last-child,.markdown-body .alert>ul:last-child{margin-bottom:0}.markdown-body summary{display:list-item}.markdown-body summary:focus{outline:none}.markdown-body details summary{cursor:pointer}.markdown-body details:not([open])>:not(summary){display:none}.markdown-body figure{margin:1em 40px}.markdown-body .mark,.markdown-body mark{background-color:#fff1a7}.vimeo,.youtube{cursor:pointer;display:table;text-align:center;background-position:50%;background-repeat:no-repeat;background-size:contain;background-color:#000;overflow:hidden}.vimeo,.youtube{position:relative;width:100%}.youtube{padding-bottom:56.25%}.vimeo img{width:100%;object-fit:contain;z-index:0}.youtube img{object-fit:cover;z-index:0}.vimeo iframe,.youtube iframe,.youtube img{width:100%;height:100%;position:absolute;top:0;left:0}.vimeo iframe,.youtube iframe{vertical-align:middle;z-index:1}.vimeo .icon,.youtube .icon{position:absolute;height:auto;width:auto;top:50%;left:50%;transform:translate(-50%,-50%);color:#fff;opacity:.3;transition:opacity .2s;z-index:0}.vimeo:hover .icon,.youtube:hover .icon{opacity:.6;transition:opacity .2s}.slideshare .inner,.speakerdeck .inner{position:relative;width:100%}.slideshare .inner iframe,.speakerdeck .inner iframe{position:absolute;top:0;bottom:0;left:0;right:0;width:100%;height:100%}.figma{display:table;position:relative;width:100%;padding-bottom:56.25%}.figma iframe{position:absolute;top:0;bottom:0;left:0;right:0;width:100%;height:100%;border:1px solid #eee}.markmap-container{height:300px}.markmap-container>svg{width:100%;height:100%}.MJX_Assistive_MathML{display:none}#MathJax_Message{z-index:1000!important}.ui-infobar{position:relative;z-index:2;max-width:760px;margin:25px auto -25px;color:#777}.toc .invisable-node{list-style-type:none}.ui-toc{position:fixed;bottom:20px;z-index:998}.ui-toc.both-mode{margin-left:8px}.ui-toc.both-mode .ui-toc-label{height:40px;padding:10px 4px;border-top-left-radius:0;border-bottom-left-radius:0}.ui-toc-label{background-color:#e6e6e6;border:none;color:#868686;transition:opacity .2s}.ui-toc .open .ui-toc-label{opacity:1;color:#fff;transition:opacity .2s}.ui-toc-label:focus{opacity:.3;background-color:#ccc;color:#000}.ui-toc-label:hover{opacity:1;background-color:#ccc;transition:opacity .2s}.ui-toc-dropdown{margin-top:20px;margin-bottom:20px;padding-left:10px;padding-right:10px;max-width:45vw;width:25vw;max-height:70vh;overflow:auto;text-align:inherit}.ui-toc-dropdown>.toc{max-height:calc(70vh - 100px);overflow:auto}.ui-toc-dropdown[dir=rtl] .nav{padding-right:0;letter-spacing:.0029em}.ui-toc-dropdown a{overflow:hidden;text-overflow:ellipsis;white-space:pre}.ui-toc-dropdown .nav>li>a{display:block;padding:4px 20px;font-size:13px;font-weight:500;color:#767676}.ui-toc-dropdown .nav>li:first-child:last-child>ul,.ui-toc-dropdown .toc.expand ul{display:block}.ui-toc-dropdown .nav>li>a:focus,.ui-toc-dropdown .nav>li>a:hover{padding-left:19px;color:#000;text-decoration:none;background-color:transparent;border-left:1px solid #000}.ui-toc-dropdown[dir=rtl] .nav>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav>li>a:hover{padding-right:19px;border-left:none;border-right:1px solid #000}.ui-toc-dropdown .nav>.active:focus>a,.ui-toc-dropdown .nav>.active:hover>a,.ui-toc-dropdown .nav>.active>a{padding-left:18px;font-weight:700;color:#000;background-color:transparent;border-left:2px solid #000}.ui-toc-dropdown[dir=rtl] .nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav>.active>a{padding-right:18px;border-left:none;border-right:2px solid #000}.ui-toc-dropdown .nav .nav{display:none;padding-bottom:10px}.ui-toc-dropdown .nav>.active>ul{display:block}.ui-toc-dropdown .nav .nav>li>a{padding-top:1px;padding-bottom:1px;padding-left:30px;font-size:12px;font-weight:400}.ui-toc-dropdown[dir=rtl] .nav .nav>li>a{padding-right:30px}.ui-toc-dropdown .nav .nav>li>ul>li>a{padding-top:1px;padding-bottom:1px;padding-left:40px;font-size:12px;font-weight:400}.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a{padding-right:40px}.ui-toc-dropdown .nav .nav>li>a:focus,.ui-toc-dropdown .nav .nav>li>a:hover{padding-left:29px}.ui-toc-dropdown[dir=rtl] .nav .nav>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav .nav>li>a:hover{padding-right:29px}.ui-toc-dropdown .nav .nav>li>ul>li>a:focus,.ui-toc-dropdown .nav .nav>li>ul>li>a:hover{padding-left:39px}.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a:focus,.ui-toc-dropdown[dir=rtl] .nav .nav>li>ul>li>a:hover{padding-right:39px}.ui-toc-dropdown .nav .nav>.active:focus>a,.ui-toc-dropdown .nav .nav>.active:hover>a,.ui-toc-dropdown .nav .nav>.active>a{padding-left:28px;font-weight:500}.ui-toc-dropdown[dir=rtl] .nav .nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>a{padding-right:28px}.ui-toc-dropdown .nav .nav>.active>.nav>.active:focus>a,.ui-toc-dropdown .nav .nav>.active>.nav>.active:hover>a,.ui-toc-dropdown .nav .nav>.active>.nav>.active>a{padding-left:38px;font-weight:500}.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active:focus>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active:hover>a,.ui-toc-dropdown[dir=rtl] .nav .nav>.active>.nav>.active>a{padding-right:38px}.markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,sans-serif}html[lang^=ja] .markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html[lang=zh-tw] .markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html[lang=zh-cn] .markdown-body{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}html .markdown-body[lang^=ja]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html .markdown-body[lang=zh-tw]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html .markdown-body[lang=zh-cn]{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica Neue,Helvetica,Roboto,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}html[lang^=ja] .ui-toc-dropdown{font-family:Source Sans Pro,Helvetica,Arial,Meiryo UI,MS PGothic,ＭＳ\ Ｐゴシック,sans-serif}html[lang=zh-tw] .ui-toc-dropdown{font-family:Source Sans Pro,Helvetica,Arial,Microsoft JhengHei UI,微軟正黑UI,sans-serif}html[lang=zh-cn] .ui-toc-dropdown{font-family:Source Sans Pro,Helvetica,Arial,Microsoft YaHei UI,微软雅黑UI,sans-serif}html .ui-toc-dropdown[lang^=ja]{font-family:Source Sans Pro,Helvetica,Arial,Meiryo UI,MS PGothic,ＭＳ\ Ｐゴシック,sans-serif}html .ui-toc-dropdown[lang=zh-tw]{font-family:Source Sans Pro,Helvetica,Arial,Microsoft JhengHei UI,微軟正黑UI,sans-serif}html .ui-toc-dropdown[lang=zh-cn]{font-family:Source Sans Pro,Helvetica,Arial,Microsoft YaHei UI,微软雅黑UI,sans-serif}.ui-affix-toc{position:fixed;top:0;max-width:15vw;max-height:70vh;overflow:auto}.back-to-top,.expand-toggle,.go-to-bottom{display:block;padding:4px 10px;margin-top:10px;margin-left:10px;font-size:12px;font-weight:500;color:#999}.back-to-top:focus,.back-to-top:hover,.expand-toggle:focus,.expand-toggle:hover,.go-to-bottom:focus,.go-to-bottom:hover{color:#563d7c;text-decoration:none}.back-to-top,.go-to-bottom{margin-top:0}.ui-user-icon{width:20px;height:20px;display:block;border-radius:50%;margin-top:2px;margin-bottom:2px;margin-right:5px;background-position:50%;background-repeat:no-repeat;background-size:cover}.ui-user-icon.small{width:18px;height:18px;display:inline-block;vertical-align:middle;margin:0 0 .2em}.ui-infobar>small>span{line-height:22px}.ui-infobar>small .dropdown{display:inline-block}.ui-infobar>small .dropdown a:focus,.ui-infobar>small .dropdown a:hover{text-decoration:none}.ui-more-info{color:#888;cursor:pointer;vertical-align:middle}.ui-more-info .fa{font-size:16px}.ui-connectedGithub,.ui-published-note{color:#888}.ui-connectedGithub{line-height:23px;white-space:nowrap}.ui-connectedGithub a.file-path{color:#888;text-decoration:none;padding-left:22px}.ui-connectedGithub a.file-path:active,.ui-connectedGithub a.file-path:hover{color:#888;text-decoration:underline}.ui-connectedGithub .fa{font-size:20px}.ui-published-note .fa{font-size:20px;vertical-align:top}.unselectable{-webkit-user-select:none;-o-user-select:none;user-select:none}.selectable{-webkit-user-select:text;-o-user-select:text;user-select:text}.inline-spoiler-section{cursor:pointer}.inline-spoiler-section .spoiler-text{border-radius:2px;background-color:#333}.inline-spoiler-section .spoiler-text>*{opacity:0}.inline-spoiler-section .spoiler-img{filter:blur(10px)}.inline-spoiler-section.raw{border-radius:2px;background-color:#333}.inline-spoiler-section.raw>*{opacity:0}.inline-spoiler-section.unveil{cursor:auto}.inline-spoiler-section.unveil .spoiler-text{background-color:rgba(51,51,51,.1)}.inline-spoiler-section.unveil .spoiler-text>*{opacity:1}.inline-spoiler-section.unveil .spoiler-img{filter:none}@media print{blockquote,div,img,pre,table{page-break-inside:avoid!important}a[href]:after{font-size:12px!important}}.markdown-body.slides{position:relative;z-index:1;color:#222}.markdown-body.slides:before{content:"";display:block;position:absolute;top:0;left:0;right:0;bottom:0;z-index:-1;background-color:currentColor;box-shadow:0 0 0 50vw}.markdown-body.slides section[data-markdown]{position:relative;margin-bottom:1.5em;background-color:#fff;text-align:center}.markdown-body.slides section[data-markdown] code{text-align:left}.markdown-body.slides section[data-markdown]:before{content:"";display:block;padding-bottom:56.23%}.markdown-body.slides section[data-markdown]>div:first-child{position:absolute;top:50%;left:1em;right:1em;transform:translateY(-50%);max-height:100%;overflow:hidden}.markdown-body.slides section[data-markdown]>ul{display:inline-block}.markdown-body.slides>section>section+section:after{content:"";position:absolute;top:-1.5em;right:1em;height:1.5em;border:3px solid #777}.site-ui-font{font-family:Source Sans Pro,Helvetica,Arial,sans-serif}html[lang^=ja] .site-ui-font{font-family:Source Sans Pro,Helvetica,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html[lang=zh-tw] .site-ui-font{font-family:Source Sans Pro,Helvetica,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html[lang=zh-cn] .site-ui-font{font-family:Source Sans Pro,Helvetica,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}body{font-smoothing:subpixel-antialiased!important;-webkit-font-smoothing:subpixel-antialiased!important;-moz-osx-font-smoothing:auto!important;-webkit-overflow-scrolling:touch;letter-spacing:.025em;font-family:Source Sans Pro,Helvetica,Arial,sans-serif}html[lang^=ja] body{font-family:Source Sans Pro,Helvetica,Arial,Hiragino Kaku Gothic Pro,ヒラギノ角ゴ Pro W3,Osaka,Meiryo,メイリオ,MS Gothic,ＭＳ\ ゴシック,sans-serif}html[lang=zh-tw] body{font-family:Source Sans Pro,Helvetica,Arial,PingFang TC,Microsoft JhengHei,微軟正黑,sans-serif}html[lang=zh-cn] body{font-family:Source Sans Pro,Helvetica,Arial,PingFang SC,Microsoft YaHei,微软雅黑,sans-serif}abbr[title]{border-bottom:none;text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted}abbr[data-original-title],abbr[title]{cursor:help}body.modal-open{overflow-y:auto;padding-right:0!important}svg{text-shadow:none}
    </style>
    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
    <!--[if lt IE 9]>
    	<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js" integrity="sha256-3Jy/GbSLrg0o9y5Z5n1uw0qxZECH7C6OQpVBgNFYa0g=" crossorigin="anonymous"></script>
    	<script src="https://cdnjs.cloudflare.com/ajax/libs/respond.js/1.4.2/respond.min.js" integrity="sha256-g6iAfvZp+nDQ2TdTR/VVKJf3bGro4ub5fvWSWVRi2NE=" crossorigin="anonymous"></script>
		<script src="https://cdnjs.cloudflare.com/ajax/libs/es5-shim/4.5.9/es5-shim.min.js" integrity="sha256-8E4Is26QH0bD52WoQpcB+R/tcWQtpzlCojrybUd7Mxo=" crossorigin="anonymous"></script>
    <![endif]-->
</head>

<body>
    <div id="doc" class="markdown-body container-fluid comment-inner comment-enabled" data-hard-breaks="true"><p><span>Document Number: P2629R0</span><br>
<span>Date: 2022-07-08</span><br>
<span>Reply to: Gonzalo Brito Gadeschi &lt;gonzalob _at_ </span><a href="http://nvidia.com" target="_blank" rel="noopener"><span>nvidia.com</span></a><span>&gt;</span><br>
<span>Authors: Gonzalo Brito Gadeschi</span><br>
<span>Audience: Concurrency</span></p><style>
ins {
    color:green; 
    background-color:yellow;
    text-decoration:underline;
}
del { 
    color:red;
    background-color:yellow;
    text-decoration:line-through;
}
.markdown-body { 
    max-width: 1100px; 
    text-align: justify;
}
</style><h1 id="barrier-token-less-split-arrivewait" data-id="barrier-token-less-split-arrivewait"><a class="anchor hidden-xs" href="#barrier-token-less-split-arrivewait" title="barrier-token-less-split-arrivewait"><span class="octicon octicon-link"></span></a><code>barrier</code><span> token-less split arrive/wait</span></h1><p><span class="toc"><ul>
<li><a href="#barrier-token-less-split-arrivewait" title="barrier token-less split arrive/wait">barrier token-less split arrive/wait</a><ul>
<li><a href="#Motivation" title="Motivation">Motivation</a></li>
<li><a href="#Tony-tables" title="Tony tables">Tony tables</a></li>
<li><a href="#User-guide" title="User guide">User guide</a><ul>
<li><a href="#Teaching-examples" title="Teaching examples">Teaching examples</a></li>
<li><a href="#Producer-consumer-pipeline-example" title="Producer-consumer pipeline example">Producer-consumer pipeline example</a></li>
</ul>
</li>
<li><a href="#Rationale" title="Rationale">Rationale</a><ul>
<li><a href="#Rationale-for-waiting-without-a-token" title="Rationale for waiting without a token">Rationale for waiting without a token</a></li>
<li><a href="#Rationale-for-arriving-without-a-token" title="Rationale for arriving without a token">Rationale for arriving without a token</a></li>
</ul>
</li>
<li><a href="#Impact" title="Impact">Impact</a><ul>
<li><a href="#Impact-on-users" title="Impact on users">Impact on users</a></li>
<li><a href="#Impact-on-implementations" title="Impact on implementations">Impact on implementations</a></li>
<li><a href="#Impact-onfrom-other-proposals" title="Impact on/from other proposals">Impact on/from other proposals</a></li>
</ul>
</li>
<li><a href="#Unresolved-questions" title="Unresolved questions">Unresolved questions</a></li>
<li><a href="#References" title="References">References</a></li>
<li><a href="#Proposed-wording" title="Proposed wording">Proposed wording</a></li>
</ul>
</li>
</ul>
</span></p><h2 id="Motivation" data-id="Motivation"><a class="anchor hidden-xs" href="#Motivation" title="Motivation"><span class="octicon octicon-link"></span></a><span>Motivation</span></h2><p><span>This proposal extends </span><code>std::barrier</code><span> with better support for data-pipelining. These pipelines are very common in, e.g., Deep Learning and HPC applications [</span><a href="https://developer.nvidia.com/blog/nvidia-hopper-architecture-in-depth/" target="_blank" rel="noopener"><span>0</span></a><span>].</span></p><p><span>In Deep Learning pipelines targetting modern NUMA systems, two groups of threads, “producers” and “consumers”, synchronize access to the shared resources of a pipeline stage using a pair of barriers. Consumer threads wait on the “consumer barrier”, process data once unblocked, and arrive on the “producer barrier” to signal that shared resources are safe to reuse. Producer threads wait on the “producer barrier”, reuse shared resources to load the next data set, and arrive on the “consumer barrier” to signal consumer threads that this data set is ready for use.</span></p><p><span>In this synchronization pattern, consumer threads wait on the consumer barrier without arriving on it, and arrive on the producer barrier without waiting on it; and vice-versa for producer threads.</span></p><p><span>However, </span><code>std::barrier</code><span> APIs require threads to arrive on a barrier before waiting on it. This proposal allows producer and consumer threads to synchronize efficiently.</span></p><h2 id="Tony-tables" data-id="Tony-tables"><a class="anchor hidden-xs" href="#Tony-tables" title="Tony-tables"><span class="octicon octicon-link"></span></a><span>Tony tables</span></h2><p><span>Real applications implement multi-stage pipelines with multiple shared resources. The following table shows a single stage of a consumer-producer pipeline involving one consumer thread, one producer thread, one </span><code>data</code><span> buffer as a shared resource, and a pair of </span><code>std::barriers</code><span>.</span></p><blockquote>
<p><strong><span>Note</span></strong><span>: the code of each thread runs in an infinite loop.</span></p>
</blockquote><table>
<thead>
<tr>
<th><span>Before</span></th>
<th></th>
<th><span>After</span></th>
<th></th>
</tr>
</thead>
<tbody>
<tr>
<td><span>Producer Thread</span></td>
<td><span>Consumer Thread</span></td>
<td><span>Producer Thread</span></td>
<td><span>Consumer Thread</span></td>
</tr>
<tr>
<td><span>producer.arrive_and_wait();</span><br><span>produce(data);</span><br><span>consumer.arrive();</span><br><br><br><br></td>
<td><br><br><br><span>consumer.arrive_and_wait();</span><br><span>consume(data);</span><br><span>producer.arrive();</span></td>
<td><span>producer.</span><ins><span>wait_parity(pp)</span></ins><span>;</span><br><span>produce(data);</span><br><span>consumer.arrive</span><ins><span>_and_discard</span></ins><span>();</span><br><br><br><br></td>
<td><br><br><br><span>consumer.</span><ins><span>wait_parity(pp)</span></ins><span>;</span><br><span>consume(data);</span><br><span>producer.arrive</span><ins><span>_and_discard</span></ins><span>();</span></td>
</tr>
</tbody>
</table><p><strong><span>Before</span></strong><span> this proposal:</span></p><ul>
<li><strong><span>Producer thread</span></strong><span>:</span>
<ol>
<li><span>Waits on the consumer thread to signal that </span><code>data</code><span> can be reused by arriving and waiting on the </span><code>producer</code><span> barrier.</span></li>
<li><span>Produces </span><code>data</code><span>.</span></li>
<li><span>Signals that </span><code>data</code><span> is ready by arriving at the </span><code>consumer</code><span> barrier; throws token away.</span></li>
</ol>
</li>
<li><strong><span>Consumer thread</span></strong><span>:</span>
<ol>
<li><span>Waits on </span><code>data</code><span> becoming ready by arriving and waiting on the </span><code>consumer</code><span> barrier.</span></li>
<li><span>Consumes </span><code>data</code><span>.</span></li>
<li><span>Signals the producer thread that it can reuse </span><code>data</code><span> by arriving on the </span><code>producer</code><span> barrier; throws token away.</span></li>
</ol>
</li>
</ul><p><span>These steps cause threads to unnecessarily:</span></p><ol>
<li><strong><span>modify a barrier</span></strong><span>: by calling </span><code>arrive_and_wait</code><span> when they only need to wait,</span></li>
<li><strong><span>stall</span></strong><span> at the </span><code>arrive</code><span> until the </span><code>barrier_token</code><span> is constructed when they will immediately discard it afterwards, and</span></li>
<li><strong><span>cast </span><code>[[nodiscard]]</code><span> away</span></strong><span>: to avoid the warning produced by discarding the token.</span></li>
</ol><p><strong><span>After</span></strong><span> this proposal:</span></p><ul>
<li><strong><span>Producer thread</span></strong><span>:</span>
<ol>
<li><span>Waits on the consumer thread to signal that </span><code>data</code><span> can be reused by </span><del><span>arriving and </span></del><span>waiting on the </span><code>producer</code><span> barrier.</span></li>
<li><span>Produces </span><code>data</code><span>.</span></li>
<li><span>Signals that </span><code>data</code><span> is ready by arriving at the </span><code>consumer</code><span> barrier</span><ins><span> without producing a </span><code>barrier_token</code><span>.</span></ins><del><span>; throws token away</span></del><span>.</span></li>
</ol>
</li>
<li><strong><span>Consumer thread</span></strong><span>:</span>
<ol>
<li><span>Waits on </span><code>data</code><span> being ready by</span><del><span> arriving and</span></del><span> waiting on the </span><code>consumer</code><span> barrier.</span></li>
<li><span>Consumes </span><code>data</code><span>.</span></li>
<li><span>Signals the producer thread that it can reuse </span><code>data</code><span> by arriving on the </span><code>producer</code><span> barrier</span><ins><span> without producing a token.</span></ins><del><span> throws token away.</span></del></li>
</ol>
</li>
</ul><p><span>This proposal addresses the three pre-existing issues raised above.</span></p><h2 id="User-guide" data-id="User-guide"><a class="anchor hidden-xs" href="#User-guide" title="User-guide"><span class="octicon octicon-link"></span></a><span>User guide</span></h2><p><span>The </span><code>arrive_and_discard</code><span> API is semantically equivalent to </span><code>static_cast&lt;void&gt;(barrier.arrive())</code><span> but is easier to implement efficiently.</span></p><p><span>This section therefore focuses on how to use:</span></p><pre><code class="cpp hljs"><div class="wrapper"><div class="gutter linenumber"><span></span></div><div class="code"><span class="token keyword">void</span> <span class="token function">wait_parity</span><span class="token punctuation">(</span><span class="token keyword">bool</span> phase_parity<span class="token punctuation">)</span> <span class="token keyword">const</span><span class="token punctuation">;</span>
</div></div></code></pre><p><span>The phase completion step of </span><code>std::barrier</code><span> advances the barrier to the “next phase” (</span><a href="http://eel.is/c++draft/thread.barrier#class-1.3" target="_blank" rel="noopener"><span>thread.barrier#class-1.3</span></a><span>).</span></p><p><span>This proposal guarantees that:</span></p><ul>
<li><span>the initial barrier phase is “phase </span><code>0</code><span>”, and</span></li>
<li><span>the successor of “phase </span><code>N</code><span>” is “phase </span><code>N+1</code><span>”.</span></li>
</ul><p><span>This proposal defines that the </span><code>parity</code><span> of:</span></p><ul>
<li><span>even barrier phases is </span><code>false</code><span>, i.e., the parity of “phase </span><code>0</code><span>” is </span><code>false</code><span> (or </span><code>0</code><span>);</span></li>
<li><span>odd barrier phases is </span><code>true</code><span>, i.e., the parity of “phase </span><code>1</code><span>” is </span><code>true</code><span> (or </span><code>1</code><span>).</span></li>
</ul><p><span>The </span><code>wait_parity(bool phase_parity)</code><span> API waits on the barrier phase parity changing from the specified </span><code>phase_parity</code><span> to a different parity. This is analogous to the </span><code>atomic::wait(value)</code><span> API, which waits on the atomic value changing from </span><code>value</code><span> to a different value.</span></p><h3 id="Teaching-examples" data-id="Teaching-examples"><a class="anchor hidden-xs" href="#Teaching-examples" title="Teaching-examples"><span class="octicon octicon-link"></span></a><span>Teaching examples</span></h3><p><span>This example uses a barrier to print text in order:</span></p><pre><code class="cpp hljs"><div class="wrapper"><div class="gutter linenumber"><span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span></div><div class="code"><span class="token keyword">void</span> <span class="token function">houston</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
    std<span class="token double-colon punctuation">::</span>barrier <span class="token function">barrier</span><span class="token punctuation">(</span><span class="token number">2</span><span class="token punctuation">)</span><span class="token punctuation">;</span>                  <span class="token comment">// A</span>
    std<span class="token double-colon punctuation">::</span>thread <span class="token function">apollo13</span><span class="token punctuation">(</span><span class="token punctuation">[</span><span class="token operator">&amp;</span>barrier<span class="token punctuation">]</span> <span class="token punctuation">{</span>
        std<span class="token double-colon punctuation">::</span>cout <span class="token operator">&lt;&lt;</span> <span class="token string">"Okay, Houston, we've had a problem here"</span> <span class="token operator">&lt;&lt;</span> std<span class="token double-colon punctuation">::</span>endl<span class="token punctuation">;</span>
        <span class="token generic-function"><span class="token function">static_cast</span><span class="token generic class-name"><span class="token operator">&lt;</span><span class="token keyword">void</span><span class="token operator">&gt;</span></span></span><span class="token punctuation">(</span>barrier<span class="token punctuation">.</span><span class="token function">arrive</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>  <span class="token comment">// B</span>
    <span class="token punctuation">}</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
    <span class="token keyword">bool</span> phase_parity <span class="token operator">=</span> <span class="token boolean">false</span><span class="token punctuation">;</span>
    barrier<span class="token punctuation">.</span><span class="token function">arrive_and_wait</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>                <span class="token comment">// C</span>
    std<span class="token double-colon punctuation">::</span>cout <span class="token operator">&lt;&lt;</span> <span class="token string">"This is Houston. Say again, please."</span> <span class="token operator">&lt;&lt;</span> std<span class="token double-colon punctuation">::</span>endl<span class="token punctuation">;</span>
    apollo13<span class="token punctuation">.</span><span class="token function">join</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
</div></div></code></pre><p><span>The initial phase of a freshly initialized </span><code>barrier</code><span> is 0 and its parity is </span><code>false</code><span>. To implement this example with the proposed APIs we can:</span></p><ul>
<li><span>Line A: use an </span><code>expected_count</code><span> of </span><code>1</code><span> instead of </span><code>2</code><span>, since only 1 thread will arrive on each barrier phase.</span></li>
<li><span>Line B: use </span><code>arrive_and_discard</code><span>, since this thread will not wait on the barrier.</span></li>
<li><span>Line C: use </span><code>wait_parity(false)</code><span> to wait on the phase parity of initial barrier phase, which is “phase 0” and has the phase parity </span><code>false</code><span>, to change to </span><code>true</code><span> (i.e. advancing to “phase 1”):</span></li>
</ul><pre><code class="cpp hljs"><div class="wrapper"><div class="gutter linenumber"><span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span></div><div class="code"><span class="token keyword">void</span> <span class="token function">houston</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
    std<span class="token double-colon punctuation">::</span>barrier <span class="token function">barrier</span><span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">;</span>          <span class="token comment">// A</span>
    std<span class="token double-colon punctuation">::</span>thread <span class="token function">apollo13</span><span class="token punctuation">(</span><span class="token punctuation">[</span><span class="token operator">&amp;</span>barrier<span class="token punctuation">]</span> <span class="token punctuation">{</span>
        std<span class="token double-colon punctuation">::</span>cout <span class="token operator">&lt;&lt;</span> <span class="token string">"Okay, Houston, we've had a problem here"</span> <span class="token operator">&lt;&lt;</span> std<span class="token double-colon punctuation">::</span>endl<span class="token punctuation">;</span>
        barrier<span class="token punctuation">.</span><span class="token function">arrive_and_discard</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span> <span class="token comment">// B</span>
    <span class="token punctuation">}</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
    <span class="token keyword">bool</span> phase_parity <span class="token operator">=</span> <span class="token boolean">false</span><span class="token punctuation">;</span>
    barrier<span class="token punctuation">.</span><span class="token function">wait_parity</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>            <span class="token comment">// C</span>
    std<span class="token double-colon punctuation">::</span>cout <span class="token operator">&lt;&lt;</span> <span class="token string">"This is Houston. Say again, please."</span> <span class="token operator">&lt;&lt;</span> std<span class="token double-colon punctuation">::</span>endl<span class="token punctuation">;</span>
    apollo13<span class="token punctuation">.</span><span class="token function">join</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
</div></div></code></pre><p><span>The following example shows a thread that uses </span><code>wait_pairty</code><span> to wait on all </span><code>barrier</code><span> phases by alternating the </span><code>phase_parity</code><span> it waits on, on every iteration:</span></p><pre><code class="cpp hljs"><div class="wrapper"><div class="gutter linenumber"><span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span></div><div class="code"><span class="token keyword">bool</span> phase_parity <span class="token operator">=</span> <span class="token boolean">false</span><span class="token punctuation">;</span>
<span class="token keyword">while</span><span class="token punctuation">(</span><span class="token boolean">true</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
    barrier<span class="token punctuation">.</span><span class="token function">wait_parity</span><span class="token punctuation">(</span>phase_parity<span class="token punctuation">)</span><span class="token punctuation">;</span>
    phase_parity <span class="token operator">=</span> <span class="token operator">!</span>phase_parity<span class="token punctuation">;</span>
    other_barrier<span class="token punctuation">.</span><span class="token function">arrive_and_discard</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
</div></div></code></pre><p><span>This thread arrives on some </span><code>other_barrier</code><span>, to make sure that other threads arriving on </span><code>barrier</code><span> don’t cause it to “skip” waiting on a particular phase.</span></p><h3 id="Producer-consumer-pipeline-example" data-id="Producer-consumer-pipeline-example"><a class="anchor hidden-xs" href="#Producer-consumer-pipeline-example" title="Producer-consumer-pipeline-example"><span class="octicon octicon-link"></span></a><span>Producer-consumer pipeline example</span></h3><p><span>The following code snippet (</span><a href="https://godbolt.org/z/W95Y1q1vY" target="_blank" rel="noopener"><span>try it out in compiler-explorer</span></a><span>) shows a full working example of the producer-consumer pipeline described above and showcases how to combine these APIs:</span></p><pre><code class="cpp hljs"><div class="wrapper"><div class="gutter linenumber"><span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span>
<span></span></div><div class="code"><span class="token macro property"><span class="token directive-hash">#</span><span class="token directive keyword">include</span> <span class="token string">&lt;thread&gt;</span></span>
<span class="token macro property"><span class="token directive-hash">#</span><span class="token directive keyword">include</span> <span class="token string">&lt;barrier&gt;</span></span>

<span class="token keyword">int</span> data <span class="token operator">=</span> <span class="token operator">-</span><span class="token number">1</span><span class="token punctuation">;</span>
<span class="token keyword">constexpr</span> <span class="token keyword">int</span> max_count <span class="token operator">=</span> <span class="token number">5</span><span class="token punctuation">;</span>

std<span class="token double-colon punctuation">::</span>barrier <span class="token function">producer</span><span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
std<span class="token double-colon punctuation">::</span>barrier <span class="token function">consumer</span><span class="token punctuation">(</span><span class="token number">1</span><span class="token punctuation">)</span><span class="token punctuation">;</span>

<span class="token keyword">void</span> <span class="token function">produce</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
    <span class="token keyword">int</span> count <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
    <span class="token comment">// Producer thread tracks the barrier phase separately.</span>
    <span class="token comment">// On the first iteration, the producer thread waits</span>
    <span class="token comment">// on "phase 1" of the barrier, which is an odd phase:</span>
    <span class="token keyword">bool</span> phase <span class="token operator">=</span> <span class="token boolean">true</span><span class="token punctuation">;</span>
    <span class="token keyword">do</span> <span class="token punctuation">{</span>
        data <span class="token operator">=</span> count<span class="token operator">++</span><span class="token punctuation">;</span>
        <span class="token comment">// Once the data is produced, it notifies the consumer</span>
        <span class="token comment">// by arriving on the "consumer" barrier using</span>
        <span class="token comment">// "arrive_and_discard", because this thread does not </span>
        <span class="token comment">// need to wait on that barrier:</span>
        consumer<span class="token punctuation">.</span><span class="token function">arrive_and_discard</span><span class="token punctuation">(</span><span class="token punctuation">)</span>
        <span class="token keyword">if</span> <span class="token punctuation">(</span>count <span class="token operator">==</span> max_count<span class="token punctuation">)</span> <span class="token keyword">return</span><span class="token punctuation">;</span>
        producer<span class="token punctuation">.</span><span class="token function">wait_parity</span><span class="token punctuation">(</span>phase<span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token comment">// After unblocking from the barrier, the phase to </span>
        <span class="token comment">// be waited on in the next iteration is updated:</span>
        phase <span class="token operator">=</span> <span class="token operator">!</span>phase<span class="token punctuation">;</span>
    <span class="token punctuation">}</span> <span class="token keyword">while</span><span class="token punctuation">(</span><span class="token boolean">true</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>

<span class="token keyword">void</span> <span class="token function">consume</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
    <span class="token keyword">int</span> count <span class="token operator">=</span> <span class="token number">0</span><span class="token punctuation">;</span>
    <span class="token keyword">bool</span> phase <span class="token operator">=</span> <span class="token boolean">true</span><span class="token punctuation">;</span>
    <span class="token keyword">while</span><span class="token punctuation">(</span><span class="token boolean">true</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
        consumer<span class="token punctuation">.</span><span class="token function">wait_parity</span><span class="token punctuation">(</span>phase<span class="token punctuation">)</span><span class="token punctuation">;</span>
        phase <span class="token operator">=</span> <span class="token operator">!</span>phase<span class="token punctuation">;</span>
        <span class="token function">assert</span><span class="token punctuation">(</span>data <span class="token operator">==</span> count<span class="token operator">++</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        producer<span class="token punctuation">.</span><span class="token function">arrive_and_discard</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
        <span class="token keyword">if</span> <span class="token punctuation">(</span>count <span class="token operator">==</span> max_count<span class="token punctuation">)</span> <span class="token keyword">return</span><span class="token punctuation">;</span>
    <span class="token punctuation">}</span>
<span class="token punctuation">}</span>

<span class="token keyword">int</span> <span class="token function">main</span><span class="token punctuation">(</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
    std<span class="token double-colon punctuation">::</span>thread <span class="token function">t</span><span class="token punctuation">(</span>produce<span class="token punctuation">)</span><span class="token punctuation">;</span>
    <span class="token function">consume</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
    t<span class="token punctuation">.</span><span class="token function">join</span><span class="token punctuation">(</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
    <span class="token keyword">return</span> <span class="token number">0</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
</div></div></code></pre><h2 id="Rationale" data-id="Rationale"><a class="anchor hidden-xs" href="#Rationale" title="Rationale"><span class="octicon octicon-link"></span></a><span>Rationale</span></h2><p><span>Applications cannot reuse </span><code>std::barrier</code><span> to solve these problems, and instead must re-implement it to add these APIs.</span></p><p><span>This proposal chooses to extend </span><code>std::barrier</code><span> with two APIs to enable applications to re-use it.</span></p><h3 id="Rationale-for-waiting-without-a-token" data-id="Rationale-for-waiting-without-a-token"><a class="anchor hidden-xs" href="#Rationale-for-waiting-without-a-token" title="Rationale-for-waiting-without-a-token"><span class="octicon octicon-link"></span></a><span>Rationale for waiting without a token</span></h3><p><span>The proposed </span><code>wait_parity</code><span> API is a replacement for </span><code>arrive_and_wait</code><span> in certain situations.</span></p><p><span>The </span><code>arrive_and_wait</code><span> API has significant latency:</span></p><ul>
<li><span>Sends a message to modify the memory to arrive at the barrier.</span></li>
<li><span>Waits on the response with the old value, required to construct the </span><code>arrival_token</code><span> that the API returns.</span></li>
<li><span>Finally calls wait to wait on the barrier phase to flip.</span></li>
</ul><p><span>The </span><code>wait_parity</code><span> API enables applications that “know” the parity of the barrier phase they want to wait on, and that are able to do so safely, to just wait on the barrier, without having to arrive on it first.</span></p><h3 id="Rationale-for-arriving-without-a-token" data-id="Rationale-for-arriving-without-a-token"><a class="anchor hidden-xs" href="#Rationale-for-arriving-without-a-token" title="Rationale-for-arriving-without-a-token"><span class="octicon octicon-link"></span></a><span>Rationale for arriving without a token</span></h3><p><span>The </span><code>arrive_and_discard</code><span> API is a replacement for </span><code>arrive</code><span> in situations where the calling thread is not going to wait at the barrier.</span></p><p><span>The </span><code>arrive</code><span> API:</span></p><ul>
<li><span>Sends a message to modify the memory to arrive at the barrier.</span></li>
<li><span>Waits on the response with the old value, required to construct the </span><code>arrival_token</code><span> that the API returns.</span></li>
</ul><p><span>The roundtrip latency of sending the message and waiting for the response is significant in large NUMA architectures.</span></p><p><a href="" target="_blank" rel="noopener"><span>P2588</span></a><span> “</span><code>barrier</code><span>’s phase completion guarantees” proposes relaxing the specification of </span><code>barrier</code><span> to enable implementations to complete the phase in one of the threads that </span><code>waits</code><span> at the barrier by requiring that at least one thread waits for the phase to flip.</span></p><p><span>On an implementation that completes the phase on a thread that waits, and on a proper hardware architecture with support for far atomics, </span><code>arrive_and_discard</code><span> is fire-and-forget. It sends a message to modify the barrier count, but it does not need to wait for a response.</span></p><p><span>During the call to </span><code>wait</code><span>, the implementation then picks one thread, and uses it to complete the phase, unblocking all other threads when the phase completes.</span></p><p><span>The </span><a href="https://godbolt.org/z/W95Y1q1vY" target="_blank" rel="noopener"><span>reference implementation of this proposal on compiler-explorer</span></a><span> provides such an implementation.</span></p><p><span>On large NUMA architectures and data-pipelining applications, the consumer barrier can be placed close to the consumer threads, and the producer barrier can be placed close to the producer threads.</span></p><p><span>This enables threads to poll the barrier locally and benefit from HW acceleration to do so, but increases the latency of arriving at the barrier.</span></p><p><span>The </span><code>arrive_and_discard</code><span> API brings the latency to zero on such an implementation, because the thread does not need to wait for a response to be able to make progress.</span></p><h2 id="Impact" data-id="Impact"><a class="anchor hidden-xs" href="#Impact" title="Impact"><span class="octicon octicon-link"></span></a><span>Impact</span></h2><h3 id="Impact-on-users" data-id="Impact-on-users"><a class="anchor hidden-xs" href="#Impact-on-users" title="Impact-on-users"><span class="octicon octicon-link"></span></a><span>Impact on users</span></h3><p><span>None, the semantics of existing code do not change, and it does not change the ABI.</span></p><h3 id="Impact-on-implementations" data-id="Impact-on-implementations"><a class="anchor hidden-xs" href="#Impact-on-implementations" title="Impact-on-implementations"><span class="octicon octicon-link"></span></a><span>Impact on implementations</span></h3><p><span>The </span><code>arrive_and_discard</code><span> API imposes minimal overhead on implementations, since it can be naively implemented as:</span></p><pre><code class="cpp hljs"><div class="wrapper"><div class="gutter linenumber"><span></span>
<span></span>
<span></span></div><div class="code"><span class="token keyword">void</span> <span class="token function">arrive_and_discard</span><span class="token punctuation">(</span>ptrdiff_t update <span class="token operator">=</span> <span class="token number">1</span><span class="token punctuation">)</span> <span class="token punctuation">{</span>
    <span class="token generic-function"><span class="token function">static_cast</span><span class="token generic class-name"><span class="token operator">&lt;</span><span class="token keyword">void</span><span class="token operator">&gt;</span></span></span><span class="token punctuation">(</span><span class="token function">arrive</span><span class="token punctuation">(</span>update<span class="token punctuation">)</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
</div></div></code></pre><p><span>The implementation impact of </span><code>wait_parity</code><span> is “to be determined”. In the reference implementation, it is implemented naively as:</span></p><pre><code class="cpp hljs"><div class="wrapper"><div class="gutter linenumber"><span></span>
<span></span>
<span></span></div><div class="code"><span class="token keyword">void</span> <span class="token function">wait_parity</span><span class="token punctuation">(</span><span class="token keyword">bool</span> phase_parity<span class="token punctuation">)</span> <span class="token keyword">const</span> <span class="token punctuation">{</span>
    <span class="token function">wait</span><span class="token punctuation">(</span>arrival_token <span class="token punctuation">{</span> <span class="token punctuation">.</span>phase <span class="token operator">=</span> phase_parity <span class="token punctuation">}</span><span class="token punctuation">)</span><span class="token punctuation">;</span>
<span class="token punctuation">}</span>
</div></div></code></pre><h3 id="Impact-onfrom-other-proposals" data-id="Impact-onfrom-other-proposals"><a class="anchor hidden-xs" href="#Impact-onfrom-other-proposals" title="Impact-onfrom-other-proposals"><span class="octicon octicon-link"></span></a><span>Impact on/from other proposals</span></h3><p><a href="" target="_blank" rel="noopener"><span>P2588</span></a><span> proposes relaxing </span><code>std::barrier</code><span>’s phase completion guarantees. Depending on the outcome of this proposal, </span><code>wait_parity</code><span> would, just like </span><code>wait</code><span>, be able to complete the barrier’s phase. In that case, it might make sense to not make </span><code>wait_parity</code><span> a </span><code>const</code><span> member function.</span></p><h2 id="Unresolved-questions" data-id="Unresolved-questions"><a class="anchor hidden-xs" href="#Unresolved-questions" title="Unresolved-questions"><span class="octicon octicon-link"></span></a><span>Unresolved questions</span></h2><ul>
<li><span>Do we need the “diff” in the barrier constructor to state that it initializes the barrier to the 0-th phase? That is already addressed in thread.barrier.class.1.</span></li>
<li><span>Impact on existing implementations, to be resolved by prototyping the feature in libc++ and libstdc++.</span></li>
</ul><h2 id="References" data-id="References"><a class="anchor hidden-xs" href="#References" title="References"><span class="octicon octicon-link"></span></a><span>References</span></h2><ul>
<li><span>[0]: See “Asynchronous Execution” section in: </span><a href="https://developer.nvidia.com/blog/nvidia-hopper-architecture-in-depth/" target="_blank" rel="noopener"><span>The NVIDIA Hopper Architecture In-Depth</span></a><span>.</span></li>
<li><a href="" target="_blank" rel="noopener"><span>P2588</span></a><span>: </span><code>barrier</code><span>’s phase completion guarantees.</span></li>
<li><span>Reference implementation of this proposal and </span><a href="" target="_blank" rel="noopener"><span>P2588</span></a><span> on </span><a href="https://godbolt.org/z/W95Y1q1vY" target="_blank" rel="noopener"><span>compiler-explorer</span></a><span>.</span></li>
</ul><h2 id="Proposed-wording" data-id="Proposed-wording"><a class="anchor hidden-xs" href="#Proposed-wording" title="Proposed-wording"><span class="octicon octicon-link"></span></a><span>Proposed wording</span></h2><p><a href="http://eel.is/c++draft/thread.barrier.class" target="_blank" rel="noopener"><span>thread.barrier.class</span></a><span>:</span></p><pre><code>namespace std {
  template&lt;class CompletionFunction = see below&gt;
  class barrier {
  public:
    using arrival_token = see below;

    static constexpr ptrdiff_t max() noexcept;

    constexpr explicit barrier(ptrdiff_t expected,
                               CompletionFunction f = CompletionFunction());
    ~barrier();

    barrier(const barrier&amp;) = delete;
    barrier&amp; operator=(const barrier&amp;) = delete;

    [[nodiscard]] arrival_token arrive(ptrdiff_t update = 1);
    <ins>void arrive_and_discard(ptrdiff_t update = 1);</ins>
    void wait(arrival_token&amp;&amp; arrival) const;
    <ins>void wait_parity(bool phase_parity) const;</ins>

    void arrive_and_wait();
    void arrive_and_drop();

  private:
    CompletionFunction completion;      // exposition only
  };
}
</code></pre><ol>
<li><ins><span>The initial phase of a barrier is the 0-th phase. The successor of the n-th barrier phase is the (n+1)-th phase.</span></ins></li>
<li><span>Each barrier phase consists of the following steps:</span>
<ol>
<li><span>The expected count is decremented by each call to </span><code>arrive</code><span> or </span><code>arrive_and_drop</code><span>.</span></li>
<li><span>When the expected count reaches zero, the phase completion step is run. For the specialization with the default value of the </span><code>CompletionFunction</code><span> template parameter, the completion step is run as part of the call to </span><code>arrive</code><span> or </span><code>arrive_and_drop</code><span> that caused the expected count to reach zero. For other specializations, the completion step is run on one of the threads that arrived at the barrier during the phase.</span></li>
<li><span>When the completion step finishes, the expected count is reset to what was specified by the expected argument to the constructor, possibly adjusted by calls to </span><code>arrive_and_drop</code><span>, and the </span><del><span>next</span></del><ins><span>successor of the current</span></ins><span> phase starts.</span></li>
</ol>
</li>
<li><span>Each phase defines a phase synchronization point. Threads that arrive at the barrier during the phase can block on the phase synchronization point by calling </span><code>wait</code><span>, and will remain blocked until the phase completion step is run.</span></li>
<li><span>The phase completion step that is executed at the end of each phase has the following effects:</span>
<ol>
<li><span>Invokes the completion function, equivalent to </span><code>completion()</code><span>.</span></li>
<li><span>Unblocks all threads that are blocked on the phase synchronization point.</span><br><br>
<span>The end of the completion step strongly happens before the returns from all calls that were unblocked by the completion step. For specializations that do not have the default value of the </span><code>CompletionFunction</code><span> template parameter, the behavior is undefined if any of the barrier object’s member functions other than </span><code>wait</code><span> are called while the completion step is in progress.</span></li>
</ol>
</li>
<li><span>Concurrent invocations of the member functions of barrier, other than its destructor, do not introduce data races. The member functions </span><code>arrive</code><span> and </span><code>arrive_and_drop</code><span> execute atomically.</span></li>
<li><code>CompletionFunction</code><span> shall meet the </span><code>Cpp17MoveConstructible</code><span> (Table 30) and </span><code>Cpp17Destructible</code><span> (Table 34) requirements. </span><code>is_nothrow_invocable_v&lt;CompletionFunction&amp;&gt;</code><span> shall be true.</span></li>
<li><span>The default value of the </span><code>CompletionFunction</code><span> template parameter is an unspecified type, such that, in addition to satisfying the requirements of </span><code>CompletionFunction</code><span>, it meets the </span><code>Cpp17DefaultConstructible</code><span> requirements (Table 29) and </span><code>completion()</code><span> has no effects.</span></li>
<li><code>barrier::arrival_token</code><span> is an unspecified type, such that it meets the </span><code>Cpp17MoveConstructible</code><span> (Table 30), </span><code>Cpp17MoveAssignable</code><span> (Table 32), and </span><code>Cpp17Destructible</code><span> (Table 34) requirements.</span></li>
</ol><pre><code>static constexpr ptrdiff_t max() noexcept;</code></pre><ul>
<li><span>Returns: The maximum expected count that the implementation supports.</span></li>
</ul><pre><code>constexpr explicit barrier(ptrdiff_t expected,
                           CompletionFunction f = CompletionFunction());</code></pre><ul>
<li><span>Preconditions: </span><code>expected &gt;= 0</code><span> is </span><code>true</code><span> and </span><code>expected &lt;= max()</code><span> is </span><code>true</code><span>.</span></li>
<li><span>Effects: Sets both the initial expected count for each barrier phase and the current expected count for the first phase to </span><code>expected</code><span>. Initializes </span><code>completion</code><span> with </span><code>std::move(f)</code><span>. Starts the first phase</span><ins><span> which is the 0-th barrier phase</span></ins><span>.</span>
<ul>
<li><span>[Note 1: If </span><code>expected</code><span> is </span><code>0</code><span> this object can only be destroyed. — end note]</span></li>
</ul>
</li>
<li><span>Throws: Any exception thrown by </span><code>CompletionFunction</code><span>’s move constructor.</span></li>
</ul><pre><code>[[nodiscard]] arrival_token arrive(ptrdiff_t update = 1);</code></pre><ul>
<li><span>Preconditions: </span><code>update &gt; 0</code><span> is </span><code>true</code><span>, and </span><code>update</code><span> is less than or equal to the expected count for the current barrier phase.</span></li>
<li><span>Effects: Constructs an object of type </span><code>arrival_token</code><span> that is associated with the phase synchronization point for the current phase. Then, decrements the expected count by </span><code>update</code><span>.</span></li>
<li><span>Synchronization: The call to arrive strongly happens before the start of the phase completion step for the current phase.</span></li>
<li><span>Returns: The constructed </span><code>arrival_token</code><span> object.</span></li>
<li><span>Throws: </span><code>system_error</code><span> when an exception is required (</span><a href="http://eel.is/c++draft/thread.req.exception" target="_blank" rel="noopener"><span>thread.req.exception</span></a><span>).</span></li>
<li><span>Error conditions: Any of the error conditions allowed for mutex types (</span><a href="http://eel.is/c++draft/thread.mutex.requirements.mutex" target="_blank" rel="noopener"><span>thread.mutex.requirements.mutex</span></a><span>).</span>
<ul>
<li><span>[Note 2: This call can cause the completion step for the current phase to start. — end note]</span></li>
</ul>
</li>
</ul><pre><code><ins>void arrive_and_discard(ptrdiff_t update = 1);</ins></code></pre><ul>
<li><ins><span>Effects: Equivalent to: </span><code>arrive(update)</code><span>.</span></ins></li>
</ul><pre><code>void wait(arrival_token&amp;&amp; arrival) const;</code></pre><ul>
<li><span>Preconditions: </span><code>arrival</code><span> is associated with the phase synchronization point for the current phase or the immediately preceding phase of the same barrier object.</span></li>
<li><span>Effects: Blocks at the synchronization point associated with </span><code>std::move(arrival)</code><span> until the phase completion step of the synchronization point’s phase is run.</span>
<ul>
<li><span>[Note 3: If </span><code>arrival</code><span> is associated with the synchronization point for a previous phase, the call returns immediately. — end note]</span></li>
</ul>
</li>
<li><span>Throws: </span><code>system_error</code><span> when an exception is required (</span><a href="http://eel.is/c++draft/thread.req.exception" target="_blank" rel="noopener"><span>thread.req.exception</span></a><span>).</span></li>
<li><span>Error conditions: Any of the error conditions allowed for mutex types (</span><a href="http://eel.is/c++draft/thread.mutex.requirements.mutex" target="_blank" rel="noopener"><span>thread.mutex.requirements.mutex</span></a><span>).</span></li>
</ul><pre><code><ins>void wait_parity(bool phase_parity) const;</ins></code></pre><ul>
<li><ins><span>Effects: Equivalent to: </span><code>wait(token)</code><span>, where </span><code>token</code><span> is associated with the synchronization point of the current phase if the parity of the phase ordinal matches </span><code>phase_parity</code><span>, or the previous phase otherwise.</span></ins></li>
</ul><blockquote>
<p><ins><strong><span>UNRESOLVED QUESTION</span></strong></ins><span>: if P2588 is accepted, we might not want to make </span><code>wait_parity</code><span> const.</span></p>
</blockquote><pre><code>void arrive_and_wait();</code></pre><ul>
<li><span>Effects: Equivalent to: </span><code>wait(arrive())</code><span>.</span></li>
</ul><pre><code>void arrive_and_drop();</code></pre><ul>
<li><span>Preconditions: The expected count for the current barrier phase is greater than zero.</span></li>
<li><span>Effects: Decrements the initial expected count for all subsequent phases by one. Then decrements the expected count for the current phase by one.</span></li>
<li><span>Synchronization: The call to </span><code>arrive_and_drop</code><span> strongly happens before the start of the phase completion step for the current phase.</span></li>
<li><span>Throws: </span><code>system_error</code><span> when an exception is required (</span><a href="http://eel.is/c++draft/thread.req.exception" target="_blank" rel="noopener"><span>thread.req.exception</span></a><span>).</span></li>
<li><span>Error conditions: Any of the error conditions allowed for mutex types (</span><a href="http://eel.is/c++draft/thread.mutex.requirements.mutex" target="_blank" rel="noopener"><span>thread.mutex.requirements.mutex</span></a><span>).</span>
<ul>
<li><span>[Note 4: This call can cause the completion step for the current phase to start. — end note]</span></li>
</ul>
</li>
</ul></div>
    <div class="ui-toc dropup unselectable hidden-print" style="display:none;">
        <div class="pull-right dropdown">
            <a id="tocLabel" class="ui-toc-label btn btn-default" data-toggle="dropdown" href="#" role="button" aria-haspopup="true" aria-expanded="false" title="Table of content">
                <i class="fa fa-bars"></i>
            </a>
            <ul id="ui-toc" class="ui-toc-dropdown dropdown-menu" aria-labelledby="tocLabel">
                <div class="toc"><ul class="nav">
<li><a href="#barrier-token-less-split-arrivewait" title="barrier token-less split arrive/wait">barrier token-less split arrive/wait</a><ul class="nav">
<li><a href="#Motivation" title="Motivation">Motivation</a></li>
<li><a href="#Tony-tables" title="Tony tables">Tony tables</a></li>
<li><a href="#User-guide" title="User guide">User guide</a><ul class="nav">
<li><a href="#Teaching-examples" title="Teaching examples">Teaching examples</a></li>
<li><a href="#Producer-consumer-pipeline-example" title="Producer-consumer pipeline example">Producer-consumer pipeline example</a></li>
</ul>
</li>
<li><a href="#Rationale" title="Rationale">Rationale</a><ul class="nav">
<li><a href="#Rationale-for-waiting-without-a-token" title="Rationale for waiting without a token">Rationale for waiting without a token</a></li>
<li><a href="#Rationale-for-arriving-without-a-token" title="Rationale for arriving without a token">Rationale for arriving without a token</a></li>
</ul>
</li>
<li><a href="#Impact" title="Impact">Impact</a><ul class="nav">
<li><a href="#Impact-on-users" title="Impact on users">Impact on users</a></li>
<li><a href="#Impact-on-implementations" title="Impact on implementations">Impact on implementations</a></li>
<li><a href="#Impact-onfrom-other-proposals" title="Impact on/from other proposals">Impact on/from other proposals</a></li>
</ul>
</li>
<li><a href="#Unresolved-questions" title="Unresolved questions">Unresolved questions</a></li>
<li><a href="#References" title="References">References</a></li>
<li><a href="#Proposed-wording" title="Proposed wording">Proposed wording</a></li>
</ul>
</li>
</ul>
</div><div class="toc-menu"><a class="expand-toggle" href="#">Expand all</a><a class="back-to-top" href="#">Back to top</a><a class="go-to-bottom" href="#">Go to bottom</a></div>
            </ul>
        </div>
    </div>
    <div id="ui-toc-affix" class="ui-affix-toc ui-toc-dropdown unselectable hidden-print" data-spy="affix" style="top:17px;display:none;" null null>
        <div class="toc"><ul class="nav">
<li><a href="#barrier-token-less-split-arrivewait" title="barrier token-less split arrive/wait">barrier token-less split arrive/wait</a><ul class="nav">
<li><a href="#Motivation" title="Motivation">Motivation</a></li>
<li><a href="#Tony-tables" title="Tony tables">Tony tables</a></li>
<li><a href="#User-guide" title="User guide">User guide</a><ul class="nav">
<li><a href="#Teaching-examples" title="Teaching examples">Teaching examples</a></li>
<li><a href="#Producer-consumer-pipeline-example" title="Producer-consumer pipeline example">Producer-consumer pipeline example</a></li>
</ul>
</li>
<li><a href="#Rationale" title="Rationale">Rationale</a><ul class="nav">
<li><a href="#Rationale-for-waiting-without-a-token" title="Rationale for waiting without a token">Rationale for waiting without a token</a></li>
<li><a href="#Rationale-for-arriving-without-a-token" title="Rationale for arriving without a token">Rationale for arriving without a token</a></li>
</ul>
</li>
<li><a href="#Impact" title="Impact">Impact</a><ul class="nav">
<li><a href="#Impact-on-users" title="Impact on users">Impact on users</a></li>
<li><a href="#Impact-on-implementations" title="Impact on implementations">Impact on implementations</a></li>
<li><a href="#Impact-onfrom-other-proposals" title="Impact on/from other proposals">Impact on/from other proposals</a></li>
</ul>
</li>
<li><a href="#Unresolved-questions" title="Unresolved questions">Unresolved questions</a></li>
<li><a href="#References" title="References">References</a></li>
<li><a href="#Proposed-wording" title="Proposed wording">Proposed wording</a></li>
</ul>
</li>
</ul>
</div><div class="toc-menu"><a class="expand-toggle" href="#">Expand all</a><a class="back-to-top" href="#">Back to top</a><a class="go-to-bottom" href="#">Go to bottom</a></div>
    </div>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js" integrity="sha256-hVVnYaiADRTO2PzUGmuLJr8BLUSjGIZsDYGmIJLv2b8=" crossorigin="anonymous"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha256-U5ZEeKfGNOja007MMD3YBI0A3OSZOQbeG6z2f2Y0hu8=" crossorigin="anonymous" defer></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/gist-embed/2.6.0/gist-embed.min.js" integrity="sha256-KyF2D6xPIJUW5sUDSs93vWyZm+1RzIpKCexxElmxl8g=" crossorigin="anonymous" defer></script>
    <script>
        var markdown = $(".markdown-body");
        //smooth all hash trigger scrolling
        function smoothHashScroll() {
            var hashElements = $("a[href^='#']").toArray();
            for (var i = 0; i < hashElements.length; i++) {
                var element = hashElements[i];
                var $element = $(element);
                var hash = element.hash;
                if (hash) {
                    $element.on('click', function (e) {
                        // store hash
                        var hash = this.hash;
                        if ($(hash).length <= 0) return;
                        // prevent default anchor click behavior
                        e.preventDefault();
                        // animate
                        $('body, html').stop(true, true).animate({
                            scrollTop: $(hash).offset().top
                        }, 100, "linear", function () {
                            // when done, add hash to url
                            // (default click behaviour)
                            window.location.hash = hash;
                        });
                    });
                }
            }
        }

        smoothHashScroll();
        var toc = $('.ui-toc');
        var tocAffix = $('.ui-affix-toc');
        var tocDropdown = $('.ui-toc-dropdown');
        //toc
        tocDropdown.click(function (e) {
            e.stopPropagation();
        });

        var enoughForAffixToc = true;

        function generateScrollspy() {
            $(document.body).scrollspy({
                target: ''
            });
            $(document.body).scrollspy('refresh');
            if (enoughForAffixToc) {
                toc.hide();
                tocAffix.show();
            } else {
                tocAffix.hide();
                toc.show();
            }
            $(document.body).scroll();
        }

        function windowResize() {
            //toc right
            var paddingRight = parseFloat(markdown.css('padding-right'));
            var right = ($(window).width() - (markdown.offset().left + markdown.outerWidth() - paddingRight));
            toc.css('right', right + 'px');
            //affix toc left
            var newbool;
            var rightMargin = (markdown.parent().outerWidth() - markdown.outerWidth()) / 2;
            //for ipad or wider device
            if (rightMargin >= 133) {
                newbool = true;
                var affixLeftMargin = (tocAffix.outerWidth() - tocAffix.width()) / 2;
                var left = markdown.offset().left + markdown.outerWidth() - affixLeftMargin;
                tocAffix.css('left', left + 'px');
            } else {
                newbool = false;
            }
            if (newbool != enoughForAffixToc) {
                enoughForAffixToc = newbool;
                generateScrollspy();
            }
        }
        $(window).resize(function () {
            windowResize();
        });
        $(document).ready(function () {
            windowResize();
            generateScrollspy();
        });

        //remove hash
        function removeHash() {
            window.location.hash = '';
        }

        var backtotop = $('.back-to-top');
        var gotobottom = $('.go-to-bottom');

        backtotop.click(function (e) {
            e.preventDefault();
            e.stopPropagation();
            if (scrollToTop)
                scrollToTop();
            removeHash();
        });
        gotobottom.click(function (e) {
            e.preventDefault();
            e.stopPropagation();
            if (scrollToBottom)
                scrollToBottom();
            removeHash();
        });

        var toggle = $('.expand-toggle');
        var tocExpand = false;

        checkExpandToggle();
        toggle.click(function (e) {
            e.preventDefault();
            e.stopPropagation();
            tocExpand = !tocExpand;
            checkExpandToggle();
        })

        function checkExpandToggle () {
            var toc = $('.ui-toc-dropdown .toc');
            var toggle = $('.expand-toggle');
            if (!tocExpand) {
                toc.removeClass('expand');
                toggle.text('Expand all');
            } else {
                toc.addClass('expand');
                toggle.text('Collapse all');
            }
        }

        function scrollToTop() {
            $('body, html').stop(true, true).animate({
                scrollTop: 0
            }, 100, "linear");
        }

        function scrollToBottom() {
            $('body, html').stop(true, true).animate({
                scrollTop: $(document.body)[0].scrollHeight
            }, 100, "linear");
        }
    </script>
</body>

</html>
