@inproceedings{eefcfe27966043249b47cb5058cbfd1e,
title = "Turbine: A distributed-memory dataflow engine for extreme-scale many-task applications",
abstract = "Efficiently utilizing the rapidly increasing concurrency of multi-petaop computing systems is a significant program- ming challenge. One approach is to structure applications with an upper layer of many loosely-coupled coarse-grained tasks, each comprising a tightly-coupled parallel function or program. {"}Many-task{"} programming models such as functional parallel dataflow may be used at the upper layer to generate massive numbers of tasks, each of which generates significant tighly-coupled parallelism at the lower level via multithreading, message passing, and/or partitioned global address spaces. At large scales, however, the management of task distribution, data dependencies, and inter-task data movement is a significant performance challenge. In this work, we describe Turbine, a new highly scalable and distributed many-task dataflow engine. Turbine executes a generalized many-task intermediate representation with automated self-distribution, and is scalable to multi-petaop infrastructures. We present here the architecture of Turbine and its performance on highly concurrent systems.",
keywords = "ADLB, Concurrency, Dataflow, Exascale, MPI, Swift, Turbine",
author = "Wozniak, {Justin M.} and Armstrong, {Timothy G.} and Ketan Maheshwari and Lusk, {Ewing L.} and Katz, {Daniel S.} and Michael Wilde and Foster, {Ian T.}",
note = "Funding Information: This study was supported through a Natural Sciences and Engineering Research Council Operating Grant to J.R.P. and funds from British Columbia Ministry of the Environment, Lands and Parks to E.A.P and N.T.J. We gratefully acknowledge assistance in the field from Brian Toth, Nick Fol-kard, Gareth Williams, Rena Vandenbos, Mark Hammond, Theresa Godin, and Gary Scrimgeour and editorial suggestions from Gary Mittelbach and Leslie Holland-Bartels. We also acknowledge the cooperation of the Douglas Lake Cattle Company for allowing access to Crater Pothole Lakes 2, 3 and 4. We thank the staff of the Fraser Valley Trout Hatchery for providing fish for this study.; 1st ACM SIGMOD Workshop on Scalable Workflow Execution Engines and Technologies, SWEET 2012 ; Conference date: 20-05-2012 Through 20-05-2012",
year = "2012",
doi = "10.1145/2443416.2443421",
language = "English (US)",
isbn = "9781450318761",
series = "ACM International Conference Proceeding Series",
booktitle = "Proceedings of the 1st ACM SIGMOD Workshop on Scalable Workflow Execution Engines and Technologies, SWEET 2012",
}