@inproceedings{efaae6547d354439922dc342acae45a9,
title = "Evaluating distributed execution of workloads",
abstract = "Resource selection and task placement for distributed execution poses conceptual and implementation difficulties. Although resource selection and task placement are at the core of many tools and workflow systems, the methods are ad hoc rather than being based on models. Consequently, partial and non-interoperable implementations proliferate. We address both the conceptual and implementation difficulties by experimentally characterizing diverse modalities of resource selection and task placement. We compare the architectures and capabilities of two systems: The AIMES middleware and Swift workflow scripting language and runtime. We integrate these systems to enable the distributed execution of Swift workflows on Pilot-Jobs managed by the AIMES middleware. Our experiments characterize and compare alternative execution strategies by measuring the time to completion of heterogeneous uncoupled workloads executed at diverse scale and on multiple resources. We measure the adverse effects of pilot fragmentation and early binding of tasks to resources and the benefits of backfill scheduling across pilots on multiple resources. We then use this insight to execute a multi-stage workflow across five production-grade resources. We discuss the importance and implications for other tools and workflow systems.",
keywords = "distributed cyberinfrastructure, distributed execution, workflow systems",
author = "Matteo Turilli and Babuji, {Yadu Nand} and Andre Merzky and Ha, {Ming Tai} and Michael Wilde and Katz, {Daniel S.} and Shantenu Jha",
note = "Publisher Copyright: {\textcopyright} 2017 IEEE.; 13th IEEE International Conference on eScience, eScience 2017 ; Conference date: 24-10-2017 Through 27-10-2017",
year = "2017",
month = nov,
day = "14",
doi = "10.1109/eScience.2017.41",
language = "English (US)",
series = "Proceedings - 13th IEEE International Conference on eScience, eScience 2017",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "276--285",
booktitle = "Proceedings - 13th IEEE International Conference on eScience, eScience 2017",
address = "United States",
}