@inproceedings{1f2b557513d84f40ad9bb59088b02286,
title = "Importance of data distribution on hive-based systems for query performance: An experimental study",
abstract = "SQL-on-Hadoop systems have been gaining popularity in recent years. One popular example of SQL-on-Hadoop systems is Apache Hive; the pioneer of SQL-on-Hadoop systems. Hive is located on the top of big data stack as an application layer. Besides the application layer, the Hadoop Ecosystem is composed of 3 different main layers: storage, the resource manager and processing engine. The demand from industry has led to the development of new efficient components for each layer. As the ecosystem evolves over time, Hive employed different execution engines too. Understanding the strengths of components is very important in order to exploit the full performance of the Hadoop Ecosystem. Therefore, recent works in the literature study the importance of each layer separately. To the best of our knowledge, the present work is the first work that focuses on the performance of the combination of both the storage layer and the execution engine. In this work, we compare the Hive's query performance by using three different execution engines: MR, Tez and Spark on the skewed/well-balanced data distribution through the full TPC-H benchmark. Our results show the importance of data distribution on the storage layer for overall job performance of SQL-on-Hadoop systems and empirically showed even distribution improves performance up to 48\% compared to skewed distribution. Moreover, the present study provides insightful findings by identifying particular SQL query cases that the certain processing engine deals exceptionally well.",
keywords = "Data distribution, Hadoop, HDFS, Software Performance, SQL-on-Hadoop",
author = "Ciritoglu, \{Hilmi Egemen\} and John Murphy and Christina Thorpe",
note = "Publisher Copyright: {\textcopyright} 2020 IEEE.; 2020 IEEE International Conference on Big Data and Smart Computing, BigComp 2020 ; Conference date: 19-02-2020 Through 22-02-2020",
year = "2020",
month = feb,
doi = "10.1109/BigComp48618.2020.00-47",
language = "English",
series = "Proceedings - 2020 IEEE International Conference on Big Data and Smart Computing, BigComp 2020",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "370--376",
editor = "Wookey Lee and Luonan Chen and Yang-Sae Moon and Julien Bourgeois and Mehdi Bennis and Yu-Feng Li and Young-Guk Ha and Hyuk-Yoon Kwon and Alfredo Cuzzocrea",
booktitle = "Proceedings - 2020 IEEE International Conference on Big Data and Smart Computing, BigComp 2020",
address = "United States",
}