@inproceedings{d958b8b4651a45e68a840bb0b4056c98,
title = "ShmStreaming: A shared memory approach for improving Hadoop streaming performance",
abstract = "The Map-Reduce programming model is now drawing both academic and industrial attentions for processing large data. Hadoop, one of the most popular implementations of the model, has been widely adopted. To support application programs written in languages other than Java, Hadoop introduces a streaming mechanism that allows it to communicate with external programs through pipes. Because of the added overhead associated with pipes and context switches, the performance of Hadoop streaming is significantly worse than native Hadoop jobs. We propose ShmStreaming, a mechanism that takes advantages of shared memory to realize Hadoop streaming for better performance. Specifically, ShmStreaming uses shared memory to implement a lockless FIFO queue that connects Hadoop and external programs. To further reduce the number of context switches, the FIFO queue adopts a batching technique to allow multiple key-value pairs to be processed together. For typical benchmarks of word count, grep and inverted index, experimental results show 20-30\% performance improvement comparing to the native Hadoop streaming implementation.",
keywords = "Hadoop streaming, Map-reduce, Shared memory",
author = "Longbin Lai and Jingyu Zhou and Long Zheng and Huakang Li and Yanchao Lu and Feilong Tang and Minyi Guo",
year = "2013",
doi = "10.1109/AINA.2013.90",
language = "英语",
isbn = "9780769549538",
series = "Proceedings - International Conference on Advanced Information Networking and Applications, AINA",
pages = "137--144",
booktitle = "Proceedings - IEEE International Conference on Advanced Information Networking and Applications, AINA 2013",
note = "27th IEEE International Conference on Advanced Information Networking and Applications, AINA 2013 ; Conference date: 25-03-2013 Through 28-03-2013",
}