How to parallelize merge sort using JavaRDD<Integer> Spark? - parallel-processing

Question

public class MergeSort {
public static List<Integer> serialSort(List<Integer> array) {
if (array.size() > 1) {
int mid = array.size() / 2;
List<Integer> firstHalf = new ArrayList<>(mid);
List<Integer> secondHalf = new ArrayList<>(array.size() - mid);
for (int i = 0; i < array.size(); i++) {
if (i <= mid) {
firstHalf.add(array.get(i));
} else {
secondHalf.add(array.get(i));
}
}
serialSort(firstHalf);
serialSort(secondHalf);
merge(firstHalf, secondHalf, array);
}
return array;
}
//
private static void merge(List firstHalf, List secondHalf, List array) {
int currentIndexFirst = 0;
int currentIndexSecond = 0;
int currentIndexArray = 0;
while (currentIndexFirst < firstHalf.size() && currentIndexSecond < secondHalf.size()) {
if (firstHalf.get(currentIndexFirst) < secondHalf.get(currentIndexSecond)) {
array.set(currentIndexArray, firstHalf.get(currentIndexFirst));
currentIndexArray++;
currentIndexFirst++;
} else {
array.set(currentIndexArray, secondHalf.get(currentIndexSecond));
currentIndexArray++;
currentIndexSecond++;
}
}
while (currentIndexFirst < firstHalf.size()) {
array.set(currentIndexArray, firstHalf.get(currentIndexFirst));
currentIndexArray++;
currentIndexFirst++;
}
while (currentIndexSecond < secondHalf.size()) {
array.set(currentIndexArray, secondHalf.get(currentIndexSecond));
currentIndexArray++;
currentIndexSecond++;
}
}
public static JavaRDD<Integer> parallelSort(JavaRDD<Integer> data) {
SparkConf conf = new SparkConf().setAppName("surya").setMaster("local[6]");
JavaSparkContext sc = new JavaSparkContext(conf);
JavaRDD<Integer> distData = (JavaRDD<Integer>) sc.parallelize(data);
return data.union(data);
}
}
public class Main {
public static void main(String[] args) {
List<Integer> unsorted = new ArrayList<>(10000);
for (int i = 0; i < 10000; i++) {
unsorted.add((int) Math.floor(Math.random() * 100000));
}
List<Integer> sorted = new ArrayList<>(unsorted);
sorted.sort(Integer::compareTo);
long startTime = System.currentTimeMillis();
List<Integer> serialSorted = MergeSort.serialSort(new ArrayList<>(unsorted));
long serialTime = System.currentTimeMillis() - startTime;
System.out.println("Unsorted serial list: " + unsorted +"\n");
System.out.println("Sorted serial list: " + sorted+"\n");
System.out.println("Serial implementation took " + serialTime + " milliseconds.");
assert serialSorted.equals(sorted);
// parallelization
// SparkConf conf = new SparkConf().setAppName("surya").setMaster("local[6]");
// JavaSparkContext sc = new JavaSparkContext(conf);
JavaRDD<Integer> data = sc.parallelize(new ArrayList<>(unsorted));
startTime = System.currentTimeMillis();
data = MergeSort.parallelSort(data);
long parallelTime = System.currentTimeMillis() - startTime;
System.out.println("Parallel implementation took " + parallelTime + " milliseconds.");
List<Integer> parallelSorted = data.collect();
sc.stop();
assert parallelSorted.equals(sorted);
}
}
I tried to parallelize this but succeeded. Any help is appreciated.