Performance Tuning

Memory

# celeborn-env.sh
export CELEBORN_MASTER_MEMORY=4g

export CELEBORN_WORKER_MEMORY=2g

export CELEBORN_WORKER_OFFHEAP_MEMORY=8g

# JVM GC tuning
export CELEBORN_MASTER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200"
export CELEBORN_WORKER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:MaxDirectMemorySize=8g"

---

Buffer Tuning

celeborn.worker.flusher.buffer.size=256k # Local disk

celeborn.worker.flusher.hdfs.buffer.size=4m # HDFS (must be >= 4 MB)

celeborn.worker.flusher.s3.buffer.size=6m # S3

---

Threading

# Partition sorting and commit (increase for HDFS/S3)

celeborn.worker.sortPartition.threads=64
celeborn.worker.commitFiles.threads=128
# Network and RPC

celeborn.data.io.threads=32
celeborn.rpc.dispatcher.numThreads=32

---

Network

# High throughput: increase connections per peer

celeborn.data.io.numConnectionsPerPeer=16
celeborn.client.push.maxReqsInFlight=128
# Timeouts (increase for large shuffles or slow storage)

celeborn.rpc.askTimeout=240s
celeborn.worker.commitFiles.timeout=240s

---

Spark Client Tuning

# Hash writer: better for moderate partition counts

spark.celeborn.client.spark.shuffle.writer=hash
spark.celeborn.client.push.buffer.max.size=64k

# Sort writer: better for very high partition counts (> 100k)
spark.celeborn.client.spark.shuffle.writer=sort

# Disable local shuffle reader for consistent performance
spark.sql.adaptive.localShuffleReader.enabled=false