Performance Tuning

# celeborn-env.sh

export CELEBORN_MASTER_MEMORY=4g

export CELEBORN_WORKER_MEMORY=2g

export CELEBORN_WORKER_OFFHEAP_MEMORY=8g

# JVM GC tuning

export CELEBORN_MASTER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200"

export CELEBORN_WORKER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:MaxDirectMemorySize=8g"

celeborn.worker.flusher.buffer.size=256k     # Local disk

celeborn.worker.flusher.hdfs.buffer.size=4m  # HDFS (must be >= 4 MB)

celeborn.worker.flusher.s3.buffer.size=6m    # S3

# Partition sorting and commit (increase for HDFS/S3)

celeborn.worker.sortPartition.threads=64

celeborn.worker.commitFiles.threads=128

# Network and RPC

celeborn.data.io.threads=32

celeborn.rpc.dispatcher.numThreads=32

# High throughput: increase connections per peer

celeborn.data.io.numConnectionsPerPeer=16

celeborn.client.push.maxReqsInFlight=128

# Timeouts (increase for large shuffles or slow storage)

celeborn.rpc.askTimeout=240s

celeborn.worker.commitFiles.timeout=240s

# Hash writer: better for moderate partition counts

spark.celeborn.client.spark.shuffle.writer=hash

spark.celeborn.client.push.buffer.max.size=64k

# Sort writer: better for very high partition counts (> 100k)

spark.celeborn.client.spark.shuffle.writer=sort

# Disable local shuffle reader for consistent performance

spark.sql.adaptive.localShuffleReader.enabled=false