Performance Tuning

# celeborn-env.sh export CELEBORN_MASTER_MEMORY=4g export CELEBORN_WORKER_MEMORY=2g export CELEBORN_WORKER_OFFHEAP_MEMORY=8g # JVM GC tuning export CELEBORN_MASTER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200" export CELEBORN_WORKER_JAVA_OPTS="-XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:MaxDirectMemorySize=8g"

celeborn.worker.flusher.buffer.size=256k # Local disk celeborn.worker.flusher.hdfs.buffer.size=4m # HDFS (must be >= 4 MB) celeborn.worker.flusher.s3.buffer.size=6m # S3

# Partition sorting and commit (increase for HDFS/S3) celeborn.worker.sortPartition.threads=64 celeborn.worker.commitFiles.threads=128 # Network and RPC celeborn.data.io.threads=32 celeborn.rpc.dispatcher.numThreads=32

# High throughput: increase connections per peer celeborn.data.io.numConnectionsPerPeer=16 celeborn.client.push.maxReqsInFlight=128 # Timeouts (increase for large shuffles or slow storage) celeborn.rpc.askTimeout=240s celeborn.worker.commitFiles.timeout=240s

# Hash writer: better for moderate partition counts spark.celeborn.client.spark.shuffle.writer=hash spark.celeborn.client.push.buffer.max.size=64k # Sort writer: better for very high partition counts (> 100k) spark.celeborn.client.spark.shuffle.writer=sort # Disable local shuffle reader for consistent performance spark.sql.adaptive.localShuffleReader.enabled=false