Configuration Reference

# Identity celeborn.master.host=<hostname> celeborn.master.port=9097 celeborn.master.http.port=9098 # Slot assignment (roundrobin or loadaware) celeborn.master.slot.assign.policy=loadaware # Worker health management celeborn.master.heartbeat.timeout=120s celeborn.master.worker.timeout=600s

# Storage — multiple disks with disk type annotation celeborn.worker.storage.dirs=/mnt/nvme1:disktype=SSD,/mnt/nvme2:disktype=SSD,/mnt/hdd1:disktype=HDD celeborn.storage.availableTypes=SSD,HDD # Memory (set in celeborn-env.sh) CELEBORN_WORKER_MEMORY=2g CELEBORN_WORKER_OFFHEAP_MEMORY=4g # Flush buffer sizes celeborn.worker.flusher.buffer.size=256k celeborn.worker.flusher.threads=4 # Threading (increase for HDFS/S3 workloads) celeborn.worker.sortPartition.threads=64 celeborn.worker.commitFiles.threads=128 celeborn.worker.commitFiles.timeout=240s

# Enable Celeborn shuffle manager spark.shuffle.manager=org.apache.spark.shuffle.celeborn.SparkShuffleManager spark.celeborn.master.endpoints=master1:9097,master2:9097,master3:9097 # Disable YARN ESS (required when using Celeborn) spark.shuffle.service.enabled=false # Serialization (required for Celeborn) spark.serializer=org.apache.spark.serializer.KryoSerializer # Replication (recommended for production) spark.celeborn.client.push.replicate.enabled=true # Writer mode: hash (moderate partitions) or sort (high partitions) spark.celeborn.client.spark.shuffle.writer=hash # Dynamic allocation (Spark 3.5+) spark.dynamicAllocation.enabled=true spark.dynamicAllocation.shuffleTracking.enabled=false spark.shuffle.sort.io.plugin.class=org.apache.spark.shuffle.celeborn.CelebornShuffleDataIO