Custom Functions

-- Register a SQL UDF CREATE FUNCTION square AS 'org.example.Square'; SELECT square(column_name) FROM table_name;

# Register via Python class Square: def evaluate(self, row): return row[0] * row[0] spark.udaf.register("square", Square()) spark.sql("SELECT square(value) FROM mytable").show()

from pyspark.sql.functions import udf from pyspark.sql.types import DoubleType @udf(returnType=DoubleType()) def square(x): return float(x * x) data = [("Alice", 3), ("Bob", 4), ("Charlie", 5)] df = spark.createDataFrame(data, ["name", "value"]) df.withColumn("squared", square("value")).show()