Custom Functions

-- Register a SQL UDF

CREATE FUNCTION square AS 'org.example.Square';

SELECT square(column_name) FROM table_name;

# Register via Python

class Square:

    def evaluate(self, row):

        return row[0] * row[0]

spark.udaf.register("square", Square())

spark.sql("SELECT square(value) FROM mytable").show()

from pyspark.sql.functions import udf

from pyspark.sql.types import DoubleType

@udf(returnType=DoubleType())

def square(x):

    return float(x * x)

data = [("Alice", 3), ("Bob", 4), ("Charlie", 5)]

df = spark.createDataFrame(data, ["name", "value"])

df.withColumn("squared", square("value")).show()