Snippets Collections
A simple example demonstrating basic Spark SQL features using fictional
data inspired by a paper on determining the optimum length of chopsticks.
Run with:
from __future__ import print_function
from pyspark.sql import SparkSession
from pyspark.sql.types import *
from pyspark.storagelevel import StorageLevel
# Get avg Food pinching effeciency by length
def AvgEffeciencyByLength(df):
    meansDf = df.groupby('ChopstickLength').mean('FoodPinchingEffeciency').orderBy('avg(FoodPinchingEffeciency)',ascending=0)
    return meansDf
# init
spark = SparkSession.builder.appName("Optimum Chopstick").getOrCreate()
sc = spark.sparkContext
input_loc = "s3://llubbe-gdelt-open-data/ChopstickEffeciency/"
# Read input by line
lines = sc.textFile(input_loc)
parts = l: l.split(","))
# Each line is converted to a tuple.
chopstickItems = p: (str(p[0]), float(p[1]), int(p[2]), int(p[3].strip())))
# Define a schema
fields = [StructField("TestID", StringType()),
          StructField("FoodPinchingEffeciency", DoubleType()), 
          StructField("Individual", IntegerType()), 
          StructField("ChopstickLength", IntegerType())]
schema = StructType(fields)
# Apply the schema to the RDD
chopsticksDF = spark.createDataFrame(chopstickItems, schema)
effeciencyByLength = AvgEffeciencyByLength(chopsticksDF)
moar_chopsticksDF =, format="csv", schema=schema)
moar_effeciencyByLength = AvgEffeciencyByLength(moar_chopsticksDF)
split_col = pyspark.sql.functions.split(df['my_str_col'], '-')
df = df.withColumn('NAME1', split_col.getItem(0))
df = df.withColumn('NAME2', split_col.getItem(1))
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName('abc').getOrCreate()

Thu Apr 01 2021 17:59:16 GMT+0000 (UTC)

#aws #emr #spark

Wed Feb 24 2021 17:36:03 GMT+0000 (UTC)

#pyspark #spark #python #etl

Fri Sep 04 2020 05:19:26 GMT+0000 (UTC)

#python #pyspark #spark #spark-session

Save snippets that work with our extensions

Available in the Chrome Web Store Get Firefox Add-on Get VS Code extension