Snippets Collections
def mkPropEqMatcher[T,V](name:String, extr: T => V)(expVal : V): HavePropertyMatcher[T,V] =
     new HavePropertyMatcher[T, V] {
       def apply(t: T): HavePropertyMatchResult[V] =
         HavePropertyMatchResult(
           extr(t) == expVal,
           name,
           expVal,
           extr(t)
         )
     }
 
   val title = mkPropEqMatcher("title", (_ : Book).title) _
   val author = mkPropEqMatcher("author", (_ : Book).author) _  
val classes = Seq(
  getClass,                   // To get the jar with our own code.
  classOf[mysql.jdbc.Driver]  // To get the connector.
)
val jars = classes.map(_.getProtectionDomain().getCodeSource().getLocation().getPath())
val conf = new SparkConf().setJars(jars)
val columnsToSum = List(col("var1"), col("var2"), col("var3"), col("var4"), col("var5"))

val output = input.withColumn("sums", columnsToSum.reduce(_ + _))
import org.apache.spark.sql.functions._

val sc: SparkContext = ...
val sqlContext = new SQLContext(sc)

import sqlContext.implicits._

val input = sc.parallelize(Seq(
  ("a", 5, 7, 9, 12, 13),
  ("b", 6, 4, 3, 20, 17),
  ("c", 4, 9, 4, 6 , 9),
  ("d", 1, 2, 6, 8 , 1)
)).toDF("ID", "var1", "var2", "var3", "var4", "var5")

val columnsToSum = List(col("var1"), col("var2"), col("var3"), col("var4"), col("var5"))

val output = input.withColumn("sums", columnsToSum.reduce(_ + _))

output.show()
val keysDF = mapTypeDF.select(explode(map_keys($"property"))).distinct()
val keys = keysDF.collect().map(f=>f.get(0))
val keyCols = keys.map(f=> col("property").getItem(f).as(f.toString))
val expandedDf = mapTypeDF.select(col("name") +: keyCols:_*).show(false)
star

Wed Dec 08 2021 17:23:04 GMT+0000 (UTC) https://users.scala-lang.org/t/type-safe-checking-of-arbitary-properties-custom-class-for-without-boilerplate-code/6751/3

#scala #scala-test
star

Tue Aug 10 2021 13:28:39 GMT+0000 (UTC) https://stackoverflow.com/questions/24916852/how-can-i-connect-to-a-postgresql-database-into-apache-spark-using-scala

#scala #spark #postgresql
star

Wed Aug 04 2021 09:11:31 GMT+0000 (UTC) https://stackoverflow.com/questions/37624699/adding-a-column-of-rowsums-across-a-list-of-columns-in-spark-dataframe

#scala #spark
star

Mon Apr 05 2021 18:00:47 GMT+0000 (UTC) https://stackoverflow.com/questions/37624699/adding-a-column-of-rowsums-across-a-list-of-columns-in-spark-dataframe

#scala
star

Mon Apr 05 2021 14:20:34 GMT+0000 (UTC) https://sparkbyexamples.com/spark/spark-convert-map-to-multiple-columns/

#scala

Save snippets that work with our extensions

Available in the Chrome Web Store Get Firefox Add-on Get VS Code extension