Spark Jobserver

Spark Jobserver and why you should use it

Transcript

5. curl --data-binary @my-spark-job-allinone.jar jobserver:8090/jars/demo curl -d "" jobserver:8090/contexts/agg?num-cpu-cores=2&memory-per-node=2048m curl -d "input.string = A lazy dog jumped mean dog" \ 'jobserver:8090/jobs?appName=demo&context=agg&classPath=spark.jobserver.WordCountExample' { "status": "STARTED", "result": { "jobId": "a3e0e19d-f10f-4b78-9eb3-e65d63853a74", "context": "agg" } }
6. object WordCountExample extends SparkJob { override def validate(sc: SparkContext, config: Config): SparkJobValidation = { Try(config.getString(“input.string”)) .map(x => SparkJobValid) .getOrElse(SparkJobInvalid(“No input.string”)) } override def runJob(sc: SparkContext, config: Config): Any = { val dd = sc.parallelize(config.getString(“input.string”).split(" ").toSeq) dd.map((_, 1)).reduceByKey(_ + _).collect().toMap } }
7. this.namedRdds.update("french_dictionary", frenchDictionaryRDD) val rdd = this.namedRdds.get[(String, String)]("french_dictionary").get val NamedRDD(frenchDictionaryRDD, _ ,_) = namedObjects.get[NamedRDD[(String, String)]]("rdd:french_dictionary").get val NamedDataFrame(frenchDictionaryDF, _, _) = namedObjects.get[NamedDataFrame]("df:some df").get