AI 기반 광고 추천 파이프라인에서 스파크 스트리밍의 배포 및 모니터링 전략

஠஠য়੄ҟҊ୶ୌ ӝઓनӏ૑಴दझమইఃఫ୊ ౵੉೐ۄੋߓನইఃఫ୊ ౵੉೐ۄੋݽפఠ݂ইఃఫ୊

஠஠য়੄ҟҊ୶ୌ ݽ؛೟ण߂प೷ ؘ੉ఠ&5- ݽ؛ࢲࡂۘఊ ҟҊ೒ۖಬ

ӝઓ૑಴दझమইఃఫ୊

ӝઓ૑಴दझమইఃఫ୊ ঱য੄౵ಞച HPMBOH TDBMB LPUMJO ߓನ੄౵ಞച BSHP$% LTNBOJGFTU झః݃ೞ٘௏٬ ߓ஖
झ౟ܻ߁౵੉೐ۄੋ੄ݽפఠ݂ࠗ੤ ౵੉೐ۄੋܻפ૑ഛੋয۰਑

नӏ૑಴दझమইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB NPOHPEC TDIFNB DPO
fi H SFQPTJUPSZ TVCNJU 4QBSL+PC LBGLB IEGT DPNNJU ѐߊ੗

౵੉೐ۄੋߓನইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB SFQPTJUPSZ TVCNJU 4QBSL+PC
LBGLB IEGT DPNNJU ѐߊ੗ NPOHPEC TDIFNB DPO fi H

౵੉೐ۄੋߓನইఃఫ୊ // mongodbী ੷੢غয ࢎਊغח ౵੉೐ۄੋ ࢸ੿(JSON) { "name": "dankeRankerTest",
"mode": "dankeRankerDruid", "config": { "sink.bootstrap.servers": "kafka.kakao.com:9092", "sink.topic": "adrec-danke-predictor-druid", "source.bootstrap.servers": "kafka.kakao.com:9092", "source.topics": "adrec-danke-serving" } } // п modeীࢲ ࢎਊؼ Config ੿੄. ೙ࣻ ২ٜ࣌ਸ ࠽؊ ಁఢਸ ഝਊ val definition: ConfigDef.ConfigDef = new ConfigDef.ConfigDef() .define("source.bootstrap.servers") .define("source.topics") .define("sink.bootstrap.servers") .define("sink.topic")

౵੉೐ۄੋߓನইఃఫ୊

౵੉೐ۄੋߓನইఃఫ୊FOUSZQPJOUTI #!/bin/bash echo "Application ID: $NAME" for APP_ID in $APPLICATION_IDS;
do echo "ઙܐ ઺: $APP_ID" yarn application -kill $APP_ID done echo "Submitting spark app" spark3-submit \ --deploy-mode cluster \ --master yarn \ --name ${NAME} \ --driver-memory 2g \ --num-executors ${SPARK_NUM_EXECUTORS} \ ... ࢤۚ echo "Running spark app: $NAME"

౵੉೐ۄੋߓನইఃఫ୊LTZBNM apiVersion: batch/v1 kind: Job metadata: name: danke-predictor-druid spec: template:
spec: containers: - name: adrec-metric-pipeline:v1.0.0 imagePullPolicy: ifNotPresent env: - name: NAME value: "dankePredictorDruid" - name: SPARK_NUM_EXECUTORS value: "100" - name: SPARK_EXECUTOR_CORES value: "2" - name: SPARK_EXECUTOR_MEMORY value: "2g" restartPolicy: Never

౵੉೐ۄੋߓನইఃఫ୊ ࢎղೞنࢲ࠺झ SFQPTJUPSZ 4QBSL+PC NPOHPEC TDIFNB DPO fi H EPDLFS
JNBHF TVCNJU LTEFQMPZ LJOE+PC BENJOXFC

4QBSLTUSVDUVSFETUSFBNJOHݽפఠ݂ইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB SFQPTJUPSZ TVCNJU 4QBSL+PC
LBGLB IEGT DPNNJU ѐߊ੗ NPOHPEC TDIFNB DPO fi H

LBGLB IEGT DPNNJU ѐߊ੗ NPOHPEC TDIFNB DPO fi H ழझథழ޿۽૒୶о

LBGLB IEGT DPNNJU ѐߊ੗ NPOHPEC TDIFNB DPO fi H +PCݽפఠ݂૑಴ࣻ૘

झ౵௼ীࢲઁҕೞח4USFBNJOH2VFSZ-JTUFOFS @Evolving abstract class StreamingQueryListener extends Serializable { import StreamingQueryListener._
// Called when a query is started. def onQueryStarted(event: QueryStartedEvent): Unit // Called when there is some status update (ingestion rate updated, etc.) def onQueryProgress(event: QueryProgressEvent): Unit // Called when a query is stopped, with or without error. def onQueryTerminated(event: QueryTerminatedEvent): Unit }

{ "id" : "be7898d0-895b-4f32-ad73-c07cfad4c2bb", "runId" : "9357ed50-fe79-4dd8-a327-3775663e97ba", "timestamp" : "2024-10-22T01:43:22.739Z",
"processedRowsPerSecond" : 644.6234119782214, "sources" : [ { "description" : "KafkaV2[Subscribe[adrec-click-log]]", "startOffset" : null, "endOffset" : { "adrec-click-log" : { "92" : 1100, // ݃૑݄ਵ۽ ୊ܻ೮؍ ["౵౭࣌ ߣഐ" : "য়೐ࣇ ߣഐ"] ؘ੉ఠ } }, "latestOffset" : { "adrec-click-log" : { "92" : 1000, } }, "numInputRows" : 5683 // ੑ۱ row ѐࣻ }, "sink" : { "description" : "org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaTable@739b9fa4", "numOutputRows" : 5683 // ୹۱ row ѐࣻ } }

{ "id" : "be7898d0-895b-4f32-ad73-c07cfad4c2bb", "runId" : "9357ed50-fe79-4dd8-a327-3775663e97ba", "timestamp" : "2024-10-22T01:43:22.739Z",
"processedRowsPerSecond" : 644.6234119782214, "sources" : [ { "description" : "KafkaV2[Subscribe[adrec-click-log]]", "startOffset" : null, "endOffset" : { "adrec-click-log" : { "92" : 1100, // ݃૑݄ਵ۽ ୊ܻ೮؍ ["౵౭࣌ ߣഐ" : "য়೐ࣇ ߣഐ"] ؘ੉ఠ } }, "latestOffset" : { "adrec-click-log" : { "92" : 1000, } }, "numInputRows" : 5683 // ੑ۱ row ѐࣻ }, "sink" : { "description" : "org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaTable@739b9fa4", "numOutputRows" : 5683 // ୹۱ row ѐࣻ } } ؘ੉ఠ੄JOQVU ؘ੉ఠ੄PVUQVU ழ޿ীࢎਊೡؘ੉ఠ

class StreamListener(bootstrapServers: String, applicationId: String) extends StreamingQueryListener { // ழ޿
੹ਊ ஶगݠ private var offsetCommitConsumer: KafkaConsumer[String, String] = _ // Sparkীࢲ झ౟ܻ߁ ௪ܻо ୊਺ द੘ؼ ٸ ழ޿ ੹ਊ ஶगݠ ࢤࢿ override def onQueryStarted(event: StreamingQueryListener.QueryStartedEvent): Unit = { val props = new Properties() props.put("bootstrap.servers", bootstrapServers) props.put("group.id", applicationId) props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") props.put("enable.auto.commit", "false") offsetCommitConsumer = new KafkaConsumer[String, String](props) } // झ౟ܻ߁ ௪ܻо ઙܐؼ ٸ ழ޿ ੹ਊ ஶगݠ ೧ઁ override def onQueryTerminated(event: StreamingQueryListener.QueryTerminatedEvent): Unit = { if (offsetCommitConsumer != null) { offsetCommitConsumer.close() } }

class StreamListener(bootstrapServers: String, applicationId: String) extends StreamingQueryListener { // ݃੉௼۽
ߓ஖о ՘դ ٍী ݒߣ ഐ୹غח ೣࣻ override def onQueryProgress(event: QueryProgressEvent): Unit = { // ࣗझо 2ѐ ੉࢚੉ۄݶ (0) ਵ۽ ࢶ঱ೞחѱ ইפۄ flatMapਵ۽ ಟ୛ࢲ ࢎਊ೧ঠೣ val endOffsetJson = event.progress.sources(0).endOffset // endOffset਷ json Stringਵ۽ غয ੓਺. ؘ੉ఠܳ ౵य೧ࢲ ೙ਃೠ ؘ੉ఠ݅ ୶୹. val parsedData = new ObjectMapper().readValue(endOffsetJson, classOf[java.util.Map[String, java.util.Map[String, Long]]]) .asScala val offsetRanges = parsedData.flatMap { case (topic, partitionOffsets) => partitionOffsets.asScala.map { case (partition, offset) => new TopicPartition(topic, partition.toInt) -> new OffsetAndMetadata(offset) } } // ழ޿ೡ য়೐ࣇ੉ ઓ੤ೠ׮ݶ ழ޿ ࣻ೯ if (offsetRanges.nonEmpty) { offsetCommitConsumer.commitSync(offsetRanges.asJava) } } +40/ਸ౵य೧ঠೣ

class StreamListener(bootstrapServers: String, applicationId: String) extends StreamingQueryListener { // ݃੉௼۽
ߓ஖о ՘դ ٍী ݒߣ ഐ୹غח ೣࣻ override def onQueryProgress(event: QueryProgressEvent): Unit = { // ࣗझо 2ѐ ੉࢚੉ۄݶ (0) ਵ۽ ࢶ঱ೞחѱ ইפۄ flatMapਵ۽ ಟ୛ࢲ ࢎਊ೧ঠೣ val endOffsetJson = event.progress.sources(0).endOffset // endOffset਷ json Stringਵ۽ غয ੓਺. ؘ੉ఠܳ ౵य೧ࢲ ೙ਃೠ ؘ੉ఠ݅ ୶୹. val parsedData = new ObjectMapper().readValue(endOffsetJson, classOf[java.util.Map[String, java.util.Map[String, Long]]]) .asScala val offsetRanges = parsedData.flatMap { case (topic, partitionOffsets) => partitionOffsets.asScala.map { case (partition, offset) => new TopicPartition(topic, partition.toInt) -> new OffsetAndMetadata(offset) } } // ழ޿ೡ য়೐ࣇ੉ ઓ੤ೠ׮ݶ ழ޿ ࣻ೯ if (offsetRanges.nonEmpty) { offsetCommitConsumer.commitSync(offsetRanges.asJava) } } .BQਸٜ݅Ҋ$PNNJUࣻ೯

4QBSL4USVDUVSFE4USFBNJOH੄ݽפఠ݂

नӏ૑಴दझమইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB NPOHPEC SFQPTJUPSZ TVCNJU
4QBSL+PC LBGLB IEGT DPNNJU ѐߊ੗

AI 기반 광고 추천 파이프라인에서 스파크 스트리밍의 배포 및 모니터링 전략

AI 기반 광고 추천 파이프라인에서 스파크 스트리밍의 배포 및 모니터링 전략

kakao

More Decks by kakao

Other Decks in Programming

Featured

Transcript

஠஠য়੄ҟҊ୶ୌ ӝઓनӏ૑಴दझమইఃఫ୊ ౵੉೐ۄੋߓನইఃఫ୊ ౵੉೐ۄੋݽפఠ݂ইఃఫ୊

஠஠য়੄ҟҊ୶ୌ ӝઓनӏ૑಴दझమইఃఫ୊ ౵੉೐ۄੋߓನইఃఫ୊ ౵੉೐ۄੋݽפఠ݂ইఃఫ୊

஠஠য়੄ҟҊ୶ୌ ݽ؛೟ण߂प೷ ؘ੉ఠ&5- ݽ؛ࢲࡂۘఊ ҟҊ೒ۖಬ

஠஠য়੄ҟҊ୶ୌ ӝઓनӏ૑಴दझమইఃఫ୊ ౵੉೐ۄੋߓನইఃఫ୊ ౵੉೐ۄੋݽפఠ݂ইఃఫ୊

ӝઓ૑಴दझమইఃఫ୊

ӝઓ૑಴दझమইఃఫ୊ ঱য੄౵ಞച HPMBOH TDBMB LPUMJO ߓನ੄౵ಞച BSHP$% LTNBOJGFTU झః݃ೞ٘௏٬ ߓ஖

नӏ૑಴दझమইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB NPOHPEC TDIFNB DPO

஠஠য়੄ҟҊ୶ୌ ӝઓनӏ૑಴दझమইఃఫ୊ ౵੉೐ۄੋߓನইఃఫ୊ ౵੉೐ۄੋݽפఠ݂ইఃఫ୊

౵੉೐ۄੋߓನইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB SFQPTJUPSZ TVCNJU 4QBSL+PC

౵੉೐ۄੋߓನইఃఫ୊ // mongodbী ੷੢غয ࢎਊغח ౵੉೐ۄੋ ࢸ੿(JSON) { "name": "dankeRankerTest",

౵੉೐ۄੋߓನইఃఫ୊

౵੉೐ۄੋߓನইఃఫ୊

౵੉೐ۄੋߓನইఃఫ୊FOUSZQPJOUTI #!/bin/bash echo "Application ID: $NAME" for APP_ID in $APPLICATION_IDS;

౵੉೐ۄੋߓನইఃఫ୊LTZBNM apiVersion: batch/v1 kind: Job metadata: name: danke-predictor-druid spec: template:

౵੉೐ۄੋߓನইఃఫ୊ ࢎղೞنࢲ࠺झ SFQPTJUPSZ 4QBSL+PC NPOHPEC TDIFNB DPO fi H EPDLFS

஠஠য়੄ҟҊ୶ୌ ӝઓनӏ૑಴दझమইఃఫ୊ ౵੉೐ۄੋߓನইఃఫ୊ ౵੉೐ۄੋݽפఠ݂ইఃఫ୊

4QBSLTUSVDUVSFETUSFBNJOHݽפఠ݂ইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB SFQPTJUPSZ TVCNJU 4QBSL+PC

4QBSLTUSVDUVSFETUSFBNJOHݽפఠ݂ইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB SFQPTJUPSZ TVCNJU 4QBSL+PC

4QBSLTUSVDUVSFETUSFBNJOHݽפఠ݂ইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB SFQPTJUPSZ TVCNJU 4QBSL+PC

झ౵௼ীࢲઁҕೞח4USFBNJOH2VFSZ-JTUFOFS @Evolving abstract class StreamingQueryListener extends Serializable { import StreamingQueryListener._

{ "id" : "be7898d0-895b-4f32-ad73-c07cfad4c2bb", "runId" : "9357ed50-fe79-4dd8-a327-3775663e97ba", "timestamp" : "2024-10-22T01:43:22.739Z",

{ "id" : "be7898d0-895b-4f32-ad73-c07cfad4c2bb", "runId" : "9357ed50-fe79-4dd8-a327-3775663e97ba", "timestamp" : "2024-10-22T01:43:22.739Z",

class StreamListener(bootstrapServers: String, applicationId: String) extends StreamingQueryListener { // ழ޿

class StreamListener(bootstrapServers: String, applicationId: String) extends StreamingQueryListener { // ݃੉௼۽

class StreamListener(bootstrapServers: String, applicationId: String) extends StreamingQueryListener { // ݃੉௼۽

4QBSL4USVDUVSFE4USFBNJOH੄ݽפఠ݂

नӏ૑಴दझమইఃఫ୊ ࢎղೞنࢲ࠺झ ݽפఠ݂౵੉೐ۄੋ UT$PLF LBGLB 4FOEFS HSBGBOB NPOHPEC SFQPTJUPSZ TVCNJU

2"

&0%