Share
Embed
Transcript
27 slides
-
profile • ᖒ ༗و / ytake • גࣜձࣾΞΠελΠϧ • PHP,
Hack, Go, Scala • Hadoop, Spark, HBase • twitter https://twitter.com/ex_takezawa • facebook https://www.facebook.com/yuuki.takezawa • github https://github.com/ytake -
Rdkafka • librdkafka(C/C++)ΛPHPͰར༻Մೳʹͨ͠ ΤΫεςϯγϣϯ • High Level API & Row
Level API • PHP5/7ରԠ • librdkafkaͱkafkaͷઃఆΛཧղ͢Δ͜ͱ -
Rdkafka Producer࣮ $rk = new RdKafka\Producer(); $rk->setLogLevel(LOG_DEBUG); $rk->addBrokers(“127.0.0.1"); $topic =
$rk->newTopic("test"); $topic->produce( RD_KAFKA_PARTITION_UA, 0, json_encode(['message' => ‘phpcon']) ); -
Rdkafka Producer࣮ $rk = new RdKafka\Producer(); $rk->setLogLevel(LOG_DEBUG); $rk->addBrokers(“127.0.0.1"); $topic =
$rk->newTopic("test"); $topic->produce( RD_KAFKA_PARTITION_UA, 0, json_encode(['message' => ‘phpcon']) ); $MVTUFSͷ߹ɺ ΧϯϚ۠ΓͰࢦఆ -
Rdkafka Producer࣮ $rk = new RdKafka\Producer(); $rk->setLogLevel(LOG_DEBUG); $rk->addBrokers(“127.0.0.1"); $topic =
$rk->newTopic("test"); $topic->produce( RD_KAFKA_PARTITION_UA, 0, json_encode(['message' => ‘phpcon']) ); $MVTUFSͷ߹ɺ ΧϯϚ۠ΓͰࢦఆ 5PQJD໊ 1BSUJUJPOΛࢦఆ -
Rdkafka Consumer࣮ $conf = new RdKafka\Conf(); $conf->set('group.id', 'myConsumerGroup'); $rk =
new RdKafka\Consumer($conf); $rk->addBrokers("127.0.0.1"); $topicConf = new RdKafka\TopicConf(); $topicConf->set('auto.commit.interval.ms', 100); $topicConf->set('offset.store.method', 'file'); $topicConf->set( 'offset.store.path', sys_get_temp_dir() ); $topicConf->set('auto.offset.reset', 'smallest'); -
Rdkafka Consumer࣮ $conf = new RdKafka\Conf(); $conf->set('group.id', 'myConsumerGroup'); $rk =
new RdKafka\Consumer($conf); $rk->addBrokers("127.0.0.1"); $topicConf = new RdKafka\TopicConf(); $topicConf->set('auto.commit.interval.ms', 100); $topicConf->set('offset.store.method', 'file'); $topicConf->set( 'offset.store.path', sys_get_temp_dir() ); $topicConf->set('auto.offset.reset', 'smallest'); 3PVOE3PCJO 1BSUJUJPOׂ͢Δ ͳΒHSPVQJEΛࢦఆ͠·͠ΐ͏ -
Rdkafka Consumer࣮ $conf = new RdKafka\Conf(); $conf->set('group.id', 'myConsumerGroup'); $rk =
new RdKafka\Consumer($conf); $rk->addBrokers("127.0.0.1"); $topicConf = new RdKafka\TopicConf(); $topicConf->set('auto.commit.interval.ms', 100); $topicConf->set('offset.store.method', 'file'); $topicConf->set( 'offset.store.path', sys_get_temp_dir() ); $topicConf->set('auto.offset.reset', 'smallest'); 3PVOE3PCJO 1BSUJUJPOׂ͢Δ ͳΒHSPVQJEΛࢦఆ͠·͠ΐ͏ $POTVNFSઃఆΛࢦఆ -
Rdkafka Consumer࣮ $topic = $rk->newTopic("test", $topicConf); $topic->consumeStart(0, RD_KAFKA_OFFSET_STORED); while (true)
{ $message = $topic->consume(0, 120*10000); switch ($message->err) { case RD_KAFKA_RESP_ERR_NO_ERROR: var_dump($message); break; case RD_KAFKA_RESP_ERR__PARTITION_EOF: echo "No more messages; will wait for more\n"; break; case RD_KAFKA_RESP_ERR__TIMED_OUT: echo "Timed out\n"; break; default: throw new \Exception($message->errstr(), $message->err); break; } } -
Rdkafka Consumer࣮ $topic = $rk->newTopic("test", $topicConf); $topic->consumeStart(0, RD_KAFKA_OFFSET_STORED); while (true)
{ $message = $topic->consume(0, 120*10000); switch ($message->err) { case RD_KAFKA_RESP_ERR_NO_ERROR: var_dump($message); break; case RD_KAFKA_RESP_ERR__PARTITION_EOF: echo "No more messages; will wait for more\n"; break; case RD_KAFKA_RESP_ERR__TIMED_OUT: echo "Timed out\n"; break; default: throw new \Exception($message->errstr(), $message->err); break; } } Ͳͷ1BSUJUJPOΛར༻͢Δ͔ औಘҐஔͲ͔͜Β͔ -
Rdkafka Consumer࣮ $topic = $rk->newTopic("test", $topicConf); $topic->consumeStart(0, RD_KAFKA_OFFSET_STORED); while (true)
{ $message = $topic->consume(0, 120*10000); switch ($message->err) { case RD_KAFKA_RESP_ERR_NO_ERROR: var_dump($message); break; case RD_KAFKA_RESP_ERR__PARTITION_EOF: echo "No more messages; will wait for more\n"; break; case RD_KAFKA_RESP_ERR__TIMED_OUT: echo "Timed out\n"; break; default: throw new \Exception($message->errstr(), $message->err); break; } } Ͳͷ1BSUJUJPOΛར༻͢Δ͔ औಘҐஔͲ͔͜Β͔ ਖ਼ৗʹड৴ͨ͠߹ʹ ҙͷॲཧΛ࣮ߦ -
ൃల • kafka + Spark Streaming͕ڧྗ͗͢Δ • ؆୯ͳσʔλੳͰ͋ΕprestoͱΈ߹Θͤͯ ଞͷσʔλϕʔεͱ݁߹Ͱ͖Δͷӡ༻ʹྑ͍ •
ࢄγεςϜɾϚΠΫϩαʔϏεʹඞཁෆՄܽͳଘࡏʹ -
·ͱΊ • ϩάऩूҎ֎ʹϚΠΫϩαʔϏεΞʔΩςΫνϟʹ • Kafka Streams, Kafka ConnectͰ͕Δબࢶ • PipelinedbɺKSQLͰσʔλΛఏڙ͢Δ͜ͱ
• Lambda/Kappa ArchitectureʹνϟϨϯδ͠·͠ΐ͏ • Need KSQL REST API Document!