]> git.vanrenterghem.biz Git - R/project-using-kafka-in-R.git/blobdiff - kafkaConsumer.R
Break out DestroyKafkaConsumer function.
[R/project-using-kafka-in-R.git] / kafkaConsumer.R
index 64c4544c21ca09b59762a7c8d08713d3ef8ee661..8b5e6c7f8acee1edb8c53debed53cc37fb002030 100644 (file)
@@ -4,34 +4,52 @@
 
 library(httr)
 library(jsonlite)
+library(dplyr)
+
 kafka_rest_proxy <- "http://localhost:8082"
+consumer <- 'my_json_consumer'
+consumer_instance <- 'my_consumer_instance'
+topic <- 'accesslogapache'
 
 # Create consumer
 # See https://docs.confluent.io/current/kafka-rest/docs/intro.html#produce-and-consume-avro-messages
 
-response <- POST(url=paste(kafka_rest_proxy, "consumers", "my_json_consumer", sep="/"),
-     content_type("application/vnd.kafka.v2+json"),
-     accept("application/vnd.kafka.v2+json"),
-     body='{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}')
-fromJSON(content(response, "text"))
-
-response <- POST(url=paste(kafka_rest_proxy,
-                           "consumers",
-                           "my_json_consumer",
-                           "instances",
-                           "my_consumer_instance",
+# Create consumer
+source("R/CreateKafkaConsumer.R")
+consumerDetails <- CreateKafkaConsumer(kafka.rest.proxy = kafka_rest_proxy, consumer = consumer, consumer_instance = consumer_instance)
+
+# Subscribe it to topic
+response <- POST(url=paste(consumerDetails$base_uri,
                            "subscription", sep="/"),
                content_type("application/vnd.kafka.v2+json"),
-               body = '{"topics":["accesslogapache"]}')
+               body = paste0('{"topics":["',
+                             topic,
+                             '"]}')
+               )
 response
 
-# Obtain all messages on the topic
-messages <- GET(url = paste(kafka_rest_proxy,"consumers","my_json_consumer","instances","my_consumer_instance","records", sep = "/"),
+# Obtain all (or latest) messages on the topic
+messagesJSON <- GET(url = paste(consumerDetails$base_uri,"records", sep = "/"),
                 accept("application/vnd.kafka.json.v2+json"),
                 encode="json")
 
-apachelog <- fromJSON(content(messages,"text"))
+messages <- fromJSON(content(messagesJSON,"text"))
+
+createPlot(messages$value)
+
+Sys.sleep(120)
+
+# Obtain latest messages
+messagesJSON <- GET(url = paste(consumerDetails$base_uri,"records", sep = "/"),
+                accept("application/vnd.kafka.json.v2+json"),
+                encode="json")
+
+messages2 <- fromJSON(content(messagesJSON,"text"))
+
+apachelog <- rbind(messages$value,messages2$value)
+
+createPlot(apachelog)
 
 # Remove the consumer
-DELETE(url = paste(kafka_rest_proxy,"consumers","my_json_consumer","instances","my_consumer_instance", sep = "/"),
-       content_type("application/vnd.kafka.v2+json"))
+source("R/DestroyKafkaConsumer.R")
+response <- DestroyKafkaConsumer(consumer.base.uri = consumerDetails$base_uri)