X-Git-Url: http://git.vanrenterghem.biz/R/project-using-kafka-in-R.git/blobdiff_plain/ca8babd7ae1bc42d81cd8605528aa01bd713718a..0ce8ac9c84fc8a3f79d46469a9c3c37232bb62f5:/kafkaConsumer.R

diff --git a/kafkaConsumer.R b/kafkaConsumer.R
index 4a77beb..d6b092b 100644
--- a/kafkaConsumer.R
+++ b/kafkaConsumer.R
@@ -15,21 +15,11 @@ topic <- 'accesslogapache'
 # See https://docs.confluent.io/current/kafka-rest/docs/intro.html#produce-and-consume-avro-messages
 
 # Create consumer
-response <- POST(url=paste(kafka_rest_proxy, "consumers", consumer, sep="/"),
-     content_type("application/vnd.kafka.v2+json"),
-     accept("application/vnd.kafka.v2+json"),
-     body=paste0('{"name": "',
-                 consumer_instance,
-                 '", "format": "json", "auto.offset.reset": "earliest"}')
-     )
-fromJSON(content(response, "text"))
+source("R/CreateKafkaConsumer.R")
+consumerDetails <- CreateKafkaConsumer(kafka.rest.proxy = kafka_rest_proxy, consumer = consumer, consumer_instance = consumer_instance)
 
 # Subscribe it to topic
-response <- POST(url=paste(kafka_rest_proxy,
-                           "consumers",
-                           consumer,
-                           "instances",
-                           consumer_instance,
+response <- POST(url=paste(consumerDetails$base_uri,
                            "subscription", sep="/"),
                content_type("application/vnd.kafka.v2+json"),
                body = paste0('{"topics":["',
@@ -39,18 +29,15 @@ response <- POST(url=paste(kafka_rest_proxy,
 response
 
 # Obtain all (or latest) messages on the topic
-messagesJSON <- GET(url = paste(kafka_rest_proxy,"consumers",consumer,"instances",consumer_instance,"records", sep = "/"),
-                accept("application/vnd.kafka.json.v2+json"),
-                encode="json")
-
-messages <- fromJSON(content(messagesJSON,"text"))
+source("R/ConsumeKafkaMessages.R")
+messages <- ConsumeKafkaMessages(consumerDetails$base_uri)
 
 createPlot(messages$value)
 
 Sys.sleep(120)
 
 # Obtain latest messages
-messagesJSON <- GET(url = paste(kafka_rest_proxy,"consumers",consumer,"instances",consumer_instance,"records", sep = "/"),
+messagesJSON <- GET(url = paste(consumerDetails$base_uri,"records", sep = "/"),
                 accept("application/vnd.kafka.json.v2+json"),
                 encode="json")
 
@@ -61,5 +48,5 @@ apachelog <- rbind(messages$value,messages2$value)
 createPlot(apachelog)
 
 # Remove the consumer
-DELETE(url = paste(kafka_rest_proxy,"consumers",consumer,"instances",consumer_instance, sep = "/"),
-       content_type("application/vnd.kafka.v2+json"))
+source("R/DestroyKafkaConsumer.R")
+response <- DestroyKafkaConsumer(consumer.base.uri = consumerDetails$base_uri)