Fixed startnode button showing for startnode

functions/extensions/kafka/docker-compose.yml

@@ -1,28 +1,16 @@
-version: "2"
-
+version: '2'
 services:
   zookeeper:
-    image: docker.io/bitnami/zookeeper:3
+    image: wurstmeister/zookeeper
     ports:
       - "2181:2181"
-    volumes:
-      - "zookeeper_data:/bitnami"
-    environment:
-      - ALLOW_ANONYMOUS_LOGIN=yes
   kafka:
-    image: docker.io/bitnami/kafka:2
+    build: .
     ports:
-      - "9092:9092"
-    volumes:
-      - "kafka_data:/bitnami"
+      - "9092"
     environment:
-      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
-      - ALLOW_PLAINTEXT_LISTENER=yes
-    depends_on:
-      - zookeeper
-
-volumes:
-  zookeeper_data:
-    driver: local
-  kafka_data:
-    driver: local
+      DOCKER_API_VERSION: 1.40
+      KAFKA_ADVERTISED_HOST_NAME: 192.168.193.140
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock

functions/extensions/kafka/kafka_local.py

@@ -11,44 +11,77 @@ shuffle_workflow = os.getenv("SHUFFLE_WORKFLOW")
     
 headers = {"Authorization": "Bearer %s" % shuffle_apikey}
 topic = "workflow_%s" % shuffle_workflow
+group = "testing"
 server = "localhost:9092"
 def produce():
-    print("Starting producer")
-    producer = KafkaProducer(
-        bootstrap_servers=[server],
-        value_serializer=lambda x:
-        json.dumps(x).encode('utf-8')
-    )
-    
-    print("Adding data!")
-    for e in range(15):
-        data = {"some": e, "data": "luuuuul"}
+    try:
+        #group_id=group,
+        #value_serializer=lambda x: json.dumps(x).encode('utf-8')
+        producer = KafkaProducer(
+            bootstrap_servers=[server],
+        )
+    except kafka.errors.NoBrokersAvailable as e:
+        print(f"Error with producer: {e}")
+        sleep(5)
+        produce()
+        return
+    except ValueError as e:
+        print(f"ValuError with producer: {e}")
+        sleep(5)
+        produce()
+        return
+        
+        
+    for i in range(15):
+        producer.send(topic, json.dumps({"some": i, "data": "luuuuul"}))
 
-        try:
-            ret = producer.send(topic, value=data)
-            print(ret.get())
-        except kafka.errors.KafkaTimeoutError as e:
-            print("Kafka error: %s" % e)
-            continue
+    producer.flush()
+    producer.close()
+    exit()
+
+        #print(f"Adding data {data}")
+
+        #try:
+        #    ret = producer.send(topic, value=data)
+        #    print(ret.get())
+        #    producer.flush()
+
+        #    #future = producer.send('foobar', b'another_message')
+        #    #result = future.get(timeout=60)
+        #    #print(result)
+
+        #except kafka.errors.KafkaTimeoutError as e:
+        #    print("Kafka error: %s" % e)
+        #    continue
 
 def consume():
     print("Starting consumer")
+    #group_id=group,
+    #auto_offset_reset="earliest",
+    #enable_auto_commit=True,
     consumer = KafkaConsumer(
         topic,
         bootstrap_servers=[server],
-        auto_offset_reset="earliest",
-        enable_auto_commit=True,
-        value_deserializer=lambda x: json.loads(x.decode('utf-8'))
+        auto_offset_reset='earliest',
+        max_poll_records=2,
     )
+    #value_deserializer=lambda x: json.loads(x.decode('utf-8'))
 
-    print("Getting data")
+    consumer.poll()
+    #consumer.seek_to_beginning()
+
+    print(f"Getting data from topic {topic}")
     for message in consumer:
-        message = message.value
+        #message = message.value
+        #message = message
         print("MSG: ", message)
         ret = requests.post("%s/api/v1/%s/execute" % shuffle_url, headers=headers, data=message)
         print(ret.status_code)
         print(ret.text)
 
 if __name__ == "__main__":
+    if topic == "workflow_None":
+        topic = "testing"
+    print("Starting producer on %s for topic %s" % (server, topic))
     produce()
     #consume()