Skip to content

Commit 47bc6a7

Browse files
author
Dana Powers
committed
Some minor updates to usage docs
1 parent 4c76da9 commit 47bc6a7

File tree

1 file changed

+40
-35
lines changed

1 file changed

+40
-35
lines changed

docs/usage.rst

Lines changed: 40 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,24 @@ SimpleProducer
99
from kafka import SimpleProducer, KafkaClient
1010
1111
# To send messages synchronously
12-
kafka = KafkaClient("localhost:9092")
12+
kafka = KafkaClient('localhost:9092')
1313
producer = SimpleProducer(kafka)
1414
15-
# Note that the application is responsible for encoding messages to type str
16-
producer.send_messages("my-topic", "some message")
17-
producer.send_messages("my-topic", "this method", "is variadic")
15+
# Note that the application is responsible for encoding messages to type bytes
16+
producer.send_messages(b'my-topic', b'some message')
17+
producer.send_messages(b'my-topic', b'this method', b'is variadic')
1818
1919
# Send unicode message
20-
producer.send_messages("my-topic", u'你怎么样?'.encode('utf-8'))
20+
producer.send_messages(b'my-topic', u'你怎么样?'.encode('utf-8'))
21+
22+
Asynchronous Mode
23+
-----------------
24+
25+
.. code:: python
2126
2227
# To send messages asynchronously
23-
# WARNING: current implementation does not guarantee message delivery on failure!
24-
# messages can get dropped! Use at your own risk! Or help us improve with a PR!
2528
producer = SimpleProducer(kafka, async=True)
26-
producer.send_messages("my-topic", "async message")
29+
producer.send_messages(b'my-topic', b'async message')
2730
2831
# To wait for acknowledgements
2932
# ACK_AFTER_LOCAL_WRITE : server will wait till the data is written to
@@ -32,13 +35,12 @@ SimpleProducer
3235
# by all in sync replicas before sending a response
3336
producer = SimpleProducer(kafka, async=False,
3437
req_acks=SimpleProducer.ACK_AFTER_LOCAL_WRITE,
35-
ack_timeout=2000)
36-
37-
response = producer.send_messages("my-topic", "another message")
38+
ack_timeout=2000,
39+
sync_fail_on_error=False)
3840
39-
if response:
40-
print(response[0].error)
41-
print(response[0].offset)
41+
responses = producer.send_messages(b'my-topic', b'another message')
42+
for r in responses:
43+
logging.info(r.offset)
4244
4345
# To send messages in batch. You can use any of the available
4446
# producers for doing this. The following producer will collect
@@ -56,16 +58,21 @@ Keyed messages
5658

5759
.. code:: python
5860
59-
from kafka import (KafkaClient, KeyedProducer, HashedPartitioner,
60-
RoundRobinPartitioner)
61+
from kafka import (
62+
KafkaClient, KeyedProducer,
63+
Murmur2Partitioner, RoundRobinPartitioner)
6164
62-
kafka = KafkaClient("localhost:9092")
65+
kafka = KafkaClient('localhost:9092')
6366
64-
# HashedPartitioner is default
67+
# HashedPartitioner is default (currently uses python hash())
6568
producer = KeyedProducer(kafka)
66-
producer.send_messages("my-topic", "key1", "some message")
67-
producer.send_messages("my-topic", "key2", "this methode")
69+
producer.send_messages(b'my-topic', b'key1', b'some message')
70+
producer.send_messages(b'my-topic', b'key2', b'this methode')
6871
72+
# Murmur2Partitioner attempts to mirror the java client hashing
73+
producer = KeyedProducer(kafka, partitioner=Murmur2Partitioner)
74+
75+
# Or just produce round-robin (or just use SimpleProducer)
6976
producer = KeyedProducer(kafka, partitioner=RoundRobinPartitioner)
7077
7178
@@ -78,18 +85,16 @@ KafkaConsumer
7885
from kafka import KafkaConsumer
7986
8087
# To consume messages
81-
consumer = KafkaConsumer("my-topic",
82-
group_id="my_group",
83-
bootstrap_servers=["localhost:9092"])
88+
consumer = KafkaConsumer('my-topic',
89+
group_id='my_group',
90+
bootstrap_servers=['localhost:9092'])
8491
for message in consumer:
8592
# message value is raw byte string -- decode if necessary!
8693
# e.g., for unicode: `message.value.decode('utf-8')`
8794
print("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
8895
message.offset, message.key,
8996
message.value))
9097
91-
kafka.close()
92-
9398
9499
messages (m) are namedtuples with attributes:
95100

@@ -121,16 +126,16 @@ messages (m) are namedtuples with attributes:
121126
# so it can be included in the next commit
122127
#
123128
# **messages that are not marked w/ task_done currently do not commit!
124-
kafka.task_done(m)
129+
consumer.task_done(m)
125130
126131
# If auto_commit_enable is False, remember to commit() periodically
127-
kafka.commit()
132+
consumer.commit()
128133
129134
# Batch process interface
130135
while True:
131136
for m in kafka.fetch_messages():
132137
process_message(m)
133-
kafka.task_done(m)
138+
consumer.task_done(m)
134139
135140
136141
Configuration settings can be passed to constructor,
@@ -162,13 +167,13 @@ Multiprocess consumer
162167
163168
from kafka import KafkaClient, MultiProcessConsumer
164169
165-
kafka = KafkaClient("localhost:9092")
170+
kafka = KafkaClient('localhost:9092')
166171
167172
# This will split the number of partitions among two processes
168-
consumer = MultiProcessConsumer(kafka, "my-group", "my-topic", num_procs=2)
173+
consumer = MultiProcessConsumer(kafka, b'my-group', b'my-topic', num_procs=2)
169174
170175
# This will spawn processes such that each handles 2 partitions max
171-
consumer = MultiProcessConsumer(kafka, "my-group", "my-topic",
176+
consumer = MultiProcessConsumer(kafka, b'my-group', b'my-topic',
172177
partitions_per_proc=2)
173178
174179
for message in consumer:
@@ -186,14 +191,14 @@ Low level
186191
from kafka.protocol import KafkaProtocol
187192
from kafka.common import ProduceRequest
188193
189-
kafka = KafkaClient("localhost:9092")
194+
kafka = KafkaClient('localhost:9092')
190195
191-
req = ProduceRequest(topic="my-topic", partition=1,
192-
messages=[create_message("some message")])
196+
req = ProduceRequest(topic=b'my-topic', partition=1,
197+
messages=[create_message(b'some message')])
193198
resps = kafka.send_produce_request(payloads=[req], fail_on_error=True)
194199
kafka.close()
195200
196-
resps[0].topic # "my-topic"
201+
resps[0].topic # b'my-topic'
197202
resps[0].partition # 1
198203
resps[0].error # 0 (hopefully)
199204
resps[0].offset # offset of the first message sent in this request

0 commit comments

Comments
 (0)