Added some documentation fixes

Author: TarasLevelUp

LICENSE

@@ -1,4 +1,5 @@
-Apache License
+
+                                 Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 
@@ -178,15 +179,15 @@ Apache License
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
       file or class name and description of purpose be included on the
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright 2016 Taras Voinarovskyi
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -198,5 +199,4 @@ Apache License
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
-
+   limitations under the License.

aiokafka/consumer.py

@@ -109,6 +109,12 @@ class AIOKafkaConsumer(object):
             AIOKafkaConsumer supports Kafka API versions >=0.9 only.
             If set to 'auto', will attempt to infer the broker version by
             probing various APIs. Default: auto
+
+
+    Note:
+        Many configuration parameters are taken from Java Client:
+        https://kafka.apache.org/documentation.html#newconsumerconfigs
+
     """
     def __init__(self, *topics, loop,
                  bootstrap_servers='localhost',
@@ -421,8 +427,9 @@ def seek(self, partition, offset):
     @asyncio.coroutine
     def seek_to_committed(self, *partitions):
         """Seek to the committed offset for partitions
+
         Arguments:
-            *partitions: optionally provide specific TopicPartitions,
+            partitions: optionally provide specific TopicPartitions,
                 otherwise default to all assigned partitions
 
         Raises:
@@ -545,16 +552,36 @@ def _on_change_subscription(self):
     def getone(self, *partitions):
         """
         Get one message from Kafka
-        If no new messages occured, this method will wait it
+        If no new messages prefetched, this method will wait for it
 
         Arguments:
-            partitions (List[TopicPartition]): The partitions that need
-                fetching message. If no one partition specified then all
-                subscribed partitions will be used
+            partitions (List[TopicPartition]): Optional list of partitions to
+                return from. If no partitions specified then returned message
+                will be from any partition, which consumer is subscribed to.
 
         Returns:
-            instance of collections.namedtuple("ConsumerRecord",
+            ConsumerRecord
+
+        Will return instance of
+
+        .. code:: python
+
+            collections.namedtuple(
+                "ConsumerRecord",
                 ["topic", "partition", "offset", "key", "value"])
+
+        Example usage:
+
+
+        .. code:: python
+
+            while True:
+                message = yield from consumer.getone()
+                topic = message.topic
+                partition = message.partition
+                # Process message
+                print(message.offset, message.key, message.value)
+
         """
         assert all(map(lambda k: isinstance(k, TopicPartition), partitions))
 
@@ -580,6 +607,20 @@ def getmany(self, *partitions, timeout_ms=0):
         Returns:
             dict: topic to list of records since the last fetch for the
                 subscribed list of topics and partitions
+
+        Example usage:
+
+
+        .. code:: python
+
+            data = yield from consumer.getmany()
+            for tp, messages in data.items():
+                topic = tp.topic
+                partition = tp.partition
+                for message in messages:
+                    # Process message
+                    print(message.offset, message.key, message.value)
+
         """
         assert all(map(lambda k: isinstance(k, TopicPartition), partitions))
 

aiokafka/producer.py

@@ -22,8 +22,8 @@ class AIOKafkaProducer(object):
     """A Kafka client that publishes records to the Kafka cluster.
 
     The producer consists of a pool of buffer space that holds records that
-    haven't yet been transmitted to the server as well as a background I/O
-    thread that is responsible for turning these records into requests and
+    haven't yet been transmitted to the server as well as a background task
+    that is responsible for turning these records into requests and
     transmitting them to the cluster.
 
     The send() method is asynchronous. When called it adds the record to a
@@ -58,7 +58,8 @@ class AIOKafkaProducer(object):
             the leader to have received before considering a request complete.
             This controls the durability of records that are sent. The
             following settings are common:
-            0:  Producer will not wait for any acknowledgment from the server
+
+            0: Producer will not wait for any acknowledgment from the server
                 at all. The message will immediately be added to the socket
                 buffer and considered sent. No guarantee can be made that the
                 server has received the record in this case, and the retries
@@ -74,6 +75,7 @@ class AIOKafkaProducer(object):
                 replicas to acknowledge the record. This guarantees that the
                 record will not be lost as long as at least one in-sync replica
                 remains alive. This is the strongest available guarantee.
+
             If unset, defaults to acks=1.
         compression_type (str): The compression type for all data generated by
             the producer. Valid values are 'gzip', 'snappy', 'lz4', or None.
@@ -110,8 +112,8 @@ class AIOKafkaProducer(object):
             probing various APIs. Default: auto
 
     Note:
-        Configuration parameters are described in more detail at
-        https://kafka.apache.org/090/configuration.html#producerconfigs
+        Many configuration parameters are taken from Java Client:
+        https://kafka.apache.org/documentation.html#producerconfigs
     """
     _PRODUCER_CLIENT_ID_SEQUENCE = 0
 
@@ -175,7 +177,7 @@ def start(self):
 
     @asyncio.coroutine
     def stop(self):
-        """Close all connections to kafka cluser"""
+        """Flush all pending data and close all connections to kafka cluser"""
         if self._closed:
             return
 

docs/conf.py

@@ -57,7 +57,8 @@ def get_version(release):
     'sphinx.ext.autodoc',
     'sphinx.ext.intersphinx',
     'sphinx.ext.viewcode',
-    'sphinxcontrib.asyncio'
+    'sphinxcontrib.asyncio',
+    'sphinx.ext.napoleon'
 ]
 
 intersphinx_mapping = {'python': ('http://docs.python.org/3', None)}

docs/group_consumer.rst

@@ -1,6 +1,6 @@
 
 Example. Group consumer
-======================================
+=======================
 
 Producer:
 

docs/index.rst

@@ -1,27 +1,20 @@
 Welcome to aiokafka's documentation!
 ====================================
 
+.. _GitHub: https://github.com/aio-libs/aiokafka
+.. _kafka-python: https://github.com/dpkp/kafka-python
 .. _asyncio: http://docs.python.org/3.4/library/asyncio.html
 
 **aiokafka** is a client for the Apache Kafka distributed stream processing system using the asyncio_.
+It is based on kafka-python_ library and reuses it's internals for protocol parsing, errors, etc. 
 Client is designed to function much like the official java client, with a sprinkling of pythonic interfaces.
 
 **aiokafka** is used with 0.9 Kafka brokers and supports fully coordinated consumer groups -- i.e., dynamic
 partition assignment to multiple consumers in the same group.
 
 
-Installation
-============
-
-.. code::
-
-   pip3 install aiokafka
-
-.. note:: *aiokafka* requires *python-kafka* library.
-
-
 Getting started
-===============
+---------------
 
 
 AIOKafkaConsumer
@@ -50,7 +43,8 @@ See consumer example:
                     print("error while consuming message: ", err)
 
         loop = asyncio.get_event_loop()
-        consumer = AIOKafkaConsumer('topic1', 'topic2', loop=loop, bootstrap_servers='localhost:1234')
+        consumer = AIOKafkaConsumer(
+            'topic1', 'topic2', loop=loop, bootstrap_servers='localhost:1234')
         loop.run_until_complete(consumer.start())
         c_task = asyncio.async(consume_task(consumer))
         try:
@@ -88,16 +82,95 @@ See producer example:
         loop.close()
 
 
-Compression
-===========
+Installation
+------------
+
+.. code::
+
+   pip3 install aiokafka
+
+.. note:: *aiokafka* requires *python-kafka* library and heavily depands on it.
+
+
+Optional LZ4 install
+++++++++++++++++++++
+
+To enable LZ4 compression/decompression, install lz4tools and xxhash:
+
+>>> pip3 install lz4tools
+>>> pip3 install xxhash
+
+
+Optional Snappy install
++++++++++++++++++++++++
+
+1. Download and build Snappy from http://code.google.com/p/snappy/downloads/list
+
+Ubuntu:
+
+.. code:: bash
+
+    apt-get install libsnappy-dev
 
-aiokafka supports gzip compression/decompression natively. To produce or
-consume lz4 compressed messages, you must install lz4tools and xxhash.
-To enable snappy, install python-snappy (also requires snappy library).
+OSX:
+
+.. code:: bash
+
+    brew install snappy
+
+From Source:
+
+.. code:: bash
+
+    wget http://snappy.googlecode.com/files/snappy-1.0.5.tar.gz
+    tar xzvf snappy-1.0.5.tar.gz
+    cd snappy-1.0.5
+    ./configure
+    make
+    sudo make install
+
+
+2. Install the `python-snappy` module
+
+.. code:: bash
+
+    pip3 install python-snappy
+
+
+
+Source code
+-----------
+
+The project is hosted on GitHub_
+
+Please feel free to file an issue on `bug tracker
+<https://github.com/aio-libs/aiokafka/issues>`_ if you have found a bug
+or have some suggestion for library improvement.
+
+The library uses `Travis <https://travis-ci.org/aio-libs/aiokafka>`_ for
+Continious Integration.
+
+
+Authors and License
+-------------------
+
+The ``aiokafka`` package is Apache 2 licensed and freely available.
+
+Feel free to improve this package and send a pull request to GitHub_.
+
+
+Contents:
 
 .. toctree::
-   :hidden:
    :maxdepth: 2
 
-   API documentation <api>
-   Examples <examples>
+   api
+   examples
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/manual_commit.rst

@@ -2,6 +2,8 @@
 Example. Manual commit
 ======================
 
+When processing data from several
+
 
 Consumer: