diff --git a/beaver/config.py b/beaver/config.py index bb37d150..8c46f02c 100644 --- a/beaver/config.py +++ b/beaver/config.py @@ -87,6 +87,7 @@ def __init__(self, args, logger=None): 'rabbitmq_delivery_mode': 1, 'redis_url': os.environ.get('REDIS_URL', 'redis://localhost:6379/0'), 'redis_namespace': os.environ.get('REDIS_NAMESPACE', 'logstash:beaver'), + 'redis_data_type': os.environ.get('REDIS_DATA_TYPE', 'list'), 'redis_password': '', 'sqs_aws_access_key': '', 'sqs_aws_secret_key': '', diff --git a/beaver/transports/redis_transport.py b/beaver/transports/redis_transport.py index 06062c90..47ae0f49 100644 --- a/beaver/transports/redis_transport.py +++ b/beaver/transports/redis_transport.py @@ -8,6 +8,8 @@ class RedisTransport(BaseTransport): + LIST_DATA_TYPE = 'list' + CHANNEL_DATA_TYPE = 'channel' def __init__(self, beaver_config, logger=None): super(RedisTransport, self).__init__(beaver_config, logger=logger) @@ -26,6 +28,11 @@ def __init__(self, beaver_config, logger=None): self._namespace = beaver_config.get('redis_namespace') self._current_server_index = 0 + self._data_type = beaver_config.get('redis_data_type') + if self._data_type not in [self.LIST_DATA_TYPE, + self.CHANNEL_DATA_TYPE]: + raise TransportException('Unknown Redis data type') + self._check_connections() def _check_connections(self): @@ -75,13 +82,22 @@ def callback(self, filename, lines, **kwargs): namespace = self._namespace self._logger.debug('Got namespace: ' + namespace) + data_type = self._data_type + self._logger.debug('Got data type: ' + data_type) + server = self._get_next_server() self._logger.debug('Got redis server: ' + server['url']) pipeline = server['redis'].pipeline(transaction=False) + callback_map = { + self.LIST_DATA_TYPE: pipeline.rpush, + self.CHANNEL_DATA_TYPE: pipeline.publish, + } + callback_method = callback_map[data_type] + for line in lines: - pipeline.rpush( + callback_method( namespace, self.format(filename, line, timestamp, **kwargs) ) diff --git a/docs/user/usage.rst b/docs/user/usage.rst index 6192634e..27b5ae77 100644 --- a/docs/user/usage.rst +++ b/docs/user/usage.rst @@ -75,6 +75,7 @@ Beaver can optionally get data from a ``configfile`` using the ``-c`` flag. This * rabbitmq_delivery_mode: Default ``1``. Message deliveryMode. 1: non persistent 2: persistent * redis_url: Default ``redis://localhost:6379/0``. Comma separated redis URLs * redis_namespace: Default ``logstash:beaver``. Redis key namespace +* redis_data_type: Default ``list``, but can also be ``channel``. Redis data type used for transporting log messages * sqs_aws_access_key: Can be left blank to use IAM Roles or AWS_ACCESS_KEY_ID environment variable (see: https://github.com/boto/boto#getting-started-with-boto) * sqs_aws_secret_key: Can be left blank to use IAM Roles or AWS_SECRET_ACCESS_KEY environment variable (see: https://github.com/boto/boto#getting-started-with-boto) * sqs_aws_region: Default ``us-east-1``. AWS Region