From 5e48aac1e252c19d2786799ef84b5db48467a63c Mon Sep 17 00:00:00 2001 From: dvora-h Date: Tue, 12 Jul 2022 19:49:29 +0300 Subject: [PATCH 01/22] Add support for async graph --- redis/commands/graph/__init__.py | 101 +++++- redis/commands/graph/commands.py | 107 +++++- redis/commands/graph/query_result.py | 163 +++++++++ redis/commands/redismodules.py | 10 + tests/test_asyncio/test_graph.py | 506 +++++++++++++++++++++++++++ 5 files changed, 885 insertions(+), 2 deletions(-) create mode 100644 tests/test_asyncio/test_graph.py diff --git a/redis/commands/graph/__init__.py b/redis/commands/graph/__init__.py index 3736195007..53b795fcb3 100644 --- a/redis/commands/graph/__init__.py +++ b/redis/commands/graph/__init__.py @@ -1,5 +1,5 @@ from ..helpers import quote_string, random_string, stringify_param_value -from .commands import GraphCommands +from .commands import AsyncGraphCommands, GraphCommands from .edge import Edge # noqa from .node import Node # noqa from .path import Path # noqa @@ -160,3 +160,102 @@ def relationship_types(self): def property_keys(self): return self.call_procedure("db.propertyKeys", read_only=True).result_set + + +class AsyncGraph(Graph, AsyncGraphCommands): + """Async version for Graph""" + + async def _refresh_labels(self): + lbls = await self.labels() + + # Unpack data. + self._labels = [None] * len(lbls) + for i, l in enumerate(lbls): + self._labels[i] = l[0] + + async def _refresh_attributes(self): + props = await self.property_keys() + + # Unpack data. + self._properties = [None] * len(props) + for i, p in enumerate(props): + self._properties[i] = p[0] + + async def _refresh_relations(self): + rels = await self.relationship_types() + + # Unpack data. + self._relationship_types = [None] * len(rels) + for i, r in enumerate(rels): + self._relationship_types[i] = r[0] + + async def get_label(self, idx): + """ + Returns a label by it's index + + Args: + + idx: + The index of the label + """ + try: + label = self._labels[idx] + except IndexError: + # Refresh labels. + await self._refresh_labels() + label = self._labels[idx] + return label + + async def get_property(self, idx): + """ + Returns a property by it's index + + Args: + + idx: + The index of the property + """ + try: + propertie = self._properties[idx] + except IndexError: + # Refresh properties. + await self._refresh_attributes() + propertie = self._properties[idx] + return propertie + + async def get_relation(self, idx): + """ + Returns a relationship type by it's index + + Args: + + idx: + The index of the relation + """ + try: + relationship_type = self._relationship_types[idx] + except IndexError: + # Refresh relationship types. + await self._refresh_relations() + relationship_type = self._relationship_types[idx] + return relationship_type + + async def call_procedure(self, procedure, *args, read_only=False, **kwagrs): + args = [quote_string(arg) for arg in args] + q = f"CALL {procedure}({','.join(args)})" + + y = kwagrs.get("y", None) + if y: + q += f" YIELD {','.join(y)}" + return await self.query(q, read_only=read_only) + + async def labels(self): + return ((await self.call_procedure("db.labels", read_only=True))).result_set + + async def property_keys(self): + return (await self.call_procedure("db.propertyKeys", read_only=True)).result_set + + async def relationship_types(self): + return ( + await self.call_procedure("db.relationshipTypes", read_only=True) + ).result_set diff --git a/redis/commands/graph/commands.py b/redis/commands/graph/commands.py index fe4224b5cf..55b8a9ba0a 100644 --- a/redis/commands/graph/commands.py +++ b/redis/commands/graph/commands.py @@ -3,7 +3,7 @@ from .exceptions import VersionMismatchException from .execution_plan import ExecutionPlan -from .query_result import QueryResult +from .query_result import AsyncQueryResult, QueryResult class GraphCommands: @@ -211,3 +211,108 @@ def explain(self, query, params=None): plan = self.execute_command("GRAPH.EXPLAIN", self.name, query) return ExecutionPlan(plan) + + +class AsyncGraphCommands(GraphCommands): + async def query(self, q, params=None, timeout=None, read_only=False, profile=False): + """ + Executes a query against the graph. + For more information see `GRAPH.QUERY `_. # noqa + + Args: + + q : str + The query. + params : dict + Query parameters. + timeout : int + Maximum runtime for read queries in milliseconds. + read_only : bool + Executes a readonly query if set to True. + profile : bool + Return details on results produced by and time + spent in each operation. + """ + + # maintain original 'q' + query = q + + # handle query parameters + if params is not None: + query = self._build_params_header(params) + query + + # construct query command + # ask for compact result-set format + # specify known graph version + if profile: + cmd = "GRAPH.PROFILE" + else: + cmd = "GRAPH.RO_QUERY" if read_only else "GRAPH.QUERY" + command = [cmd, self.name, query, "--compact"] + + # include timeout is specified + if timeout: + if not isinstance(timeout, int): + raise Exception("Timeout argument must be a positive integer") + command += ["timeout", timeout] + + # issue query + try: + response = await self.execute_command(*command) + return await AsyncQueryResult().initialize(self, response, profile) + except ResponseError as e: + if "wrong number of arguments" in str(e): + print( + "Note: RedisGraph Python requires server version 2.2.8 or above" + ) # noqa + if "unknown command" in str(e) and read_only: + # `GRAPH.RO_QUERY` is unavailable in older versions. + return await self.query(q, params, timeout, read_only=False) + raise e + except VersionMismatchException as e: + # client view over the graph schema is out of sync + # set client version and refresh local schema + self.version = e.version + self._refresh_schema() + # re-issue query + return await self.query(q, params, timeout, read_only) + + async def execution_plan(self, query, params=None): + """ + Get the execution plan for given query, + GRAPH.EXPLAIN returns an array of operations. + + Args: + query: the query that will be executed + params: query parameters + """ + if params is not None: + query = self._build_params_header(params) + query + + plan = await self.execute_command("GRAPH.EXPLAIN", self.name, query) + if isinstance(plan[0], bytes): + plan = [b.decode() for b in plan] + return "\n".join(plan) + + async def explain(self, query, params=None): + """ + Get the execution plan for given query, + GRAPH.EXPLAIN returns ExecutionPlan object. + + Args: + query: the query that will be executed + params: query parameters + """ + if params is not None: + query = self._build_params_header(params) + query + + plan = await self.execute_command("GRAPH.EXPLAIN", self.name, query) + return ExecutionPlan(plan) + + async def flush(self): + """ + Commit the graph and reset the edges and the nodes to zero length. + """ + await self.commit() + self.nodes = {} + self.edges = [] diff --git a/redis/commands/graph/query_result.py b/redis/commands/graph/query_result.py index 644ac5a3db..c560b2eb52 100644 --- a/redis/commands/graph/query_result.py +++ b/redis/commands/graph/query_result.py @@ -360,3 +360,166 @@ def cached_execution(self): @property def run_time_ms(self): return self._get_stat(INTERNAL_EXECUTION_TIME) + + +class AsyncQueryResult(QueryResult): + def __init__(self): + pass + + async def initialize(self, graph, response, profile=False): + self.graph = graph + self.header = [] + self.result_set = [] + + # in case of an error an exception will be raised + self._check_for_errors(response) + + if len(response) == 1: + self.parse_statistics(response[0]) + elif profile: + self.parse_profile(response) + else: + # start by parsing statistics, matches the one we have + self.parse_statistics(response[-1]) # Last element. + await self.parse_results(response) + + return self + + async def parse_node(self, cell): + # Node ID (integer), + # [label string offset (integer)], + # [[name, value type, value] X N] + + node_id = int(cell[0]) + labels = None + if len(cell[1]) > 0: + labels = [] + for inner_label in cell[1]: + labels.append(await self.graph.get_label(inner_label)) + properties = await self.parse_entity_properties(cell[2]) + return Node(node_id=node_id, label=labels, properties=properties) + + async def parse_scalar(self, cell): + scalar_type = int(cell[0]) + value = cell[1] + scalar = None + + if scalar_type == ResultSetScalarTypes.VALUE_NULL: + scalar = None + + elif scalar_type == ResultSetScalarTypes.VALUE_STRING: + scalar = self.parse_string(value) + + elif scalar_type == ResultSetScalarTypes.VALUE_INTEGER: + scalar = int(value) + + elif scalar_type == ResultSetScalarTypes.VALUE_BOOLEAN: + value = value.decode() if isinstance(value, bytes) else value + if value == "true": + scalar = True + elif value == "false": + scalar = False + else: + print("Unknown boolean type\n") + + elif scalar_type == ResultSetScalarTypes.VALUE_DOUBLE: + scalar = float(value) + + elif scalar_type == ResultSetScalarTypes.VALUE_ARRAY: + # array variable is introduced only for readability + scalar = array = value + for i in range(len(array)): + scalar[i] = await self.parse_scalar(array[i]) + + elif scalar_type == ResultSetScalarTypes.VALUE_NODE: + scalar = await self.parse_node(value) + + elif scalar_type == ResultSetScalarTypes.VALUE_EDGE: + scalar = await self.parse_edge(value) + + elif scalar_type == ResultSetScalarTypes.VALUE_PATH: + scalar = await self.parse_path(value) + + elif scalar_type == ResultSetScalarTypes.VALUE_MAP: + scalar = await self.parse_map(value) + + elif scalar_type == ResultSetScalarTypes.VALUE_POINT: + scalar = self.parse_point(value) + + elif scalar_type == ResultSetScalarTypes.VALUE_UNKNOWN: + print("Unknown scalar type\n") + + return scalar + + async def parse_records(self, raw_result_set): + records = [] + result_set = raw_result_set[1] + for row in result_set: + record = [] + for idx, cell in enumerate(row): + if self.header[idx][0] == ResultSetColumnTypes.COLUMN_SCALAR: # noqa + record.append(await self.parse_scalar(cell)) + elif self.header[idx][0] == ResultSetColumnTypes.COLUMN_NODE: # noqa + record.append(await self.parse_node(cell)) + elif ( + self.header[idx][0] == ResultSetColumnTypes.COLUMN_RELATION + ): # noqa + record.append(await self.parse_edge(cell)) + else: + print("Unknown column type.\n") + records.append(record) + + return records + + async def parse_results(self, raw_result_set): + self.header = self.parse_header(raw_result_set) + + # Empty header. + if len(self.header) == 0: + return + + self.result_set = await self.parse_records(raw_result_set) + + async def parse_entity_properties(self, props): + # [[name, value type, value] X N] + properties = {} + for prop in props: + prop_name = await self.graph.get_property(prop[0]) + prop_value = await self.parse_scalar(prop[1:]) + properties[prop_name] = prop_value + + return properties + + async def parse_edge(self, cell): + # Edge ID (integer), + # reltype string offset (integer), + # src node ID offset (integer), + # dest node ID offset (integer), + # [[name, value, value type] X N] + + edge_id = int(cell[0]) + relation = await self.graph.get_relation(cell[1]) + src_node_id = int(cell[2]) + dest_node_id = int(cell[3]) + properties = await self.parse_entity_properties(cell[4]) + return Edge( + src_node_id, relation, dest_node_id, edge_id=edge_id, properties=properties + ) + + async def parse_path(self, cell): + nodes = await self.parse_scalar(cell[0]) + edges = await self.parse_scalar(cell[1]) + return Path(nodes, edges) + + async def parse_map(self, cell): + m = OrderedDict() + n_entries = len(cell) + + # A map is an array of key value pairs. + # 1. key (string) + # 2. array: (value type, value) + for i in range(0, n_entries, 2): + key = self.parse_string(cell[i]) + m[key] = await self.parse_scalar(cell[i + 1]) + + return m diff --git a/redis/commands/redismodules.py b/redis/commands/redismodules.py index 875f3fca25..3882a3d374 100644 --- a/redis/commands/redismodules.py +++ b/redis/commands/redismodules.py @@ -91,3 +91,13 @@ def ft(self, index_name="idx"): s = AsyncSearch(client=self, index_name=index_name) return s + + def graph(self, index_name="idx"): + """Access the timeseries namespace, providing support for + redis timeseries data. + """ + + from .graph import AsyncGraph + + g = AsyncGraph(client=self, name=index_name) + return g diff --git a/tests/test_asyncio/test_graph.py b/tests/test_asyncio/test_graph.py new file mode 100644 index 0000000000..3fdd134e5c --- /dev/null +++ b/tests/test_asyncio/test_graph.py @@ -0,0 +1,506 @@ +import pytest + +import redis.asyncio as redis +from redis.commands.graph import Edge, Node, Path +from redis.commands.graph.execution_plan import Operation +from redis.exceptions import ResponseError +from tests.conftest import skip_if_redis_enterprise + + +pytestmark = pytest.mark.asyncio + + +@pytest.mark.redismod +async def test_bulk(modclient): + with pytest.raises(NotImplementedError): + await modclient.graph().bulk() + await modclient.graph().bulk(foo="bar!") + + +@pytest.mark.redismod +async def test_graph_creation(modclient: redis.Redis): + graph = modclient.graph() + + john = Node( + label="person", + properties={ + "name": "John Doe", + "age": 33, + "gender": "male", + "status": "single", + }, + ) + graph.add_node(john) + japan = Node(label="country", properties={"name": "Japan"}) + + graph.add_node(japan) + edge = Edge(john, "visited", japan, properties={"purpose": "pleasure"}) + graph.add_edge(edge) + + await graph.commit() + + query = ( + 'MATCH (p:person)-[v:visited {purpose:"pleasure"}]->(c:country) ' + "RETURN p, v, c" + ) + + result = await graph.query(query) + + person = result.result_set[0][0] + visit = result.result_set[0][1] + country = result.result_set[0][2] + + assert person == john + assert visit.properties == edge.properties + assert country == japan + + query = """RETURN [1, 2.3, "4", true, false, null]""" + result = await graph.query(query) + assert [1, 2.3, "4", True, False, None] == result.result_set[0][0] + + # All done, remove graph. + await graph.delete() + + +@pytest.mark.redismod +async def test_array_functions(modclient: redis.Redis): + graph = modclient.graph() + + query = """CREATE (p:person{name:'a',age:32, array:[0,1,2]})""" + await graph.query(query) + + query = """WITH [0,1,2] as x return x""" + result = await graph.query(query) + assert [0, 1, 2] == result.result_set[0][0] + + query = """MATCH(n) return collect(n)""" + result = await graph.query(query) + + a = Node( + node_id=0, + label="person", + properties={"name": "a", "age": 32, "array": [0, 1, 2]}, + ) + + assert [a] == result.result_set[0][0] + + +@pytest.mark.redismod +async def test_path(modclient: redis.Redis): + node0 = Node(node_id=0, label="L1") + node1 = Node(node_id=1, label="L1") + edge01 = Edge(node0, "R1", node1, edge_id=0, properties={"value": 1}) + + graph = modclient.graph() + graph.add_node(node0) + graph.add_node(node1) + graph.add_edge(edge01) + await graph.flush() + + path01 = Path.new_empty_path().add_node(node0).add_edge(edge01).add_node(node1) + expected_results = [[path01]] + + query = "MATCH p=(:L1)-[:R1]->(:L1) RETURN p ORDER BY p" + result = await graph.query(query) + assert expected_results == result.result_set + + +@pytest.mark.redismod +async def test_param(modclient: redis.Redis): + params = [1, 2.3, "str", True, False, None, [0, 1, 2]] + query = "RETURN $param" + for param in params: + result = await modclient.graph().query(query, {"param": param}) + expected_results = [[param]] + assert expected_results == result.result_set + + +@pytest.mark.redismod +async def test_map(modclient: redis.Redis): + query = "RETURN {a:1, b:'str', c:NULL, d:[1,2,3], e:True, f:{x:1, y:2}}" + + actual = (await modclient.graph().query(query)).result_set[0][0] + expected = { + "a": 1, + "b": "str", + "c": None, + "d": [1, 2, 3], + "e": True, + "f": {"x": 1, "y": 2}, + } + + assert actual == expected + + +@pytest.mark.redismod +async def test_point(modclient: redis.Redis): + query = "RETURN point({latitude: 32.070794860, longitude: 34.820751118})" + expected_lat = 32.070794860 + expected_lon = 34.820751118 + actual = (await modclient.graph().query(query)).result_set[0][0] + assert abs(actual["latitude"] - expected_lat) < 0.001 + assert abs(actual["longitude"] - expected_lon) < 0.001 + + query = "RETURN point({latitude: 32, longitude: 34.0})" + expected_lat = 32 + expected_lon = 34 + actual = (await modclient.graph().query(query)).result_set[0][0] + assert abs(actual["latitude"] - expected_lat) < 0.001 + assert abs(actual["longitude"] - expected_lon) < 0.001 + + +@pytest.mark.redismod +async def test_index_response(modclient: redis.Redis): + result_set = await modclient.graph().query("CREATE INDEX ON :person(age)") + assert 1 == result_set.indices_created + + result_set = await modclient.graph().query("CREATE INDEX ON :person(age)") + assert 0 == result_set.indices_created + + result_set = await modclient.graph().query("DROP INDEX ON :person(age)") + assert 1 == result_set.indices_deleted + + with pytest.raises(ResponseError): + await modclient.graph().query("DROP INDEX ON :person(age)") + + +@pytest.mark.redismod +async def test_stringify_query_result(modclient: redis.Redis): + graph = modclient.graph() + + john = Node( + alias="a", + label="person", + properties={ + "name": "John Doe", + "age": 33, + "gender": "male", + "status": "single", + }, + ) + graph.add_node(john) + + japan = Node(alias="b", label="country", properties={"name": "Japan"}) + graph.add_node(japan) + + edge = Edge(john, "visited", japan, properties={"purpose": "pleasure"}) + graph.add_edge(edge) + + assert ( + str(john) + == """(a:person{age:33,gender:"male",name:"John Doe",status:"single"})""" # noqa + ) + assert ( + str(edge) + == """(a:person{age:33,gender:"male",name:"John Doe",status:"single"})""" # noqa + + """-[:visited{purpose:"pleasure"}]->""" + + """(b:country{name:"Japan"})""" + ) + assert str(japan) == """(b:country{name:"Japan"})""" + + await graph.commit() + + query = """MATCH (p:person)-[v:visited {purpose:"pleasure"}]->(c:country) + RETURN p, v, c""" + + result = await graph.query(query) + person = result.result_set[0][0] + visit = result.result_set[0][1] + country = result.result_set[0][2] + + assert ( + str(person) + == """(:person{age:33,gender:"male",name:"John Doe",status:"single"})""" # noqa + ) + assert str(visit) == """()-[:visited{purpose:"pleasure"}]->()""" + assert str(country) == """(:country{name:"Japan"})""" + + await graph.delete() + + +@pytest.mark.redismod +async def test_optional_match(modclient: redis.Redis): + # Build a graph of form (a)-[R]->(b) + node0 = Node(node_id=0, label="L1", properties={"value": "a"}) + node1 = Node(node_id=1, label="L1", properties={"value": "b"}) + + edge01 = Edge(node0, "R", node1, edge_id=0) + + graph = modclient.graph() + graph.add_node(node0) + graph.add_node(node1) + graph.add_edge(edge01) + await graph.flush() + + # Issue a query that collects all outgoing edges from both nodes + # (the second has none) + query = """MATCH (a) OPTIONAL MATCH (a)-[e]->(b) RETURN a, e, b ORDER BY a.value""" # noqa + expected_results = [[node0, edge01, node1], [node1, None, None]] + + result = await graph.query(query) + assert expected_results == result.result_set + + await graph.delete() + + +@pytest.mark.redismod +async def test_cached_execution(modclient: redis.Redis): + await modclient.graph().query("CREATE ()") + + uncached_result = await modclient.graph().query( + "MATCH (n) RETURN n, $param", {"param": [0]} + ) + assert uncached_result.cached_execution is False + + # loop to make sure the query is cached on each thread on server + for x in range(0, 64): + cached_result = await modclient.graph().query( + "MATCH (n) RETURN n, $param", {"param": [0]} + ) + assert uncached_result.result_set == cached_result.result_set + + # should be cached on all threads by now + assert cached_result.cached_execution + + +@pytest.mark.redismod +async def test_slowlog(modclient: redis.Redis): + create_query = """CREATE (:Rider {name:'Valentino Rossi'})-[:rides]->(:Team {name:'Yamaha'}), + (:Rider {name:'Dani Pedrosa'})-[:rides]->(:Team {name:'Honda'}), + (:Rider {name:'Andrea Dovizioso'})-[:rides]->(:Team {name:'Ducati'})""" + await modclient.graph().query(create_query) + + results = await modclient.graph().slowlog() + assert results[0][1] == "GRAPH.QUERY" + assert results[0][2] == create_query + + +@pytest.mark.redismod +async def test_query_timeout(modclient: redis.Redis): + # Build a sample graph with 1000 nodes. + await modclient.graph().query("UNWIND range(0,1000) as val CREATE ({v: val})") + # Issue a long-running query with a 1-millisecond timeout. + with pytest.raises(ResponseError): + await modclient.graph().query("MATCH (a), (b), (c), (d) RETURN *", timeout=1) + assert False is False + + with pytest.raises(Exception): + await modclient.graph().query("RETURN 1", timeout="str") + assert False is False + + +@pytest.mark.redismod +async def test_read_only_query(modclient: redis.Redis): + with pytest.raises(Exception): + # Issue a write query, specifying read-only true, + # this call should fail. + await modclient.graph().query("CREATE (p:person {name:'a'})", read_only=True) + assert False is False + + +@pytest.mark.redismod +async def test_profile(modclient: redis.Redis): + q = """UNWIND range(1, 3) AS x CREATE (p:Person {v:x})""" + profile = (await modclient.graph().profile(q)).result_set + assert "Create | Records produced: 3" in profile + assert "Unwind | Records produced: 3" in profile + + q = "MATCH (p:Person) WHERE p.v > 1 RETURN p" + profile = (await modclient.graph().profile(q)).result_set + assert "Results | Records produced: 2" in profile + assert "Project | Records produced: 2" in profile + assert "Filter | Records produced: 2" in profile + assert "Node By Label Scan | (p:Person) | Records produced: 3" in profile + + +@pytest.mark.redismod +@skip_if_redis_enterprise() +async def test_config(modclient: redis.Redis): + config_name = "RESULTSET_SIZE" + config_value = 3 + + # Set configuration + response = await modclient.graph().config(config_name, config_value, set=True) + assert response == "OK" + + # Make sure config been updated. + response = await modclient.graph().config(config_name, set=False) + expected_response = [config_name, config_value] + assert response == expected_response + + config_name = "QUERY_MEM_CAPACITY" + config_value = 1 << 20 # 1MB + + # Set configuration + response = await modclient.graph().config(config_name, config_value, set=True) + assert response == "OK" + + # Make sure config been updated. + response = await modclient.graph().config(config_name, set=False) + expected_response = [config_name, config_value] + assert response == expected_response + + # reset to default + await modclient.graph().config("QUERY_MEM_CAPACITY", 0, set=True) + await modclient.graph().config("RESULTSET_SIZE", -100, set=True) + + +@pytest.mark.redismod +@pytest.mark.onlynoncluster +async def test_list_keys(modclient: redis.Redis): + result = await modclient.graph().list_keys() + assert result == [] + + await modclient.execute_command("GRAPH.EXPLAIN", "G", "RETURN 1") + result = await modclient.graph().list_keys() + assert result == ["G"] + + await modclient.execute_command("GRAPH.EXPLAIN", "X", "RETURN 1") + result = await modclient.graph().list_keys() + assert result == ["G", "X"] + + await modclient.delete("G") + await modclient.rename("X", "Z") + result = await modclient.graph().list_keys() + assert result == ["Z"] + + await modclient.delete("Z") + result = await modclient.graph().list_keys() + assert result == [] + + +@pytest.mark.redismod +async def test_multi_label(modclient: redis.Redis): + redis_graph = modclient.graph("g") + + node = Node(label=["l", "ll"]) + redis_graph.add_node(node) + await redis_graph.commit() + + query = "MATCH (n) RETURN n" + result = await redis_graph.query(query) + result_node = result.result_set[0][0] + assert result_node == node + + try: + Node(label=1) + assert False + except AssertionError: + assert True + + try: + Node(label=["l", 1]) + assert False + except AssertionError: + assert True + + +@pytest.mark.redismod +async def test_execution_plan(modclient: redis.Redis): + redis_graph = modclient.graph("execution_plan") + create_query = """CREATE (:Rider {name:'Valentino Rossi'})-[:rides]->(:Team {name:'Yamaha'}), + (:Rider {name:'Dani Pedrosa'})-[:rides]->(:Team {name:'Honda'}), + (:Rider {name:'Andrea Dovizioso'})-[:rides]->(:Team {name:'Ducati'})""" + await redis_graph.query(create_query) + + result = await redis_graph.execution_plan( + "MATCH (r:Rider)-[:rides]->(t:Team) WHERE t.name = $name RETURN r.name, t.name, $params", # noqa + {"name": "Yehuda"}, + ) + expected = "Results\n Project\n Conditional Traverse | (t:Team)->(r:Rider)\n Filter\n Node By Label Scan | (t:Team)" # noqa + assert result == expected + + await redis_graph.delete() + + +@pytest.mark.redismod +async def test_explain(modclient: redis.Redis): + redis_graph = modclient.graph("execution_plan") + # graph creation / population + create_query = """CREATE +(:Rider {name:'Valentino Rossi'})-[:rides]->(:Team {name:'Yamaha'}), +(:Rider {name:'Dani Pedrosa'})-[:rides]->(:Team {name:'Honda'}), +(:Rider {name:'Andrea Dovizioso'})-[:rides]->(:Team {name:'Ducati'})""" + await redis_graph.query(create_query) + + result = await redis_graph.explain( + """MATCH (r:Rider)-[:rides]->(t:Team) +WHERE t.name = $name +RETURN r.name, t.name +UNION +MATCH (r:Rider)-[:rides]->(t:Team) +WHERE t.name = $name +RETURN r.name, t.name""", + {"name": "Yamaha"}, + ) + expected = """\ +Results +Distinct + Join + Project + Conditional Traverse | (t:Team)->(r:Rider) + Filter + Node By Label Scan | (t:Team) + Project + Conditional Traverse | (t:Team)->(r:Rider) + Filter + Node By Label Scan | (t:Team)""" + assert str(result).replace(" ", "").replace("\n", "") == expected.replace( + " ", "" + ).replace("\n", "") + + expected = Operation("Results").append_child( + Operation("Distinct").append_child( + Operation("Join") + .append_child( + Operation("Project").append_child( + Operation( + "Conditional Traverse", "(t:Team)->(r:Rider)" + ).append_child( + Operation("Filter").append_child( + Operation("Node By Label Scan", "(t:Team)") + ) + ) + ) + ) + .append_child( + Operation("Project").append_child( + Operation( + "Conditional Traverse", "(t:Team)->(r:Rider)" + ).append_child( + Operation("Filter").append_child( + Operation("Node By Label Scan", "(t:Team)") + ) + ) + ) + ) + ) + ) + + assert result.structured_plan == expected + + result = await redis_graph.explain( + """MATCH (r:Rider), (t:Team) + RETURN r.name, t.name""" + ) + expected = """\ +Results +Project + Cartesian Product + Node By Label Scan | (r:Rider) + Node By Label Scan | (t:Team)""" + assert str(result).replace(" ", "").replace("\n", "") == expected.replace( + " ", "" + ).replace("\n", "") + + expected = Operation("Results").append_child( + Operation("Project").append_child( + Operation("Cartesian Product") + .append_child(Operation("Node By Label Scan")) + .append_child(Operation("Node By Label Scan")) + ) + ) + + assert result.structured_plan == expected + + await redis_graph.delete() From f5ee75ef5b5760c3fbb1b409d506010abbf62335 Mon Sep 17 00:00:00 2001 From: dvora-h Date: Tue, 12 Jul 2022 19:55:11 +0300 Subject: [PATCH 02/22] linters --- tests/test_asyncio/test_graph.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_asyncio/test_graph.py b/tests/test_asyncio/test_graph.py index 3fdd134e5c..d4e9293d0d 100644 --- a/tests/test_asyncio/test_graph.py +++ b/tests/test_asyncio/test_graph.py @@ -6,7 +6,6 @@ from redis.exceptions import ResponseError from tests.conftest import skip_if_redis_enterprise - pytestmark = pytest.mark.asyncio From 224778ff6550d57908afa24b2b20eedaed37763c Mon Sep 17 00:00:00 2001 From: dvora-h Date: Wed, 20 Jul 2022 01:28:44 +0300 Subject: [PATCH 03/22] fix docstring --- redis/commands/redismodules.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/redis/commands/redismodules.py b/redis/commands/redismodules.py index 3882a3d374..7e2045a722 100644 --- a/redis/commands/redismodules.py +++ b/redis/commands/redismodules.py @@ -73,8 +73,8 @@ def tdigest(self): return tdigest def graph(self, index_name="idx"): - """Access the timeseries namespace, providing support for - redis timeseries data. + """Access the graph namespace, providing support for + redis graph data. """ from .graph import Graph @@ -93,8 +93,8 @@ def ft(self, index_name="idx"): return s def graph(self, index_name="idx"): - """Access the timeseries namespace, providing support for - redis timeseries data. + """Access the graph namespace, providing support for + redis graph data. """ from .graph import AsyncGraph From fb2c74b92760c3aa22370422e91ec1b8e95ffeee Mon Sep 17 00:00:00 2001 From: szumka <106675199+szumka@users.noreply.github.com> Date: Thu, 21 Jul 2022 14:13:42 +0200 Subject: [PATCH 04/22] Use retry mechanism in async version of Connection objects (#2271) --- CHANGES | 1 + redis/asyncio/connection.py | 6 ++- redis/asyncio/sentinel.py | 8 ++- tests/test_asyncio/test_connection.py | 53 ++++++++++++++++++- .../test_sentinel_managed_connection.py | 37 +++++++++++++ 5 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 tests/test_asyncio/test_sentinel_managed_connection.py diff --git a/CHANGES b/CHANGES index 0af421b4b7..b4841ddb2b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,5 @@ + * Add retry mechanism to async version of Connection * Compare commands case-insensitively in the asyncio command parser * Allow negative `retries` for `Retry` class to retry forever * Add `items` parameter to `hset` signature diff --git a/redis/asyncio/connection.py b/redis/asyncio/connection.py index 35536fc883..5bf2a0fd74 100644 --- a/redis/asyncio/connection.py +++ b/redis/asyncio/connection.py @@ -637,6 +637,8 @@ def __init__( retry_on_error = [] if retry_on_timeout: retry_on_error.append(TimeoutError) + retry_on_error.append(socket.timeout) + retry_on_error.append(asyncio.TimeoutError) self.retry_on_error = retry_on_error if retry_on_error: if not retry: @@ -706,7 +708,9 @@ async def connect(self): if self.is_connected: return try: - await self._connect() + await self.retry.call_with_retry( + lambda: self._connect(), lambda error: self.disconnect() + ) except asyncio.CancelledError: raise except (socket.timeout, asyncio.TimeoutError): diff --git a/redis/asyncio/sentinel.py b/redis/asyncio/sentinel.py index 5aefd09ebd..99c5074950 100644 --- a/redis/asyncio/sentinel.py +++ b/redis/asyncio/sentinel.py @@ -44,7 +44,7 @@ async def connect_to(self, address): if str_if_bytes(await self.read_response()) != "PONG": raise ConnectionError("PING failed") - async def connect(self): + async def _connect_retry(self): if self._reader: return # already connected if self.connection_pool.is_master: @@ -57,6 +57,12 @@ async def connect(self): continue raise SlaveNotFoundError # Never be here + async def connect(self): + return await self.retry.call_with_retry( + self._connect_retry, + lambda error: asyncio.sleep(0), + ) + async def read_response(self, disable_decoding: bool = False): try: return await super().read_response(disable_decoding=disable_decoding) diff --git a/tests/test_asyncio/test_connection.py b/tests/test_asyncio/test_connection.py index f6259adbd2..78a3efd2a0 100644 --- a/tests/test_asyncio/test_connection.py +++ b/tests/test_asyncio/test_connection.py @@ -1,10 +1,18 @@ import asyncio +import socket import types +from unittest.mock import patch import pytest -from redis.asyncio.connection import PythonParser, UnixDomainSocketConnection -from redis.exceptions import InvalidResponse +from redis.asyncio.connection import ( + Connection, + PythonParser, + UnixDomainSocketConnection, +) +from redis.asyncio.retry import Retry +from redis.backoff import NoBackoff +from redis.exceptions import ConnectionError, InvalidResponse, TimeoutError from redis.utils import HIREDIS_AVAILABLE from tests.conftest import skip_if_server_version_lt @@ -60,3 +68,44 @@ async def test_socket_param_regression(r): async def test_can_run_concurrent_commands(r): assert await r.ping() is True assert all(await asyncio.gather(*(r.ping() for _ in range(10)))) + + +async def test_connect_retry_on_timeout_error(): + """Test that the _connect function is retried in case of a timeout""" + conn = Connection(retry_on_timeout=True, retry=Retry(NoBackoff(), 3)) + origin_connect = conn._connect + conn._connect = mock.AsyncMock() + + async def mock_connect(): + # connect only on the last retry + if conn._connect.call_count <= 2: + raise socket.timeout + else: + return await origin_connect() + + conn._connect.side_effect = mock_connect + await conn.connect() + assert conn._connect.call_count == 3 + + +async def test_connect_without_retry_on_os_error(): + """Test that the _connect function is not being retried in case of a OSError""" + with patch.object(Connection, "_connect") as _connect: + _connect.side_effect = OSError("") + conn = Connection(retry_on_timeout=True, retry=Retry(NoBackoff(), 2)) + with pytest.raises(ConnectionError): + await conn.connect() + assert _connect.call_count == 1 + + +async def test_connect_timeout_error_without_retry(): + """Test that the _connect function is not being retried if retry_on_timeout is + set to False""" + conn = Connection(retry_on_timeout=False) + conn._connect = mock.AsyncMock() + conn._connect.side_effect = socket.timeout + + with pytest.raises(TimeoutError) as e: + await conn.connect() + assert conn._connect.call_count == 1 + assert str(e.value) == "Timeout connecting to server" diff --git a/tests/test_asyncio/test_sentinel_managed_connection.py b/tests/test_asyncio/test_sentinel_managed_connection.py new file mode 100644 index 0000000000..a6e9f37a63 --- /dev/null +++ b/tests/test_asyncio/test_sentinel_managed_connection.py @@ -0,0 +1,37 @@ +import socket + +import pytest + +from redis.asyncio.retry import Retry +from redis.asyncio.sentinel import SentinelManagedConnection +from redis.backoff import NoBackoff + +from .compat import mock + +pytestmark = pytest.mark.asyncio + + +async def test_connect_retry_on_timeout_error(): + """Test that the _connect function is retried in case of a timeout""" + connection_pool = mock.AsyncMock() + connection_pool.get_master_address = mock.AsyncMock( + return_value=("localhost", 6379) + ) + conn = SentinelManagedConnection( + retry_on_timeout=True, + retry=Retry(NoBackoff(), 3), + connection_pool=connection_pool, + ) + origin_connect = conn._connect + conn._connect = mock.AsyncMock() + + async def mock_connect(): + # connect only on the last retry + if conn._connect.call_count <= 2: + raise socket.timeout + else: + return await origin_connect() + + conn._connect.side_effect = mock_connect + await conn.connect() + assert conn._connect.call_count == 3 From 439f2eac488cdcbcd898fb05a4582c97cfbc0b3b Mon Sep 17 00:00:00 2001 From: dvora-h <67596500+dvora-h@users.noreply.github.com> Date: Thu, 21 Jul 2022 15:17:14 +0300 Subject: [PATCH 05/22] fix is_connected (#2278) --- redis/asyncio/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis/asyncio/connection.py b/redis/asyncio/connection.py index 5bf2a0fd74..a528abce04 100644 --- a/redis/asyncio/connection.py +++ b/redis/asyncio/connection.py @@ -687,7 +687,7 @@ def __del__(self): @property def is_connected(self): - return self._reader and self._writer + return self._reader is not None and self._writer is not None def register_connect_callback(self, callback): self._connect_callbacks.append(weakref.WeakMethod(callback)) From d7e4ea16a433ba4c29d9ed2d6017ae6dd54bffd2 Mon Sep 17 00:00:00 2001 From: Mehdi ABAAKOUK Date: Sun, 24 Jul 2022 14:33:10 +0200 Subject: [PATCH 06/22] fix: workaround asyncio bug on connection reset by peer (#2259) Fixes #2237 --- redis/asyncio/connection.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/redis/asyncio/connection.py b/redis/asyncio/connection.py index a528abce04..4c75d2f1df 100644 --- a/redis/asyncio/connection.py +++ b/redis/asyncio/connection.py @@ -771,7 +771,16 @@ async def _connect(self): def _error_message(self, exception): # args for socket.error can either be (errno, "message") # or just "message" - if len(exception.args) == 1: + if not exception.args: + # asyncio has a bug where on Connection reset by peer, the + # exception is not instanciated, so args is empty. This is the + # workaround. + # See: https://github.com/redis/redis-py/issues/2237 + # See: https://github.com/python/cpython/issues/94061 + return ( + f"Error connecting to {self.host}:{self.port}. Connection reset by peer" + ) + elif len(exception.args) == 1: return f"Error connecting to {self.host}:{self.port}. {exception.args[0]}." else: return ( From 13941b88dcac812786fb6002f01f4c9a21526748 Mon Sep 17 00:00:00 2001 From: dvora-h <67596500+dvora-h@users.noreply.github.com> Date: Sun, 24 Jul 2022 15:33:43 +0300 Subject: [PATCH 07/22] Fix crash: key expire while search (#2270) * fix expire while search * sleep --- redis/commands/search/result.py | 2 +- tests/test_search.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/redis/commands/search/result.py b/redis/commands/search/result.py index 5f4aca6411..451bf89bb7 100644 --- a/redis/commands/search/result.py +++ b/redis/commands/search/result.py @@ -38,7 +38,7 @@ def __init__( score = float(res[i + 1]) if with_scores else None fields = {} - if hascontent: + if hascontent and res[i + fields_offset] is not None: fields = ( dict( dict( diff --git a/tests/test_search.py b/tests/test_search.py index f0a1190fcb..642418ecdf 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -1698,3 +1698,17 @@ def test_dialect(modclient: redis.Redis): with pytest.raises(redis.ResponseError) as err: modclient.ft().explain(Query("@title:(@num:[0 10])").dialect(2)) assert "Syntax error" in str(err) + + +@pytest.mark.redismod +def test_expire_while_search(modclient: redis.Redis): + modclient.ft().create_index((TextField("txt"),)) + modclient.hset("hset:1", "txt", "a") + modclient.hset("hset:2", "txt", "b") + modclient.hset("hset:3", "txt", "c") + assert 3 == modclient.ft().search(Query("*")).total + modclient.pexpire("hset:2", 300) + for _ in range(500): + modclient.ft().search(Query("*")).docs[1] + time.sleep(1) + assert 2 == modclient.ft().search(Query("*")).total From e993e2c2e2761f75fdc7a2ca8bed31d26e552ad1 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 24 Jul 2022 22:34:26 +1000 Subject: [PATCH 08/22] docs: Fix a few typos (#2274) * docs: Fix a few typos There are small typos in: - redis/cluster.py - redis/commands/core.py - redis/ocsp.py - tests/test_cluster.py Fixes: - Should read `validity` rather than `valididy`. - Should read `reinitialize` rather than `reinitilize`. - Should read `farthest` rather than `farest`. - Should read `commands` rather than `comamnds`. * Update core.py --- redis/cluster.py | 2 +- redis/commands/core.py | 2 +- redis/ocsp.py | 2 +- tests/test_cluster.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/redis/cluster.py b/redis/cluster.py index 6034e9606f..9e773b2190 100644 --- a/redis/cluster.py +++ b/redis/cluster.py @@ -2025,7 +2025,7 @@ def _send_cluster_commands( ) if attempt and allow_redirections: # RETRY MAGIC HAPPENS HERE! - # send these remaing comamnds one at a time using `execute_command` + # send these remaing commands one at a time using `execute_command` # in the main client. This keeps our retry logic # in one place mostly, # and allows us to be more confident in correctness of behavior. diff --git a/redis/commands/core.py b/redis/commands/core.py index 6d67415aab..027d3dbc7c 100644 --- a/redis/commands/core.py +++ b/redis/commands/core.py @@ -5459,7 +5459,7 @@ def geosearch( `m` for meters (the default value), `km` for kilometers, `mi` for miles and `ft` for feet. ``sort`` indicates to return the places in a sorted way, - ASC for nearest to farest and DESC for farest to nearest. + ASC for nearest to furthest and DESC for furthest to nearest. ``count`` limit the results to the first count matching items. ``any`` is set to True, the command will return as soon as enough matches are found. Can't be provided without ``count`` diff --git a/redis/ocsp.py b/redis/ocsp.py index 4753434fba..0d2c0fc453 100644 --- a/redis/ocsp.py +++ b/redis/ocsp.py @@ -292,7 +292,7 @@ def is_valid(self): This first retrieves for validate the certificate, issuer_url, and ocsp_server for certificate validate. Then retrieves the issuer certificate from the issuer_url, and finally checks - the valididy of OCSP revocation status. + the validity of OCSP revocation status. """ # validate the certificate diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 03533230fe..5652673af9 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -672,7 +672,7 @@ def __init__(self, val=0): with patch.object(Redis, "parse_response") as parse_response: def moved_redirect_effect(connection, *args, **options): - # raise a timeout for 5 times so we'll need to reinitilize the topology + # raise a timeout for 5 times so we'll need to reinitialize the topology if count.val == 4: parse_response.side_effect = real_func count.val += 1 From efe7c7a9359fe034f50538c9809402c926eef4f3 Mon Sep 17 00:00:00 2001 From: Utkarsh Gupta Date: Sun, 24 Jul 2022 18:05:01 +0530 Subject: [PATCH 09/22] async_cluster: fix concurrent pipeline (#2280) - each pipeline should create separate stacks for each node --- redis/asyncio/cluster.py | 18 +++++++++--------- tests/test_asyncio/test_cluster.py | 8 ++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/redis/asyncio/cluster.py b/redis/asyncio/cluster.py index 2894004403..3fe3ebc47e 100644 --- a/redis/asyncio/cluster.py +++ b/redis/asyncio/cluster.py @@ -755,7 +755,6 @@ class ClusterNode: """ __slots__ = ( - "_command_stack", "_connections", "_free", "connection_class", @@ -796,7 +795,6 @@ def __init__( self._connections: List[Connection] = [] self._free: Deque[Connection] = collections.deque(maxlen=self.max_connections) - self._command_stack: List["PipelineCommand"] = [] def __repr__(self) -> str: return ( @@ -887,18 +885,18 @@ async def execute_command(self, *args: Any, **kwargs: Any) -> Any: # Release connection self._free.append(connection) - async def execute_pipeline(self) -> bool: + async def execute_pipeline(self, commands: List["PipelineCommand"]) -> bool: # Acquire connection connection = self.acquire_connection() # Execute command await connection.send_packed_command( - connection.pack_commands(cmd.args for cmd in self._command_stack), False + connection.pack_commands(cmd.args for cmd in commands), False ) # Read responses ret = False - for cmd in self._command_stack: + for cmd in commands: try: cmd.result = await self.parse_response( connection, cmd.args[0], **cmd.kwargs @@ -1365,12 +1363,14 @@ async def _execute( node = target_nodes[0] if node.name not in nodes: - nodes[node.name] = node - node._command_stack = [] - node._command_stack.append(cmd) + nodes[node.name] = (node, []) + nodes[node.name][1].append(cmd) errors = await asyncio.gather( - *(asyncio.ensure_future(node.execute_pipeline()) for node in nodes.values()) + *( + asyncio.ensure_future(node[0].execute_pipeline(node[1])) + for node in nodes.values() + ) ) if any(errors): diff --git a/tests/test_asyncio/test_cluster.py b/tests/test_asyncio/test_cluster.py index f4ea5cd7ac..0d0ea33db2 100644 --- a/tests/test_asyncio/test_cluster.py +++ b/tests/test_asyncio/test_cluster.py @@ -2476,3 +2476,11 @@ async def test_readonly_pipeline_from_readonly_client( executed_on_replica = True break assert executed_on_replica + + async def test_can_run_concurrent_pipelines(self, r: RedisCluster) -> None: + """Test that the pipeline can be used concurrently.""" + await asyncio.gather( + *(self.test_redis_cluster_pipeline(r) for i in range(100)), + *(self.test_multi_key_operation_with_a_single_slot(r) for i in range(100)), + *(self.test_multi_key_operation_with_multi_slots(r) for i in range(100)), + ) From b12b025981caba4f3d26c3f6cb45d7fac2f9fce5 Mon Sep 17 00:00:00 2001 From: dvora-h <67596500+dvora-h@users.noreply.github.com> Date: Sun, 24 Jul 2022 15:39:02 +0300 Subject: [PATCH 10/22] Add support for TIMESERIES 1.8 (#2296) * Add support for timeseries 1.8 * fix info * linters * linters * fix info test * type hints * linters --- redis/commands/timeseries/commands.py | 668 +++++++++++++++----------- redis/commands/timeseries/info.py | 18 +- tests/test_timeseries.py | 243 +++++++++- 3 files changed, 648 insertions(+), 281 deletions(-) diff --git a/redis/commands/timeseries/commands.py b/redis/commands/timeseries/commands.py index 3a30c246d5..13e3cdf498 100644 --- a/redis/commands/timeseries/commands.py +++ b/redis/commands/timeseries/commands.py @@ -1,4 +1,7 @@ +from typing import Dict, List, Optional, Tuple, Union + from redis.exceptions import DataError +from redis.typing import KeyT, Number ADD_CMD = "TS.ADD" ALTER_CMD = "TS.ALTER" @@ -22,7 +25,15 @@ class TimeSeriesCommands: """RedisTimeSeries Commands.""" - def create(self, key, **kwargs): + def create( + self, + key: KeyT, + retention_msecs: Optional[int] = None, + uncompressed: Optional[bool] = False, + labels: Optional[Dict[str, str]] = None, + chunk_size: Optional[int] = None, + duplicate_policy: Optional[str] = None, + ): """ Create a new time-series. @@ -31,40 +42,26 @@ def create(self, key, **kwargs): key: time-series key retention_msecs: - Maximum age for samples compared to last event time (in milliseconds). + Maximum age for samples compared to highest reported timestamp (in milliseconds). If None or 0 is passed then the series is not trimmed at all. uncompressed: - Since RedisTimeSeries v1.2, both timestamps and values are - compressed by default. - Adding this flag will keep data in an uncompressed form. - Compression not only saves - memory but usually improve performance due to lower number - of memory accesses. + Changes data storage from compressed (by default) to uncompressed labels: Set of label-value pairs that represent metadata labels of the key. chunk_size: - Each time-serie uses chunks of memory of fixed size for - time series samples. - You can alter the default TSDB chunk size by passing the - chunk_size argument (in Bytes). + Memory size, in bytes, allocated for each data chunk. + Must be a multiple of 8 in the range [128 .. 1048576]. duplicate_policy: - Since RedisTimeSeries v1.4 you can specify the duplicate sample policy - ( Configure what to do on duplicate sample. ) + Policy for handling multiple samples with identical timestamps. Can be one of: - 'block': an error will occur for any out of order sample. - 'first': ignore the new value. - 'last': override with latest value. - 'min': only override if the value is lower than the existing value. - 'max': only override if the value is higher than the existing value. - When this is not set, the server-wide default will be used. - For more information: https://oss.redis.com/redistimeseries/commands/#tscreate + For more information: https://redis.io/commands/ts.create/ """ # noqa - retention_msecs = kwargs.get("retention_msecs", None) - uncompressed = kwargs.get("uncompressed", False) - labels = kwargs.get("labels", {}) - chunk_size = kwargs.get("chunk_size", None) - duplicate_policy = kwargs.get("duplicate_policy", None) params = [key] self._append_retention(params, retention_msecs) self._append_uncompressed(params, uncompressed) @@ -74,29 +71,62 @@ def create(self, key, **kwargs): return self.execute_command(CREATE_CMD, *params) - def alter(self, key, **kwargs): + def alter( + self, + key: KeyT, + retention_msecs: Optional[int] = None, + labels: Optional[Dict[str, str]] = None, + chunk_size: Optional[int] = None, + duplicate_policy: Optional[str] = None, + ): """ - Update the retention, labels of an existing key. - For more information see + Update the retention, chunk size, duplicate policy, and labels of an existing + time series. + + Args: - The parameters are the same as TS.CREATE. + key: + time-series key + retention_msecs: + Maximum retention period, compared to maximal existing timestamp (in milliseconds). + If None or 0 is passed then the series is not trimmed at all. + labels: + Set of label-value pairs that represent metadata labels of the key. + chunk_size: + Memory size, in bytes, allocated for each data chunk. + Must be a multiple of 8 in the range [128 .. 1048576]. + duplicate_policy: + Policy for handling multiple samples with identical timestamps. + Can be one of: + - 'block': an error will occur for any out of order sample. + - 'first': ignore the new value. + - 'last': override with latest value. + - 'min': only override if the value is lower than the existing value. + - 'max': only override if the value is higher than the existing value. - For more information: https://oss.redis.com/redistimeseries/commands/#tsalter + For more information: https://redis.io/commands/ts.alter/ """ # noqa - retention_msecs = kwargs.get("retention_msecs", None) - labels = kwargs.get("labels", {}) - duplicate_policy = kwargs.get("duplicate_policy", None) params = [key] self._append_retention(params, retention_msecs) + self._append_chunk_size(params, chunk_size) self._append_duplicate_policy(params, ALTER_CMD, duplicate_policy) self._append_labels(params, labels) return self.execute_command(ALTER_CMD, *params) - def add(self, key, timestamp, value, **kwargs): + def add( + self, + key: KeyT, + timestamp: Union[int, str], + value: Number, + retention_msecs: Optional[int] = None, + uncompressed: Optional[bool] = False, + labels: Optional[Dict[str, str]] = None, + chunk_size: Optional[int] = None, + duplicate_policy: Optional[str] = None, + ): """ - Append (or create and append) a new sample to the series. - For more information see + Append (or create and append) a new sample to a time series. Args: @@ -107,35 +137,26 @@ def add(self, key, timestamp, value, **kwargs): value: Numeric data value of the sample retention_msecs: - Maximum age for samples compared to last event time (in milliseconds). + Maximum retention period, compared to maximal existing timestamp (in milliseconds). If None or 0 is passed then the series is not trimmed at all. uncompressed: - Since RedisTimeSeries v1.2, both timestamps and values are compressed by default. - Adding this flag will keep data in an uncompressed form. Compression not only saves - memory but usually improve performance due to lower number of memory accesses. + Changes data storage from compressed (by default) to uncompressed labels: Set of label-value pairs that represent metadata labels of the key. chunk_size: - Each time-serie uses chunks of memory of fixed size for time series samples. - You can alter the default TSDB chunk size by passing the chunk_size argument (in Bytes). + Memory size, in bytes, allocated for each data chunk. + Must be a multiple of 8 in the range [128 .. 1048576]. duplicate_policy: - Since RedisTimeSeries v1.4 you can specify the duplicate sample policy - (Configure what to do on duplicate sample). + Policy for handling multiple samples with identical timestamps. Can be one of: - 'block': an error will occur for any out of order sample. - 'first': ignore the new value. - 'last': override with latest value. - 'min': only override if the value is lower than the existing value. - 'max': only override if the value is higher than the existing value. - When this is not set, the server-wide default will be used. - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsadd + For more information: https://redis.io/commands/ts.add/ """ # noqa - retention_msecs = kwargs.get("retention_msecs", None) - uncompressed = kwargs.get("uncompressed", False) - labels = kwargs.get("labels", {}) - chunk_size = kwargs.get("chunk_size", None) - duplicate_policy = kwargs.get("duplicate_policy", None) params = [key, timestamp, value] self._append_retention(params, retention_msecs) self._append_uncompressed(params, uncompressed) @@ -145,28 +166,34 @@ def add(self, key, timestamp, value, **kwargs): return self.execute_command(ADD_CMD, *params) - def madd(self, ktv_tuples): + def madd(self, ktv_tuples: List[Tuple[KeyT, Union[int, str], Number]]): """ Append (or create and append) a new `value` to series `key` with `timestamp`. Expects a list of `tuples` as (`key`,`timestamp`, `value`). Return value is an array with timestamps of insertions. - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsmadd + For more information: https://redis.io/commands/ts.madd/ """ # noqa params = [] for ktv in ktv_tuples: - for item in ktv: - params.append(item) + params.extend(ktv) return self.execute_command(MADD_CMD, *params) - def incrby(self, key, value, **kwargs): + def incrby( + self, + key: KeyT, + value: Number, + timestamp: Optional[Union[int, str]] = None, + retention_msecs: Optional[int] = None, + uncompressed: Optional[bool] = False, + labels: Optional[Dict[str, str]] = None, + chunk_size: Optional[int] = None, + ): """ - Increment (or create an time-series and increment) the latest - sample's of a series. - This command can be used as a counter or gauge that automatically gets - history as a time series. + Increment (or create an time-series and increment) the latest sample's of a series. + This command can be used as a counter or gauge that automatically gets history as a time series. Args: @@ -175,27 +202,19 @@ def incrby(self, key, value, **kwargs): value: Numeric data value of the sample timestamp: - Timestamp of the sample. None can be used for automatic timestamp (using the system clock). + Timestamp of the sample. * can be used for automatic timestamp (using the system clock). retention_msecs: Maximum age for samples compared to last event time (in milliseconds). If None or 0 is passed then the series is not trimmed at all. uncompressed: - Since RedisTimeSeries v1.2, both timestamps and values are compressed by default. - Adding this flag will keep data in an uncompressed form. Compression not only saves - memory but usually improve performance due to lower number of memory accesses. + Changes data storage from compressed (by default) to uncompressed labels: Set of label-value pairs that represent metadata labels of the key. chunk_size: - Each time-series uses chunks of memory of fixed size for time series samples. - You can alter the default TSDB chunk size by passing the chunk_size argument (in Bytes). + Memory size, in bytes, allocated for each data chunk. - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsincrbytsdecrby + For more information: https://redis.io/commands/ts.incrby/ """ # noqa - timestamp = kwargs.get("timestamp", None) - retention_msecs = kwargs.get("retention_msecs", None) - uncompressed = kwargs.get("uncompressed", False) - labels = kwargs.get("labels", {}) - chunk_size = kwargs.get("chunk_size", None) params = [key, value] self._append_timestamp(params, timestamp) self._append_retention(params, retention_msecs) @@ -205,12 +224,19 @@ def incrby(self, key, value, **kwargs): return self.execute_command(INCRBY_CMD, *params) - def decrby(self, key, value, **kwargs): + def decrby( + self, + key: KeyT, + value: Number, + timestamp: Optional[Union[int, str]] = None, + retention_msecs: Optional[int] = None, + uncompressed: Optional[bool] = False, + labels: Optional[Dict[str, str]] = None, + chunk_size: Optional[int] = None, + ): """ - Decrement (or create an time-series and decrement) the - latest sample's of a series. - This command can be used as a counter or gauge that - automatically gets history as a time series. + Decrement (or create an time-series and decrement) the latest sample's of a series. + This command can be used as a counter or gauge that automatically gets history as a time series. Args: @@ -219,31 +245,19 @@ def decrby(self, key, value, **kwargs): value: Numeric data value of the sample timestamp: - Timestamp of the sample. None can be used for automatic - timestamp (using the system clock). + Timestamp of the sample. * can be used for automatic timestamp (using the system clock). retention_msecs: Maximum age for samples compared to last event time (in milliseconds). If None or 0 is passed then the series is not trimmed at all. uncompressed: - Since RedisTimeSeries v1.2, both timestamps and values are - compressed by default. - Adding this flag will keep data in an uncompressed form. - Compression not only saves - memory but usually improve performance due to lower number - of memory accesses. + Changes data storage from compressed (by default) to uncompressed labels: Set of label-value pairs that represent metadata labels of the key. chunk_size: - Each time-series uses chunks of memory of fixed size for time series samples. - You can alter the default TSDB chunk size by passing the chunk_size argument (in Bytes). + Memory size, in bytes, allocated for each data chunk. - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsincrbytsdecrby + For more information: https://redis.io/commands/ts.decrby/ """ # noqa - timestamp = kwargs.get("timestamp", None) - retention_msecs = kwargs.get("retention_msecs", None) - uncompressed = kwargs.get("uncompressed", False) - labels = kwargs.get("labels", {}) - chunk_size = kwargs.get("chunk_size", None) params = [key, value] self._append_timestamp(params, timestamp) self._append_retention(params, retention_msecs) @@ -253,14 +267,9 @@ def decrby(self, key, value, **kwargs): return self.execute_command(DECRBY_CMD, *params) - def delete(self, key, from_time, to_time): + def delete(self, key: KeyT, from_time: int, to_time: int): """ - Delete data points for a given timeseries and interval range - in the form of start and end delete timestamps. - The given timestamp interval is closed (inclusive), meaning start - and end data points will also be deleted. - Return the count for deleted items. - For more information see + Delete all samples between two timestamps for a given time series. Args: @@ -271,68 +280,98 @@ def delete(self, key, from_time, to_time): to_time: End timestamp for the range deletion. - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsdel + For more information: https://redis.io/commands/ts.del/ """ # noqa return self.execute_command(DEL_CMD, key, from_time, to_time) - def createrule(self, source_key, dest_key, aggregation_type, bucket_size_msec): + def createrule( + self, + source_key: KeyT, + dest_key: KeyT, + aggregation_type: str, + bucket_size_msec: int, + align_timestamp: Optional[int] = None, + ): """ Create a compaction rule from values added to `source_key` into `dest_key`. - Aggregating for `bucket_size_msec` where an `aggregation_type` can be - [`avg`, `sum`, `min`, `max`, `range`, `count`, `first`, `last`, - `std.p`, `std.s`, `var.p`, `var.s`] - For more information: https://oss.redis.com/redistimeseries/master/commands/#tscreaterule + Args: + + source_key: + Key name for source time series + dest_key: + Key name for destination (compacted) time series + aggregation_type: + Aggregation type: One of the following: + [`avg`, `sum`, `min`, `max`, `range`, `count`, `first`, `last`, `std.p`, + `std.s`, `var.p`, `var.s`, `twa`] + bucket_size_msec: + Duration of each bucket, in milliseconds + align_timestamp: + Assure that there is a bucket that starts at exactly align_timestamp and + align all other buckets accordingly. + + For more information: https://redis.io/commands/ts.createrule/ """ # noqa params = [source_key, dest_key] self._append_aggregation(params, aggregation_type, bucket_size_msec) + if align_timestamp is not None: + params.append(align_timestamp) return self.execute_command(CREATERULE_CMD, *params) - def deleterule(self, source_key, dest_key): + def deleterule(self, source_key: KeyT, dest_key: KeyT): """ - Delete a compaction rule. - For more information see + Delete a compaction rule from `source_key` to `dest_key`.. - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsdeleterule + For more information: https://redis.io/commands/ts.deleterule/ """ # noqa return self.execute_command(DELETERULE_CMD, source_key, dest_key) def __range_params( self, - key, - from_time, - to_time, - count, - aggregation_type, - bucket_size_msec, - filter_by_ts, - filter_by_min_value, - filter_by_max_value, - align, + key: KeyT, + from_time: Union[int, str], + to_time: Union[int, str], + count: Optional[int], + aggregation_type: Optional[str], + bucket_size_msec: Optional[int], + filter_by_ts: Optional[List[int]], + filter_by_min_value: Optional[int], + filter_by_max_value: Optional[int], + align: Optional[Union[int, str]], + latest: Optional[bool], + bucket_timestamp: Optional[str], + empty: Optional[bool], ): """Create TS.RANGE and TS.REVRANGE arguments.""" params = [key, from_time, to_time] + self._append_latest(params, latest) self._append_filer_by_ts(params, filter_by_ts) self._append_filer_by_value(params, filter_by_min_value, filter_by_max_value) self._append_count(params, count) self._append_align(params, align) self._append_aggregation(params, aggregation_type, bucket_size_msec) + self._append_bucket_timestamp(params, bucket_timestamp) + self._append_empty(params, empty) return params def range( self, - key, - from_time, - to_time, - count=None, - aggregation_type=None, - bucket_size_msec=0, - filter_by_ts=None, - filter_by_min_value=None, - filter_by_max_value=None, - align=None, + key: KeyT, + from_time: Union[int, str], + to_time: Union[int, str], + count: Optional[int] = None, + aggregation_type: Optional[str] = None, + bucket_size_msec: Optional[int] = 0, + filter_by_ts: Optional[List[int]] = None, + filter_by_min_value: Optional[int] = None, + filter_by_max_value: Optional[int] = None, + align: Optional[Union[int, str]] = None, + latest: Optional[bool] = False, + bucket_timestamp: Optional[str] = None, + empty: Optional[bool] = False, ): """ Query a range in forward direction for a specific time-serie. @@ -342,31 +381,34 @@ def range( key: Key name for timeseries. from_time: - Start timestamp for the range query. - can be used to express - the minimum possible timestamp (0). + Start timestamp for the range query. - can be used to express the minimum possible timestamp (0). to_time: - End timestamp for range query, + can be used to express the - maximum possible timestamp. + End timestamp for range query, + can be used to express the maximum possible timestamp. count: - Optional maximum number of returned results. + Limits the number of returned samples. aggregation_type: - Optional aggregation type. Can be one of - [`avg`, `sum`, `min`, `max`, `range`, `count`, - `first`, `last`, `std.p`, `std.s`, `var.p`, `var.s`] + Optional aggregation type. Can be one of [`avg`, `sum`, `min`, `max`, + `range`, `count`, `first`, `last`, `std.p`, `std.s`, `var.p`, `var.s`, `twa`] bucket_size_msec: Time bucket for aggregation in milliseconds. filter_by_ts: List of timestamps to filter the result by specific timestamps. filter_by_min_value: - Filter result by minimum value (must mention also filter - by_max_value). + Filter result by minimum value (must mention also filter by_max_value). filter_by_max_value: - Filter result by maximum value (must mention also filter - by_min_value). + Filter result by maximum value (must mention also filter by_min_value). align: Timestamp for alignment control for aggregation. - - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsrangetsrevrange + latest: + Used when a time series is a compaction, reports the compacted value of the + latest possibly partial bucket + bucket_timestamp: + Controls how bucket timestamps are reported. Can be one of [`-`, `low`, `+`, + `high`, `~`, `mid`]. + empty: + Reports aggregations for empty buckets. + + For more information: https://redis.io/commands/ts.range/ """ # noqa params = self.__range_params( key, @@ -379,21 +421,27 @@ def range( filter_by_min_value, filter_by_max_value, align, + latest, + bucket_timestamp, + empty, ) return self.execute_command(RANGE_CMD, *params) def revrange( self, - key, - from_time, - to_time, - count=None, - aggregation_type=None, - bucket_size_msec=0, - filter_by_ts=None, - filter_by_min_value=None, - filter_by_max_value=None, - align=None, + key: KeyT, + from_time: Union[int, str], + to_time: Union[int, str], + count: Optional[int] = None, + aggregation_type: Optional[str] = None, + bucket_size_msec: Optional[int] = 0, + filter_by_ts: Optional[List[int]] = None, + filter_by_min_value: Optional[int] = None, + filter_by_max_value: Optional[int] = None, + align: Optional[Union[int, str]] = None, + latest: Optional[bool] = False, + bucket_timestamp: Optional[str] = None, + empty: Optional[bool] = False, ): """ Query a range in reverse direction for a specific time-series. @@ -409,10 +457,10 @@ def revrange( to_time: End timestamp for range query, + can be used to express the maximum possible timestamp. count: - Optional maximum number of returned results. + Limits the number of returned samples. aggregation_type: - Optional aggregation type. Can be one of [`avg`, `sum`, `min`, `max`, `range`, `count`, - `first`, `last`, `std.p`, `std.s`, `var.p`, `var.s`] + Optional aggregation type. Can be one of [`avg`, `sum`, `min`, `max`, + `range`, `count`, `first`, `last`, `std.p`, `std.s`, `var.p`, `var.s`, `twa`] bucket_size_msec: Time bucket for aggregation in milliseconds. filter_by_ts: @@ -423,8 +471,16 @@ def revrange( Filter result by maximum value (must mention also filter_by_min_value). align: Timestamp for alignment control for aggregation. - - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsrangetsrevrange + latest: + Used when a time series is a compaction, reports the compacted value of the + latest possibly partial bucket + bucket_timestamp: + Controls how bucket timestamps are reported. Can be one of [`-`, `low`, `+`, + `high`, `~`, `mid`]. + empty: + Reports aggregations for empty buckets. + + For more information: https://redis.io/commands/ts.revrange/ """ # noqa params = self.__range_params( key, @@ -437,34 +493,43 @@ def revrange( filter_by_min_value, filter_by_max_value, align, + latest, + bucket_timestamp, + empty, ) return self.execute_command(REVRANGE_CMD, *params) def __mrange_params( self, - aggregation_type, - bucket_size_msec, - count, - filters, - from_time, - to_time, - with_labels, - filter_by_ts, - filter_by_min_value, - filter_by_max_value, - groupby, - reduce, - select_labels, - align, + aggregation_type: Optional[str], + bucket_size_msec: Optional[int], + count: Optional[int], + filters: List[str], + from_time: Union[int, str], + to_time: Union[int, str], + with_labels: Optional[bool], + filter_by_ts: Optional[List[int]], + filter_by_min_value: Optional[int], + filter_by_max_value: Optional[int], + groupby: Optional[str], + reduce: Optional[str], + select_labels: Optional[List[str]], + align: Optional[Union[int, str]], + latest: Optional[bool], + bucket_timestamp: Optional[str], + empty: Optional[bool], ): """Create TS.MRANGE and TS.MREVRANGE arguments.""" params = [from_time, to_time] + self._append_latest(params, latest) self._append_filer_by_ts(params, filter_by_ts) self._append_filer_by_value(params, filter_by_min_value, filter_by_max_value) + self._append_with_labels(params, with_labels, select_labels) self._append_count(params, count) self._append_align(params, align) self._append_aggregation(params, aggregation_type, bucket_size_msec) - self._append_with_labels(params, with_labels, select_labels) + self._append_bucket_timestamp(params, bucket_timestamp) + self._append_empty(params, empty) params.extend(["FILTER"]) params += filters self._append_groupby_reduce(params, groupby, reduce) @@ -472,20 +537,23 @@ def __mrange_params( def mrange( self, - from_time, - to_time, - filters, - count=None, - aggregation_type=None, - bucket_size_msec=0, - with_labels=False, - filter_by_ts=None, - filter_by_min_value=None, - filter_by_max_value=None, - groupby=None, - reduce=None, - select_labels=None, - align=None, + from_time: Union[int, str], + to_time: Union[int, str], + filters: List[str], + count: Optional[int] = None, + aggregation_type: Optional[str] = None, + bucket_size_msec: Optional[int] = 0, + with_labels: Optional[bool] = False, + filter_by_ts: Optional[List[int]] = None, + filter_by_min_value: Optional[int] = None, + filter_by_max_value: Optional[int] = None, + groupby: Optional[str] = None, + reduce: Optional[str] = None, + select_labels: Optional[List[str]] = None, + align: Optional[Union[int, str]] = None, + latest: Optional[bool] = False, + bucket_timestamp: Optional[str] = None, + empty: Optional[bool] = False, ): """ Query a range across multiple time-series by filters in forward direction. @@ -493,46 +561,45 @@ def mrange( Args: from_time: - Start timestamp for the range query. `-` can be used to - express the minimum possible timestamp (0). + Start timestamp for the range query. `-` can be used to express the minimum possible timestamp (0). to_time: - End timestamp for range query, `+` can be used to express - the maximum possible timestamp. + End timestamp for range query, `+` can be used to express the maximum possible timestamp. filters: filter to match the time-series labels. count: - Optional maximum number of returned results. + Limits the number of returned samples. aggregation_type: - Optional aggregation type. Can be one of - [`avg`, `sum`, `min`, `max`, `range`, `count`, - `first`, `last`, `std.p`, `std.s`, `var.p`, `var.s`] + Optional aggregation type. Can be one of [`avg`, `sum`, `min`, `max`, + `range`, `count`, `first`, `last`, `std.p`, `std.s`, `var.p`, `var.s`, `twa`] bucket_size_msec: Time bucket for aggregation in milliseconds. with_labels: - Include in the reply the label-value pairs that represent metadata - labels of the time-series. - If this argument is not set, by default, an empty Array will be - replied on the labels array position. + Include in the reply all label-value pairs representing metadata labels of the time series. filter_by_ts: List of timestamps to filter the result by specific timestamps. filter_by_min_value: - Filter result by minimum value (must mention also - filter_by_max_value). + Filter result by minimum value (must mention also filter_by_max_value). filter_by_max_value: - Filter result by maximum value (must mention also - filter_by_min_value). + Filter result by maximum value (must mention also filter_by_min_value). groupby: Grouping by fields the results (must mention also reduce). reduce: - Applying reducer functions on each group. Can be one - of [`sum`, `min`, `max`]. + Applying reducer functions on each group. Can be one of [`avg` `sum`, `min`, + `max`, `range`, `count`, `std.p`, `std.s`, `var.p`, `var.s`]. select_labels: - Include in the reply only a subset of the key-value - pair labels of a series. + Include in the reply only a subset of the key-value pair labels of a series. align: Timestamp for alignment control for aggregation. - - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsmrangetsmrevrange + latest: + Used when a time series is a compaction, reports the compacted + value of the latest possibly partial bucket + bucket_timestamp: + Controls how bucket timestamps are reported. Can be one of [`-`, `low`, `+`, + `high`, `~`, `mid`]. + empty: + Reports aggregations for empty buckets. + + For more information: https://redis.io/commands/ts.mrange/ """ # noqa params = self.__mrange_params( aggregation_type, @@ -549,26 +616,32 @@ def mrange( reduce, select_labels, align, + latest, + bucket_timestamp, + empty, ) return self.execute_command(MRANGE_CMD, *params) def mrevrange( self, - from_time, - to_time, - filters, - count=None, - aggregation_type=None, - bucket_size_msec=0, - with_labels=False, - filter_by_ts=None, - filter_by_min_value=None, - filter_by_max_value=None, - groupby=None, - reduce=None, - select_labels=None, - align=None, + from_time: Union[int, str], + to_time: Union[int, str], + filters: List[str], + count: Optional[int] = None, + aggregation_type: Optional[str] = None, + bucket_size_msec: Optional[int] = 0, + with_labels: Optional[bool] = False, + filter_by_ts: Optional[List[int]] = None, + filter_by_min_value: Optional[int] = None, + filter_by_max_value: Optional[int] = None, + groupby: Optional[str] = None, + reduce: Optional[str] = None, + select_labels: Optional[List[str]] = None, + align: Optional[Union[int, str]] = None, + latest: Optional[bool] = False, + bucket_timestamp: Optional[str] = None, + empty: Optional[bool] = False, ): """ Query a range across multiple time-series by filters in reverse direction. @@ -576,48 +649,45 @@ def mrevrange( Args: from_time: - Start timestamp for the range query. - can be used to express - the minimum possible timestamp (0). + Start timestamp for the range query. - can be used to express the minimum possible timestamp (0). to_time: - End timestamp for range query, + can be used to express - the maximum possible timestamp. + End timestamp for range query, + can be used to express the maximum possible timestamp. filters: Filter to match the time-series labels. count: - Optional maximum number of returned results. + Limits the number of returned samples. aggregation_type: - Optional aggregation type. Can be one of - [`avg`, `sum`, `min`, `max`, `range`, `count`, - `first`, `last`, `std.p`, `std.s`, `var.p`, `var.s`] + Optional aggregation type. Can be one of [`avg`, `sum`, `min`, `max`, + `range`, `count`, `first`, `last`, `std.p`, `std.s`, `var.p`, `var.s`, `twa`] bucket_size_msec: Time bucket for aggregation in milliseconds. with_labels: - Include in the reply the label-value pairs that represent - metadata labels - of the time-series. - If this argument is not set, by default, an empty Array - will be replied - on the labels array position. + Include in the reply all label-value pairs representing metadata labels of the time series. filter_by_ts: List of timestamps to filter the result by specific timestamps. filter_by_min_value: - Filter result by minimum value (must mention also filter - by_max_value). + Filter result by minimum value (must mention also filter_by_max_value). filter_by_max_value: - Filter result by maximum value (must mention also filter - by_min_value). + Filter result by maximum value (must mention also filter_by_min_value). groupby: Grouping by fields the results (must mention also reduce). reduce: - Applying reducer functions on each group. Can be one - of [`sum`, `min`, `max`]. + Applying reducer functions on each group. Can be one of [`avg` `sum`, `min`, + `max`, `range`, `count`, `std.p`, `std.s`, `var.p`, `var.s`]. select_labels: - Include in the reply only a subset of the key-value pair - labels of a series. + Include in the reply only a subset of the key-value pair labels of a series. align: Timestamp for alignment control for aggregation. - - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsmrangetsmrevrange + latest: + Used when a time series is a compaction, reports the compacted + value of the latest possibly partial bucket + bucket_timestamp: + Controls how bucket timestamps are reported. Can be one of [`-`, `low`, `+`, + `high`, `~`, `mid`]. + empty: + Reports aggregations for empty buckets. + + For more information: https://redis.io/commands/ts.mrevrange/ """ # noqa params = self.__mrange_params( aggregation_type, @@ -634,54 +704,85 @@ def mrevrange( reduce, select_labels, align, + latest, + bucket_timestamp, + empty, ) return self.execute_command(MREVRANGE_CMD, *params) - def get(self, key): + def get(self, key: KeyT, latest: Optional[bool] = False): """# noqa Get the last sample of `key`. + `latest` used when a time series is a compaction, reports the compacted + value of the latest (possibly partial) bucket - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsget + For more information: https://redis.io/commands/ts.get/ """ # noqa - return self.execute_command(GET_CMD, key) + params = [key] + self._append_latest(params, latest) + return self.execute_command(GET_CMD, *params) - def mget(self, filters, with_labels=False): + def mget( + self, + filters: List[str], + with_labels: Optional[bool] = False, + select_labels: Optional[List[str]] = None, + latest: Optional[bool] = False, + ): """# noqa Get the last samples matching the specific `filter`. - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsmget + Args: + + filters: + Filter to match the time-series labels. + with_labels: + Include in the reply all label-value pairs representing metadata + labels of the time series. + select_labels: + Include in the reply only a subset of the key-value pair labels of a series. + latest: + Used when a time series is a compaction, reports the compacted + value of the latest possibly partial bucket + + For more information: https://redis.io/commands/ts.mget/ """ # noqa params = [] - self._append_with_labels(params, with_labels) + self._append_latest(params, latest) + self._append_with_labels(params, with_labels, select_labels) params.extend(["FILTER"]) params += filters return self.execute_command(MGET_CMD, *params) - def info(self, key): + def info(self, key: KeyT): """# noqa Get information of `key`. - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsinfo + For more information: https://redis.io/commands/ts.info/ """ # noqa return self.execute_command(INFO_CMD, key) - def queryindex(self, filters): + def queryindex(self, filters: List[str]): """# noqa - Get all the keys matching the `filter` list. + Get all time series keys matching the `filter` list. - For more information: https://oss.redis.com/redistimeseries/master/commands/#tsqueryindex + For more information: https://redis.io/commands/ts.queryindex/ """ # noq return self.execute_command(QUERYINDEX_CMD, *filters) @staticmethod - def _append_uncompressed(params, uncompressed): + def _append_uncompressed(params: List[str], uncompressed: Optional[bool]): """Append UNCOMPRESSED tag to params.""" if uncompressed: params.extend(["UNCOMPRESSED"]) @staticmethod - def _append_with_labels(params, with_labels, select_labels=None): + def _append_with_labels( + params: List[str], + with_labels: Optional[bool], + select_labels: Optional[List[str]], + ): """Append labels behavior to params.""" if with_labels and select_labels: raise DataError( @@ -694,19 +795,21 @@ def _append_with_labels(params, with_labels, select_labels=None): params.extend(["SELECTED_LABELS", *select_labels]) @staticmethod - def _append_groupby_reduce(params, groupby, reduce): + def _append_groupby_reduce( + params: List[str], groupby: Optional[str], reduce: Optional[str] + ): """Append GROUPBY REDUCE property to params.""" if groupby is not None and reduce is not None: params.extend(["GROUPBY", groupby, "REDUCE", reduce.upper()]) @staticmethod - def _append_retention(params, retention): + def _append_retention(params: List[str], retention: Optional[int]): """Append RETENTION property to params.""" if retention is not None: params.extend(["RETENTION", retention]) @staticmethod - def _append_labels(params, labels): + def _append_labels(params: List[str], labels: Optional[List[str]]): """Append LABELS property to params.""" if labels: params.append("LABELS") @@ -714,38 +817,43 @@ def _append_labels(params, labels): params.extend([k, v]) @staticmethod - def _append_count(params, count): + def _append_count(params: List[str], count: Optional[int]): """Append COUNT property to params.""" if count is not None: params.extend(["COUNT", count]) @staticmethod - def _append_timestamp(params, timestamp): + def _append_timestamp(params: List[str], timestamp: Optional[int]): """Append TIMESTAMP property to params.""" if timestamp is not None: params.extend(["TIMESTAMP", timestamp]) @staticmethod - def _append_align(params, align): + def _append_align(params: List[str], align: Optional[Union[int, str]]): """Append ALIGN property to params.""" if align is not None: params.extend(["ALIGN", align]) @staticmethod - def _append_aggregation(params, aggregation_type, bucket_size_msec): + def _append_aggregation( + params: List[str], + aggregation_type: Optional[str], + bucket_size_msec: Optional[int], + ): """Append AGGREGATION property to params.""" if aggregation_type is not None: - params.append("AGGREGATION") - params.extend([aggregation_type, bucket_size_msec]) + params.extend(["AGGREGATION", aggregation_type, bucket_size_msec]) @staticmethod - def _append_chunk_size(params, chunk_size): + def _append_chunk_size(params: List[str], chunk_size: Optional[int]): """Append CHUNK_SIZE property to params.""" if chunk_size is not None: params.extend(["CHUNK_SIZE", chunk_size]) @staticmethod - def _append_duplicate_policy(params, command, duplicate_policy): + def _append_duplicate_policy( + params: List[str], command: Optional[str], duplicate_policy: Optional[str] + ): """Append DUPLICATE_POLICY property to params on CREATE and ON_DUPLICATE on ADD. """ @@ -756,13 +864,33 @@ def _append_duplicate_policy(params, command, duplicate_policy): params.extend(["DUPLICATE_POLICY", duplicate_policy]) @staticmethod - def _append_filer_by_ts(params, ts_list): + def _append_filer_by_ts(params: List[str], ts_list: Optional[List[int]]): """Append FILTER_BY_TS property to params.""" if ts_list is not None: params.extend(["FILTER_BY_TS", *ts_list]) @staticmethod - def _append_filer_by_value(params, min_value, max_value): + def _append_filer_by_value( + params: List[str], min_value: Optional[int], max_value: Optional[int] + ): """Append FILTER_BY_VALUE property to params.""" if min_value is not None and max_value is not None: params.extend(["FILTER_BY_VALUE", min_value, max_value]) + + @staticmethod + def _append_latest(params: List[str], latest: Optional[bool]): + """Append LATEST property to params.""" + if latest: + params.append("LATEST") + + @staticmethod + def _append_bucket_timestamp(params: List[str], bucket_timestamp: Optional[str]): + """Append BUCKET_TIMESTAMP property to params.""" + if bucket_timestamp is not None: + params.extend(["BUCKETTIMESTAMP", bucket_timestamp]) + + @staticmethod + def _append_empty(params: List[str], empty: Optional[bool]): + """Append EMPTY property to params.""" + if empty: + params.append("EMPTY") diff --git a/redis/commands/timeseries/info.py b/redis/commands/timeseries/info.py index fba7f093b1..65f3baacd0 100644 --- a/redis/commands/timeseries/info.py +++ b/redis/commands/timeseries/info.py @@ -60,15 +60,15 @@ def __init__(self, args): https://oss.redis.com/redistimeseries/configuration/#duplicate_policy """ response = dict(zip(map(nativestr, args[::2]), args[1::2])) - self.rules = response["rules"] - self.source_key = response["sourceKey"] - self.chunk_count = response["chunkCount"] - self.memory_usage = response["memoryUsage"] - self.total_samples = response["totalSamples"] - self.labels = list_to_dict(response["labels"]) - self.retention_msecs = response["retentionTime"] - self.lastTimeStamp = response["lastTimestamp"] - self.first_time_stamp = response["firstTimestamp"] + self.rules = response.get("rules") + self.source_key = response.get("sourceKey") + self.chunk_count = response.get("chunkCount") + self.memory_usage = response.get("memoryUsage") + self.total_samples = response.get("totalSamples") + self.labels = list_to_dict(response.get("labels")) + self.retention_msecs = response.get("retentionTime") + self.last_timestamp = response.get("lastTimestamp") + self.first_timestamp = response.get("firstTimestamp") if "maxSamplesPerChunk" in response: self.max_samples_per_chunk = response["maxSamplesPerChunk"] self.chunk_size = ( diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py index 7d42147a16..b4b85e1715 100644 --- a/tests/test_timeseries.py +++ b/tests/test_timeseries.py @@ -1,8 +1,11 @@ +import math import time from time import sleep import pytest +import redis + from .conftest import skip_ifmodversion_lt @@ -230,6 +233,84 @@ def test_range_advanced(client): assert [(0, 5.0), (5, 6.0)] == client.ts().range( 1, 0, 10, aggregation_type="count", bucket_size_msec=10, align=5 ) + assert [(0, 2.5500000000000003), (10, 3.95)] == client.ts().range( + 1, 0, 10, aggregation_type="twa", bucket_size_msec=10 + ) + + +@pytest.mark.redismod +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_range_latest(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.create("t2") + timeseries.createrule("t1", "t2", aggregation_type="sum", bucket_size_msec=10) + timeseries.add("t1", 1, 1) + timeseries.add("t1", 2, 3) + timeseries.add("t1", 11, 7) + timeseries.add("t1", 13, 1) + res = timeseries.range("t1", 0, 20) + assert res == [(1, 1.0), (2, 3.0), (11, 7.0), (13, 1.0)] + res = timeseries.range("t2", 0, 10) + assert res == [(0, 4.0)] + res = timeseries.range("t2", 0, 10, latest=True) + assert res == [(0, 4.0), (10, 8.0)] + res = timeseries.range("t2", 0, 9, latest=True) + assert res == [(0, 4.0)] + + +@pytest.mark.redismod +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_range_bucket_timestamp(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.add("t1", 15, 1) + timeseries.add("t1", 17, 4) + timeseries.add("t1", 51, 3) + timeseries.add("t1", 73, 5) + timeseries.add("t1", 75, 3) + assert [(10, 4.0), (50, 3.0), (70, 5.0)] == timeseries.range( + "t1", 0, 100, align=0, aggregation_type="max", bucket_size_msec=10 + ) + assert [(20, 4.0), (60, 3.0), (80, 5.0)] == timeseries.range( + "t1", + 0, + 100, + align=0, + aggregation_type="max", + bucket_size_msec=10, + bucket_timestamp="+", + ) + + +@pytest.mark.redismod +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_range_empty(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.add("t1", 15, 1) + timeseries.add("t1", 17, 4) + timeseries.add("t1", 51, 3) + timeseries.add("t1", 73, 5) + timeseries.add("t1", 75, 3) + assert [(10, 4.0), (50, 3.0), (70, 5.0)] == timeseries.range( + "t1", 0, 100, align=0, aggregation_type="max", bucket_size_msec=10 + ) + res = timeseries.range( + "t1", 0, 100, align=0, aggregation_type="max", bucket_size_msec=10, empty=True + ) + for i in range(len(res)): + if math.isnan(res[i][1]): + res[i] = (res[i][0], None) + assert [ + (10, 4.0), + (20, None), + (30, None), + (40, None), + (50, 3.0), + (60, None), + (70, 5.0), + ] == res @pytest.mark.redismod @@ -262,11 +343,87 @@ def test_rev_range(client): assert [(1, 10.0), (0, 1.0)] == client.ts().revrange( 1, 0, 10, aggregation_type="count", bucket_size_msec=10, align=1 ) + assert [(10, 3.0), (0, 2.5500000000000003)] == client.ts().revrange( + 1, 0, 10, aggregation_type="twa", bucket_size_msec=10 + ) + + +@pytest.mark.redismod +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_revrange_latest(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.create("t2") + timeseries.createrule("t1", "t2", aggregation_type="sum", bucket_size_msec=10) + timeseries.add("t1", 1, 1) + timeseries.add("t1", 2, 3) + timeseries.add("t1", 11, 7) + timeseries.add("t1", 13, 1) + res = timeseries.revrange("t2", 0, 10) + assert res == [(0, 4.0)] + res = timeseries.revrange("t2", 0, 10, latest=True) + assert res == [(10, 8.0), (0, 4.0)] + res = timeseries.revrange("t2", 0, 9, latest=True) + assert res == [(0, 4.0)] + + +@pytest.mark.redismod +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_revrange_bucket_timestamp(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.add("t1", 15, 1) + timeseries.add("t1", 17, 4) + timeseries.add("t1", 51, 3) + timeseries.add("t1", 73, 5) + timeseries.add("t1", 75, 3) + assert [(70, 5.0), (50, 3.0), (10, 4.0)] == timeseries.revrange( + "t1", 0, 100, align=0, aggregation_type="max", bucket_size_msec=10 + ) + assert [(20, 4.0), (60, 3.0), (80, 5.0)] == timeseries.range( + "t1", + 0, + 100, + align=0, + aggregation_type="max", + bucket_size_msec=10, + bucket_timestamp="+", + ) + + +@pytest.mark.redismod +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_revrange_empty(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.add("t1", 15, 1) + timeseries.add("t1", 17, 4) + timeseries.add("t1", 51, 3) + timeseries.add("t1", 73, 5) + timeseries.add("t1", 75, 3) + assert [(70, 5.0), (50, 3.0), (10, 4.0)] == timeseries.revrange( + "t1", 0, 100, align=0, aggregation_type="max", bucket_size_msec=10 + ) + res = timeseries.revrange( + "t1", 0, 100, align=0, aggregation_type="max", bucket_size_msec=10, empty=True + ) + for i in range(len(res)): + if math.isnan(res[i][1]): + res[i] = (res[i][0], None) + assert [ + (70, 5.0), + (60, None), + (50, 3.0), + (40, None), + (30, None), + (20, None), + (10, 4.0), + ] == res @pytest.mark.redismod @pytest.mark.onlynoncluster -def testMultiRange(client): +def test_mrange(client): client.ts().create(1, labels={"Test": "This", "team": "ny"}) client.ts().create(2, labels={"Test": "This", "Taste": "That", "team": "sf"}) for i in range(100): @@ -351,6 +508,31 @@ def test_multi_range_advanced(client): assert [(0, 5.0), (5, 6.0)] == res[0]["1"][1] +@pytest.mark.redismod +@pytest.mark.onlynoncluster +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_mrange_latest(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.create("t2", labels={"is_compaction": "true"}) + timeseries.create("t3") + timeseries.create("t4", labels={"is_compaction": "true"}) + timeseries.createrule("t1", "t2", aggregation_type="sum", bucket_size_msec=10) + timeseries.createrule("t3", "t4", aggregation_type="sum", bucket_size_msec=10) + timeseries.add("t1", 1, 1) + timeseries.add("t1", 2, 3) + timeseries.add("t1", 11, 7) + timeseries.add("t1", 13, 1) + timeseries.add("t3", 1, 1) + timeseries.add("t3", 2, 3) + timeseries.add("t3", 11, 7) + timeseries.add("t3", 13, 1) + assert client.ts().mrange(0, 10, filters=["is_compaction=true"], latest=True) == [ + {"t2": [{}, [(0, 4.0), (10, 8.0)]]}, + {"t4": [{}, [(0, 4.0), (10, 8.0)]]}, + ] + + @pytest.mark.redismod @pytest.mark.onlynoncluster @skip_ifmodversion_lt("99.99.99", "timeseries") @@ -434,6 +616,30 @@ def test_multi_reverse_range(client): assert [(1, 10.0), (0, 1.0)] == res[0]["1"][1] +@pytest.mark.redismod +@pytest.mark.onlynoncluster +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_mrevrange_latest(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.create("t2", labels={"is_compaction": "true"}) + timeseries.create("t3") + timeseries.create("t4", labels={"is_compaction": "true"}) + timeseries.createrule("t1", "t2", aggregation_type="sum", bucket_size_msec=10) + timeseries.createrule("t3", "t4", aggregation_type="sum", bucket_size_msec=10) + timeseries.add("t1", 1, 1) + timeseries.add("t1", 2, 3) + timeseries.add("t1", 11, 7) + timeseries.add("t1", 13, 1) + timeseries.add("t3", 1, 1) + timeseries.add("t3", 2, 3) + timeseries.add("t3", 11, 7) + timeseries.add("t3", 13, 1) + assert client.ts().mrevrange( + 0, 10, filters=["is_compaction=true"], latest=True + ) == [{"t2": [{}, [(10, 8.0), (0, 4.0)]]}, {"t4": [{}, [(10, 8.0), (0, 4.0)]]}] + + @pytest.mark.redismod def test_get(client): name = "test" @@ -445,6 +651,21 @@ def test_get(client): assert 4 == client.ts().get(name)[1] +@pytest.mark.redismod +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_get_latest(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.create("t2") + timeseries.createrule("t1", "t2", aggregation_type="sum", bucket_size_msec=10) + timeseries.add("t1", 1, 1) + timeseries.add("t1", 2, 3) + timeseries.add("t1", 11, 7) + timeseries.add("t1", 13, 1) + assert (0, 4.0) == timeseries.get("t2") + assert (10, 8.0) == timeseries.get("t2", latest=True) + + @pytest.mark.redismod @pytest.mark.onlynoncluster def test_mget(client): @@ -467,6 +688,24 @@ def test_mget(client): assert {"Taste": "That", "Test": "This"} == res[0]["2"][0] +@pytest.mark.redismod +@pytest.mark.onlynoncluster +@skip_ifmodversion_lt("1.8.0", "timeseries") +def test_mget_latest(client: redis.Redis): + timeseries = client.ts() + timeseries.create("t1") + timeseries.create("t2", labels={"is_compaction": "true"}) + timeseries.createrule("t1", "t2", aggregation_type="sum", bucket_size_msec=10) + timeseries.add("t1", 1, 1) + timeseries.add("t1", 2, 3) + timeseries.add("t1", 11, 7) + timeseries.add("t1", 13, 1) + assert timeseries.mget(filters=["is_compaction=true"]) == [{"t2": [{}, 0, 4.0]}] + assert [{"t2": [{}, 10, 8.0]}] == timeseries.mget( + filters=["is_compaction=true"], latest=True + ) + + @pytest.mark.redismod def test_info(client): client.ts().create(1, retention_msecs=5, labels={"currentLabel": "currentData"}) @@ -506,7 +745,7 @@ def test_pipeline(client): pipeline.execute() info = client.ts().info("with_pipeline") - assert info.lastTimeStamp == 99 + assert info.last_timestamp == 99 assert info.total_samples == 100 assert client.ts().get("with_pipeline")[1] == 99 * 1.1 From e9f8447ec2b0bc4dbecc5d7bb713ebed1cfd742f Mon Sep 17 00:00:00 2001 From: Nial Daly <34862917+nialdaly@users.noreply.github.com> Date: Sun, 24 Jul 2022 14:31:23 +0100 Subject: [PATCH 11/22] Remove verbose logging from `redis-py/redis/cluster.py` (#2238) * removed the logging module and its corresponding methods * updated CHANGES * except block for RedisClusterException and BusyLoadingError removed * removed unused import (redis.exceptions.BusyLoadingError) * empty commit to re-trigger Actions workflow * replaced BaseException with Exception * empty commit to re-trigger Actions workflow * empty commit to re-trigger Actions workflow * redundant logic removed * re-trigger pipeline * reverted changes * re-trigger pipeline * except logic changed --- CHANGES | 2 +- redis/cluster.py | 63 ++++++------------------------------------------ 2 files changed, 8 insertions(+), 57 deletions(-) diff --git a/CHANGES b/CHANGES index b4841ddb2b..6a890ef122 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,4 @@ - + * Remove verbose logging from cluster.py * Add retry mechanism to async version of Connection * Compare commands case-insensitively in the asyncio command parser * Allow negative `retries` for `Retry` class to retry forever diff --git a/redis/cluster.py b/redis/cluster.py index 9e773b2190..b2d4f3b044 100644 --- a/redis/cluster.py +++ b/redis/cluster.py @@ -1,5 +1,4 @@ import copy -import logging import random import socket import sys @@ -15,7 +14,6 @@ from redis.exceptions import ( AskError, AuthenticationError, - BusyLoadingError, ClusterCrossSlotError, ClusterDownError, ClusterError, @@ -39,8 +37,6 @@ str_if_bytes, ) -log = logging.getLogger(__name__) - def get_node_name(host: str, port: int) -> str: return f"{host}:{port}" @@ -535,7 +531,6 @@ def __init__( " RedisCluster(startup_nodes=[ClusterNode('localhost', 6379)," " ClusterNode('localhost', 6378)])" ) - log.debug(f"startup_nodes : {startup_nodes}") # Update the connection arguments # Whenever a new connection is established, RedisCluster's on_connect # method should be run @@ -666,13 +661,8 @@ def set_default_node(self, node): :return True if the default node was set, else False """ if node is None or self.get_node(node_name=node.name) is None: - log.info( - "The requested node does not exist in the cluster, so " - "the default node was not changed." - ) return False self.nodes_manager.default_node = node - log.info(f"Changed the default cluster node to {node}") return True def monitor(self, target_node=None): @@ -816,8 +806,6 @@ def _determine_nodes(self, *args, **kwargs): else: # get the nodes group for this command if it was predefined command_flag = self.command_flags.get(command) - if command_flag: - log.debug(f"Target node/s for {command}: {command_flag}") if command_flag == self.__class__.RANDOM: # return a random node return [self.get_random_node()] @@ -841,7 +829,6 @@ def _determine_nodes(self, *args, **kwargs): node = self.nodes_manager.get_node_from_slot( slot, self.read_from_replicas and command in READ_COMMANDS ) - log.debug(f"Target for {args}: slot {slot}") return [node] def _should_reinitialized(self): @@ -1019,7 +1006,7 @@ def execute_command(self, *args, **kwargs): res[node.name] = self._execute_command(node, *args, **kwargs) # Return the processed result return self._process_result(args[0], res, **kwargs) - except BaseException as e: + except Exception as e: if type(e) in self.__class__.ERRORS_ALLOW_RETRY: # The nodes and slots cache were reinitialized. # Try again with the new cluster setup. @@ -1059,10 +1046,6 @@ def _execute_command(self, target_node, *args, **kwargs): ) moved = False - log.debug( - f"Executing command {command} on target node: " - f"{target_node.server_type} {target_node.name}" - ) redis_node = self.get_redis_connection(target_node) connection = get_connection(redis_node, *args, **kwargs) if asking: @@ -1077,12 +1060,9 @@ def _execute_command(self, target_node, *args, **kwargs): response, **kwargs ) return response - - except (RedisClusterException, BusyLoadingError, AuthenticationError) as e: - log.exception(type(e)) + except AuthenticationError: raise except (ConnectionError, TimeoutError) as e: - log.exception(type(e)) # ConnectionError can also be raised if we couldn't get a # connection from the pool before timing out, so check that # this is an actual connection before attempting to disconnect. @@ -1101,7 +1081,7 @@ def _execute_command(self, target_node, *args, **kwargs): # and try again with the new setup target_node.redis_connection = None self.nodes_manager.initialize() - raise + raise e except MovedError as e: # First, we will try to patch the slots/nodes cache with the # redirected node output and try again. If MovedError exceeds @@ -1111,7 +1091,6 @@ def _execute_command(self, target_node, *args, **kwargs): # the same client object is shared between multiple threads. To # reduce the frequency you can set this variable in the # RedisCluster constructor. - log.exception("MovedError") self.reinitialize_counter += 1 if self._should_reinitialized(): self.nodes_manager.initialize() @@ -1121,29 +1100,21 @@ def _execute_command(self, target_node, *args, **kwargs): self.nodes_manager.update_moved_exception(e) moved = True except TryAgainError: - log.exception("TryAgainError") - if ttl < self.RedisClusterRequestTTL / 2: time.sleep(0.05) except AskError as e: - log.exception("AskError") - redirect_addr = get_node_name(host=e.host, port=e.port) asking = True except ClusterDownError as e: - log.exception("ClusterDownError") # ClusterDownError can occur during a failover and to get # self-healed, we will try to reinitialize the cluster layout # and retry executing the command time.sleep(0.25) self.nodes_manager.initialize() raise e - except ResponseError as e: - message = e.__str__() - log.exception(f"ResponseError: {message}") - raise e - except BaseException as e: - log.exception("BaseException") + except ResponseError: + raise + except Exception as e: if connection: connection.disconnect() raise e @@ -1280,11 +1251,6 @@ def get_node(self, host=None, port=None, node_name=None): elif node_name: return self.nodes_cache.get(node_name) else: - log.error( - "get_node requires one of the following: " - "1. node name " - "2. host and port" - ) return None def update_moved_exception(self, exception): @@ -1432,7 +1398,6 @@ def initialize(self): :startup_nodes: Responsible for discovering other nodes in the cluster """ - log.debug("Initializing the nodes' topology of the cluster") self.reset() tmp_nodes_cache = {} tmp_slots = {} @@ -1460,17 +1425,9 @@ def initialize(self): ) cluster_slots = str_if_bytes(r.execute_command("CLUSTER SLOTS")) startup_nodes_reachable = True - except (ConnectionError, TimeoutError) as e: - msg = e.__str__ - log.exception( - "An exception occurred while trying to" - " initialize the cluster using the seed node" - f" {startup_node.name}:\n{msg}" - ) + except (ConnectionError, TimeoutError): continue except ResponseError as e: - log.exception('ReseponseError sending "cluster slots" to redis server') - # Isn't a cluster connection, so it won't parse these # exceptions automatically message = e.__str__() @@ -2042,12 +1999,6 @@ def _send_cluster_commands( # If a lot of commands have failed, we'll be setting the # flag to rebuild the slots table from scratch. # So MOVED errors should correct themselves fairly quickly. - log.exception( - f"An exception occurred during pipeline execution. " - f"args: {attempt[-1].args}, " - f"error: {type(attempt[-1].result).__name__} " - f"{str(attempt[-1].result)}" - ) self.reinitialize_counter += 1 if self._should_reinitialized(): self.nodes_manager.initialize() From 8529170a191ba0abb205c8ad8d6a72fec70ad16a Mon Sep 17 00:00:00 2001 From: pedrofrazao <603718+pedrofrazao@users.noreply.github.com> Date: Sun, 24 Jul 2022 14:32:11 +0100 Subject: [PATCH 12/22] redis stream example (#2269) * redis stream example * redis stream example on docs/examples.rst Co-authored-by: pedro.frazao --- docs/examples.rst | 1 + docs/examples/redis-stream-example.ipynb | 754 +++++++++++++++++++++++ 2 files changed, 755 insertions(+) create mode 100644 docs/examples/redis-stream-example.ipynb diff --git a/docs/examples.rst b/docs/examples.rst index 722fae2d03..08526ff614 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -12,3 +12,4 @@ Examples examples/set_and_get_examples examples/search_vector_similarity_examples examples/pipeline_examples + examples/redis-stream-example.ipynb diff --git a/docs/examples/redis-stream-example.ipynb b/docs/examples/redis-stream-example.ipynb new file mode 100644 index 0000000000..9303b527ca --- /dev/null +++ b/docs/examples/redis-stream-example.ipynb @@ -0,0 +1,754 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Redis Stream Examples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## basic config" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "redis_host = \"redis\"\n", + "stream_key = \"skey\"\n", + "stream2_key = \"s2key\"\n", + "group1 = \"grp1\"\n", + "group2 = \"grp2\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## connection" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import redis\n", + "from time import time\n", + "from redis.exceptions import ConnectionError, DataError, NoScriptError, RedisError, ResponseError\n", + "\n", + "r = redis.Redis( redis_host )\n", + "r.ping()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## xadd and xread" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### add some data to the stream" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stream length: 10\n" + ] + } + ], + "source": [ + "for i in range(0,10):\n", + " r.xadd( stream_key, { 'ts': time(), 'v': i } )\n", + "print( f\"stream length: {r.xlen( stream_key )}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### read some data from the stream" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[b'skey', [(b'1657571033115-0', {b'ts': b'1657571033.1128936', b'v': b'0'}), (b'1657571033117-0', {b'ts': b'1657571033.1176307', b'v': b'1'})]]]\n" + ] + } + ], + "source": [ + "## read 2 entries from stream_key\n", + "l = r.xread( count=2, streams={stream_key:0} )\n", + "print(l)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### extract data from the returned structure" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "got data from stream: b'skey'\n", + "id: b'1657571033115-0' value: b'0'\n", + "id: b'1657571033117-0' value: b'1'\n" + ] + } + ], + "source": [ + "first_stream = l[0]\n", + "print( f\"got data from stream: {first_stream[0]}\")\n", + "fs_data = first_stream[1]\n", + "for id, value in fs_data:\n", + " print( f\"id: {id} value: {value[b'v']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### read more data from the stream\n", + "if we call the `xread` with the same arguments we will get the same data" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "id: b'1657571033115-0' value: b'0'\n", + "id: b'1657571033117-0' value: b'1'\n" + ] + } + ], + "source": [ + "l = r.xread( count=2, streams={stream_key:0} )\n", + "for id, value in l[0][1]:\n", + " print( f\"id: {id} value: {value[b'v']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "to get new data we need to change the key passed to the call" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "id: b'1657571033118-0' value: b'2'\n", + "id: b'1657571033119-0' value: b'3'\n" + ] + } + ], + "source": [ + "last_id_returned = l[0][1][-1][0]\n", + "l = r.xread( count=2, streams={stream_key: last_id_returned} )\n", + "for id, value in l[0][1]:\n", + " print( f\"id: {id} value: {value[b'v']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "id: b'1657571033119-1' value: b'4'\n", + "id: b'1657571033121-0' value: b'5'\n" + ] + } + ], + "source": [ + "last_id_returned = l[0][1][-1][0]\n", + "l = r.xread( count=2, streams={stream_key: last_id_returned} )\n", + "for id, value in l[0][1]:\n", + " print( f\"id: {id} value: {value[b'v']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "to get only newer entries" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stream length: 10\n", + "after 5s block, got an empty list [], no *new* messages on the stream\n", + "stream length: 10\n" + ] + } + ], + "source": [ + "print( f\"stream length: {r.xlen( stream_key )}\")\n", + "# wait for 5s for new messages\n", + "l = r.xread( count=1, block=5000, streams={stream_key: '$'} )\n", + "print( f\"after 5s block, got an empty list {l}, no *new* messages on the stream\")\n", + "print( f\"stream length: {r.xlen( stream_key )}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2nd stream\n", + "Add some messages to a 2nd stream" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stream length: 10\n" + ] + } + ], + "source": [ + "for i in range(1000,1010):\n", + " r.xadd( stream2_key, { 'v': i } )\n", + "print( f\"stream length: {r.xlen( stream2_key )}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "get messages from the 2 streams" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "got from b'skey' the entry [(b'1657571033115-0', {b'ts': b'1657571033.1128936', b'v': b'0'})]\n", + "got from b's2key' the entry [(b'1657571042111-0', {b'v': b'1000'})]\n" + ] + } + ], + "source": [ + "l = r.xread( count=1, streams={stream_key:0,stream2_key:0} )\n", + "for k,d in l:\n", + " print(f\"got from {k} the entry {d}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# stream groups\n", + "With the groups is possible track, for many consumers, and at the Redis side, which message have been already consumed.\n", + "## add some data to streams\n", + "Creating 2 streams with 10 messages each." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stream 'skey' length: 20\n", + "stream 's2key' length: 20\n" + ] + } + ], + "source": [ + "def add_some_data_to_stream( sname, key_range ):\n", + " for i in key_range:\n", + " r.xadd( sname, { 'ts': time(), 'v': i } )\n", + " print( f\"stream '{sname}' length: {r.xlen( stream_key )}\")\n", + "\n", + "add_some_data_to_stream( stream_key, range(0,10) )\n", + "add_some_data_to_stream( stream2_key, range(1000,1010) )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## use a group to read from the stream\n", + "* create a group `grp1` with the stream `skey`, and\n", + "* create a group `grp2` with the streams `skey` and `s2key`\n", + "\n", + "Use the `xinfo_group` to verify the result of the group creation." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "skey -> group name: b'grp1' with 0 consumers and b'0-0' as last read id\n", + "skey -> group name: b'grp2' with 0 consumers and b'0-0' as last read id\n", + "s2key -> group name: b'grp2' with 0 consumers and b'0-0' as last read id\n" + ] + } + ], + "source": [ + "## create the group\n", + "def create_group( skey, gname ):\n", + " try:\n", + " r.xgroup_create( name=skey, groupname=gname, id=0 )\n", + " except ResponseError as e:\n", + " print(f\"raised: {e}\")\n", + "\n", + "# group1 read the stream 'skey'\n", + "create_group( stream_key, group1 )\n", + "# group2 read the streams 'skey' and 's2key'\n", + "create_group( stream_key, group2 )\n", + "create_group( stream2_key, group2 )\n", + "\n", + "def group_info( skey ):\n", + " res = r.xinfo_groups( name=skey )\n", + " for i in res:\n", + " print( f\"{skey} -> group name: {i['name']} with {i['consumers']} consumers and {i['last-delivered-id']}\"\n", + " + f\" as last read id\")\n", + " \n", + "group_info( stream_key )\n", + "group_info( stream2_key )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## group read\n", + "The `xreadgroup` method permit to read from a stream group." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def print_xreadgroup_reply( reply, group = None, run = None):\n", + " for d_stream in reply:\n", + " for element in d_stream[1]:\n", + " print( f\"got element {element[0]}\"\n", + " + f\"from stream {d_stream[0]}\" )\n", + " if run is not None:\n", + " run( d_stream[0], group, element[0] )" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "got element b'1657571033115-0'from stream b'skey'\n", + "got element b'1657571033117-0'from stream b'skey'\n" + ] + } + ], + "source": [ + "# read some messages on group1 with consumer 'c' \n", + "d = r.xreadgroup( groupname=group1, consumername='c', block=10,\n", + " count=2, streams={stream_key:'>'})\n", + "print_xreadgroup_reply( d )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A **2nd consumer** for the same stream group will get not delivered messages." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "got element b'1657571033118-0'from stream b'skey'\n", + "got element b'1657571033119-0'from stream b'skey'\n" + ] + } + ], + "source": [ + "# read some messages on group1 with consumer 'c' \n", + "d = r.xreadgroup( groupname=group1, consumername='c2', block=10,\n", + " count=2, streams={stream_key:'>'})\n", + "print_xreadgroup_reply( d )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "But a **2nd stream group** can read the already delivered messages again.\n", + "\n", + "Note that the 2nd stream group include also the 2nd stream.\n", + "That can be identified in the reply (1st element of the reply list)." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "got element b'1657571033115-0'from stream b'skey'\n", + "got element b'1657571033117-0'from stream b'skey'\n", + "got element b'1657571042111-0'from stream b's2key'\n", + "got element b'1657571042113-0'from stream b's2key'\n" + ] + } + ], + "source": [ + "d2 = r.xreadgroup( groupname=group2, consumername='c', block=10,\n", + " count=2, streams={stream_key:'>',stream2_key:'>'})\n", + "print_xreadgroup_reply( d2 )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To check for pending messages (delivered messages without acknowledgment) we can use the `xpending`." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4 pending messages on 'skey' for group 'grp1'\n", + "2 pending messages on 'skey' for group 'grp2'\n", + "2 pending messages on 's2key' for group 'grp2'\n" + ] + } + ], + "source": [ + "# check pending status (read messages without a ack)\n", + "def print_pending_info( key_group ):\n", + " for s,k in key_group:\n", + " pr = r.xpending( name=s, groupname=k )\n", + " print( f\"{pr.get('pending')} pending messages on '{s}' for group '{k}'\" )\n", + " \n", + "print_pending_info( ((stream_key,group1),(stream_key,group2),(stream2_key,group2)) )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ack\n", + "Acknowledge some messages with `xack`." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "got element b'1657571033118-0'from stream b'skey'\n", + "got element b'1657571033119-0'from stream b'skey'\n" + ] + } + ], + "source": [ + "# do acknowledges for group1\n", + "toack = lambda k,g,e: r.xack( k,g, e )\n", + "print_xreadgroup_reply( d, group=group1, run=toack )" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2 pending messages on 'skey' for group 'grp1'\n", + "2 pending messages on 'skey' for group 'grp2'\n", + "2 pending messages on 's2key' for group 'grp2'\n" + ] + } + ], + "source": [ + "# check pending again\n", + "print_pending_info( ((stream_key,group1),(stream_key,group2),(stream2_key,group2)) )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "ack all messages on the `group1`." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "got element b'1657571033119-1'from stream b'skey'\n", + "got element b'1657571033121-0'from stream b'skey'\n", + "got element b'1657571033121-1'from stream b'skey'\n", + "got element b'1657571033121-2'from stream b'skey'\n", + "got element b'1657571033122-0'from stream b'skey'\n", + "got element b'1657571033122-1'from stream b'skey'\n", + "got element b'1657571049557-0'from stream b'skey'\n", + "got element b'1657571049557-1'from stream b'skey'\n", + "got element b'1657571049558-0'from stream b'skey'\n", + "got element b'1657571049559-0'from stream b'skey'\n", + "got element b'1657571049559-1'from stream b'skey'\n", + "got element b'1657571049559-2'from stream b'skey'\n", + "got element b'1657571049560-0'from stream b'skey'\n", + "got element b'1657571049562-0'from stream b'skey'\n", + "got element b'1657571049563-0'from stream b'skey'\n", + "got element b'1657571049563-1'from stream b'skey'\n", + "2 pending messages on 'skey' for group 'grp1'\n" + ] + } + ], + "source": [ + "d = r.xreadgroup( groupname=group1, consumername='c', block=10,\n", + " count=100, streams={stream_key:'>'})\n", + "print_xreadgroup_reply( d, group=group1, run=toack)\n", + "print_pending_info( ((stream_key,group1),) )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "But stream length will be the same after the `xack` of all messages on the `group1`." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "20" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r.xlen(stream_key)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## delete all\n", + "To remove the messages with need to remote them explicitly with `xdel`." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "s1 = r.xread( streams={stream_key:0} )\n", + "for streams in s1:\n", + " stream_name, messages = streams\n", + " # del all ids from the message list\n", + " [ r.xdel( stream_name, i[0] ) for i in messages ]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "stream length" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "r.xlen(stream_key)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "But with the `xdel` the 2nd group can read any not processed message from the `skey`." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "got element b'1657571042113-1'from stream b's2key'\n", + "got element b'1657571042114-0'from stream b's2key'\n" + ] + } + ], + "source": [ + "d2 = r.xreadgroup( groupname=group2, consumername='c', block=10,\n", + " count=2, streams={stream_key:'>',stream2_key:'>'})\n", + "print_xreadgroup_reply( d2 )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 8e2ea28076b3ebc8e05faf2ced03717013bd835f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD=20=D0=91=D0=B5=D0=B7=D0=B4?= =?UTF-8?q?=D0=B5=D0=BD=D0=B5=D0=B6=D0=BD=D1=8B=D1=85?= Date: Sun, 24 Jul 2022 20:40:50 +0500 Subject: [PATCH 13/22] Fix: `start_id` type for `XAUTOCLAIM` (#2257) * Changed start_id type for xautoclaim * Added to changes Co-authored-by: dvora-h <67596500+dvora-h@users.noreply.github.com> --- CHANGES | 1 + redis/commands/core.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 6a890ef122..2abf62766f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,4 @@ + * Fix start_id type for XAUTOCLAIM * Remove verbose logging from cluster.py * Add retry mechanism to async version of Connection * Compare commands case-insensitively in the asyncio command parser diff --git a/redis/commands/core.py b/redis/commands/core.py index 027d3dbc7c..455c3f46cb 100644 --- a/redis/commands/core.py +++ b/redis/commands/core.py @@ -3420,7 +3420,7 @@ def xautoclaim( groupname: GroupT, consumername: ConsumerT, min_idle_time: int, - start_id: int = 0, + start_id: StreamIdT = "0-0", count: Union[int, None] = None, justid: bool = False, ) -> ResponseT: From 20fa54fc84a3ef1d0cfd3127791b09737345bc42 Mon Sep 17 00:00:00 2001 From: Iglesys Date: Mon, 25 Jul 2022 13:13:39 +0200 Subject: [PATCH 14/22] Doc add timeseries example (#2267) * DOC add timeseries example * DOC add timeseries examples * Apply suggestions * Fix typo Detention period => Retention period Co-authored-by: Gauthier Imbert --- docs/examples.rst | 1 + docs/examples/timeseries_examples.ipynb | 631 ++++++++++++++++++++++++ 2 files changed, 632 insertions(+) create mode 100644 docs/examples/timeseries_examples.ipynb diff --git a/docs/examples.rst b/docs/examples.rst index 08526ff614..3fed8b4195 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -12,4 +12,5 @@ Examples examples/set_and_get_examples examples/search_vector_similarity_examples examples/pipeline_examples + examples/timeseries_examples examples/redis-stream-example.ipynb diff --git a/docs/examples/timeseries_examples.ipynb b/docs/examples/timeseries_examples.ipynb new file mode 100644 index 0000000000..fefc0c8f37 --- /dev/null +++ b/docs/examples/timeseries_examples.ipynb @@ -0,0 +1,631 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Timeseries\n", + "\n", + "`redis-py` supports [RedisTimeSeries](https://github.com/RedisTimeSeries/RedisTimeSeries/) which is a time-series-database module for Redis.\n", + "\n", + "This example shows how to handle timeseries data with `redis-py`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Health check" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import redis \n", + "\n", + "r = redis.Redis(decode_responses=True)\n", + "ts = r.ts()\n", + "\n", + "r.ping()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Simple example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.create(\"ts_key\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add samples to the timeseries\n", + "\n", + "We can either set the timestamp with an UNIX timestamp in milliseconds or use * to set the timestamp based en server's clock." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1657272304448" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.add(\"ts_key\", 1657265437756, 1)\n", + "ts.add(\"ts_key\", \"1657265437757\", 2)\n", + "ts.add(\"ts_key\", \"*\", 3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get the last sample" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(1657272304448, 3.0)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.get(\"ts_key\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get samples between two timestamps\n", + "\n", + "The minimum and maximum possible timestamps can be expressed with respectfully - and +." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(1657265437756, 1.0), (1657265437757, 2.0), (1657272304448, 3.0)]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.range(\"ts_key\", \"-\", \"+\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(1657265437756, 1.0), (1657265437757, 2.0)]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.range(\"ts_key\", 1657265437756, 1657265437757)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Delete samples between two timestamps" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Before deletion: [(1657265437756, 1.0), (1657265437757, 2.0), (1657272304448, 3.0)]\n", + "After deletion: [(1657272304448, 3.0)]\n" + ] + } + ], + "source": [ + "print(\"Before deletion: \", ts.range(\"ts_key\", \"-\", \"+\"))\n", + "ts.delete(\"ts_key\", 1657265437756, 1657265437757)\n", + "print(\"After deletion: \", ts.range(\"ts_key\", \"-\", \"+\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multiple timeseries with labels" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.create(\"ts_key1\")\n", + "ts.create(\"ts_key2\", labels={\"label1\": 1, \"label2\": 2})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add samples to multiple timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[1657272306147, 1657272306147]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.madd([(\"ts_key1\", \"*\", 1), (\"ts_key2\", \"*\", 2)])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add samples with labels" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1657272306457" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.add(\"ts_key2\", \"*\", 2, labels={\"label1\": 1, \"label2\": 2})\n", + "ts.add(\"ts_key2\", \"*\", 2, labels={\"label1\": 3, \"label2\": 4})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get the last sample matching specific label\n", + "\n", + "Get the last sample that matches \"label1=1\", see [Redis documentation](https://redis.io/commands/ts.mget/) to see the posible filter values." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'ts_key2': [{}, 1657272306457, 2.0]}]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.mget([\"label1=1\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get also the label-value pairs of the sample:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'ts_key2': [{'label1': '1', 'label2': '2'}, 1657272306457, 2.0]}]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.mget([\"label1=1\"], with_labels=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retention period\n", + "\n", + "You can specify a retention period when creating timeseries objects or when adding a sample timeseries object. Once the retention period has elapsed, the sample is removed from the timeseries." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "retention_time = 1000\n", + "ts.create(\"ts_key_ret\", retention_msecs=retention_time)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Base timeseries: [(1657272307670, 1.0)]\n", + "Timeseries after 1000 milliseconds: [(1657272307670, 1.0)]\n" + ] + } + ], + "source": [ + "import time\n", + "# this will be deleted in 1000 milliseconds\n", + "ts.add(\"ts_key_ret\", \"*\", 1, retention_msecs=retention_time)\n", + "print(\"Base timeseries: \", ts.range(\"ts_key_ret\", \"-\", \"+\"))\n", + "# sleeping for 1000 milliseconds (1 second)\n", + "time.sleep(1)\n", + "print(\"Timeseries after 1000 milliseconds: \", ts.range(\"ts_key_ret\", \"-\", \"+\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The two lists are the same, this is because the oldest values are deleted when a new sample is added." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1657272308849" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.add(\"ts_key_ret\", \"*\", 10)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(1657272308849, 10.0)]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.range(\"ts_key_ret\", \"-\", \"+\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here the first sample has been deleted." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Specify duplicate policies\n", + "\n", + "By default, the policy for duplicates timestamp keys is set to \"BLOCK\", we cannot create two samples with the same timestamp:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TSDB: Error at upsert, update is not supported when DUPLICATE_POLICY is set to BLOCK mode\n" + ] + } + ], + "source": [ + "ts.add(\"ts_key\", 123456789, 1)\n", + "try:\n", + " ts.add(\"ts_key\", 123456789, 2)\n", + "except Exception as err:\n", + " print(err)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can change this default behaviour using `duplicate_policy` parameter, for instance:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(123456789, 2.0), (1657272304448, 3.0)]" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# using policy \"LAST\", we keep the last added sample\n", + "ts.add(\"ts_key\", 123456789, 2, duplicate_policy=\"LAST\")\n", + "ts.range(\"ts_key\", \"-\", \"+\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For more informations about duplicate policies, see [Redis documentation](https://redis.io/commands/ts.add/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Redis TSDB to keep track of a value" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1657272310241" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.add(\"ts_key_incr\", \"*\", 0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Increment the value:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "for _ in range(10):\n", + " ts.incrby(\"ts_key_incr\", 1)\n", + " # sleeping a bit so the timestamp are not duplicates\n", + " time.sleep(0.01)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(1657272310241, 0.0),\n", + " (1657272310533, 1.0),\n", + " (1657272310545, 2.0),\n", + " (1657272310556, 3.0),\n", + " (1657272310567, 4.0),\n", + " (1657272310578, 5.0),\n", + " (1657272310589, 6.0),\n", + " (1657272310600, 7.0),\n", + " (1657272310611, 8.0),\n", + " (1657272310622, 9.0),\n", + " (1657272310632, 10.0)]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ts.range(\"ts_key_incr\", \"-\", \"+\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9.2 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.2" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From a44b9522151b5a8ddd8b7fbe9e4c60175a236b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Tue, 26 Jul 2022 12:25:23 +0000 Subject: [PATCH 15/22] Fix warnings and resource usage problems in asyncio unittests (#2258) * Use pytest-asyncio in auto mode Remove overly genereric `pytestmark=pytest.mark.asyncio` causing lots of warning noise * Use "Factories as Fixtures" test pattern for the `create_redis` fixture this fixture is now async, avoiding teardown problems with missing event loops. * Fix sporadic error on fast event loops, such as `--uvloop` * Close connection, even if "username" was in kwargs This fixes a resource usage warning in the async unittests. * Do async cleanup of acl passwords via a fixture * Remove unused import, fix whitespace * Fix test with missing "await" * Close pubsub objects after use in unittest Use a simple fixture where possible, otherwise manually call pubsub.close() * re-introduce `pytestmark=pytest.mark.asyncio` for python 3.6 * Use context manager to clean up connections in connection pool for unit tests * Provide asynccontextmanager for python 3.6 * make `test_late_subscribe()` more robuste * Catch a couple of additional leaked resources --- tests/test_asyncio/conftest.py | 124 ++++++++++------ tests/test_asyncio/test_bloom.py | 9 +- tests/test_asyncio/test_cluster.py | 4 +- tests/test_asyncio/test_commands.py | 110 ++++---------- tests/test_asyncio/test_connection.py | 4 +- tests/test_asyncio/test_connection_pool.py | 160 +++++++++++---------- tests/test_asyncio/test_encoding.py | 4 +- tests/test_asyncio/test_json.py | 2 - tests/test_asyncio/test_lock.py | 4 +- tests/test_asyncio/test_monitor.py | 5 +- tests/test_asyncio/test_pipeline.py | 5 +- tests/test_asyncio/test_pubsub.py | 144 +++++++++++-------- tests/test_asyncio/test_scripting.py | 2 + tests/test_asyncio/test_search.py | 4 +- tests/test_asyncio/test_sentinel.py | 4 +- tests/test_asyncio/test_timeseries.py | 4 +- tests/test_json.py | 3 +- tests/test_ssl.py | 4 +- tox.ini | 1 + 19 files changed, 319 insertions(+), 278 deletions(-) diff --git a/tests/test_asyncio/conftest.py b/tests/test_asyncio/conftest.py index 8166588d8e..04fbf62cf7 100644 --- a/tests/test_asyncio/conftest.py +++ b/tests/test_asyncio/conftest.py @@ -1,14 +1,9 @@ -import asyncio +import functools import random import sys from typing import Union from urllib.parse import urlparse -if sys.version_info[0:2] == (3, 6): - import pytest as pytest_asyncio -else: - import pytest_asyncio - import pytest from packaging.version import Version @@ -26,6 +21,13 @@ from .compat import mock +if sys.version_info[0:2] == (3, 6): + import pytest as pytest_asyncio + + pytestmark = pytest.mark.asyncio +else: + import pytest_asyncio + async def _get_info(redis_url): client = redis.Redis.from_url(redis_url) @@ -69,11 +71,13 @@ async def _get_info(redis_url): "pool-hiredis", ], ) -def create_redis(request, event_loop: asyncio.BaseEventLoop): +async def create_redis(request): """Wrapper around redis.create_redis.""" single_connection, parser_cls = request.param - async def f( + teardown_clients = [] + + async def client_factory( url: str = request.config.getoption("--redis-url"), cls=redis.Redis, flushdb=True, @@ -95,56 +99,50 @@ async def f( client = client.client() await client.initialize() - def teardown(): - async def ateardown(): - if not cluster_mode: - if "username" in kwargs: - return - if flushdb: - try: - await client.flushdb() - except redis.ConnectionError: - # handle cases where a test disconnected a client - # just manually retry the flushdb - await client.flushdb() - await client.close() - await client.connection_pool.disconnect() - else: - if flushdb: - try: - await client.flushdb(target_nodes="primaries") - except redis.ConnectionError: - # handle cases where a test disconnected a client - # just manually retry the flushdb - await client.flushdb(target_nodes="primaries") - await client.close() - - if event_loop.is_running(): - event_loop.create_task(ateardown()) + async def teardown(): + if not cluster_mode: + if flushdb and "username" not in kwargs: + try: + await client.flushdb() + except redis.ConnectionError: + # handle cases where a test disconnected a client + # just manually retry the flushdb + await client.flushdb() + await client.close() + await client.connection_pool.disconnect() else: - event_loop.run_until_complete(ateardown()) - - request.addfinalizer(teardown) - + if flushdb: + try: + await client.flushdb(target_nodes="primaries") + except redis.ConnectionError: + # handle cases where a test disconnected a client + # just manually retry the flushdb + await client.flushdb(target_nodes="primaries") + await client.close() + + teardown_clients.append(teardown) return client - return f + yield client_factory + + for teardown in teardown_clients: + await teardown() @pytest_asyncio.fixture() -async def r(request, create_redis): - yield await create_redis() +async def r(create_redis): + return await create_redis() @pytest_asyncio.fixture() async def r2(create_redis): """A second client for tests that need multiple""" - yield await create_redis() + return await create_redis() @pytest_asyncio.fixture() async def modclient(request, create_redis): - yield await create_redis( + return await create_redis( url=request.config.getoption("--redismod-url"), decode_responses=True ) @@ -222,7 +220,7 @@ async def mock_cluster_resp_slaves(create_redis, **kwargs): def master_host(request): url = request.config.getoption("--redis-url") parts = urlparse(url) - yield parts.hostname + return parts.hostname async def wait_for_command( @@ -246,3 +244,41 @@ async def wait_for_command( return monitor_response if key in monitor_response["command"]: return None + + +# python 3.6 doesn't have the asynccontextmanager decorator. Provide it here. +class AsyncContextManager: + def __init__(self, async_generator): + self.gen = async_generator + + async def __aenter__(self): + try: + return await self.gen.__anext__() + except StopAsyncIteration as err: + raise RuntimeError("Pickles") from err + + async def __aexit__(self, exc_type, exc_inst, tb): + if exc_type: + await self.gen.athrow(exc_type, exc_inst, tb) + return True + try: + await self.gen.__anext__() + except StopAsyncIteration: + return + raise RuntimeError("More pickles") + + +if sys.version_info[0:2] == (3, 6): + + def asynccontextmanager(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + return AsyncContextManager(func(*args, **kwargs)) + + return wrapper + +else: + from contextlib import asynccontextmanager as _asynccontextmanager + + def asynccontextmanager(func): + return _asynccontextmanager(func) diff --git a/tests/test_asyncio/test_bloom.py b/tests/test_asyncio/test_bloom.py index feb98cc41e..2bf4e030e6 100644 --- a/tests/test_asyncio/test_bloom.py +++ b/tests/test_asyncio/test_bloom.py @@ -1,10 +1,13 @@ +import sys + import pytest import redis.asyncio as redis from redis.exceptions import ModuleError, RedisError from redis.utils import HIREDIS_AVAILABLE -pytestmark = pytest.mark.asyncio +if sys.version_info[0:2] == (3, 6): + pytestmark = pytest.mark.asyncio def intlist(obj): @@ -91,7 +94,7 @@ async def do_verify(): res += rv == x assert res < 5 - do_verify() + await do_verify() cmds = [] if HIREDIS_AVAILABLE: with pytest.raises(ModuleError): @@ -120,7 +123,7 @@ async def do_verify(): cur_info = await modclient.bf().execute_command("bf.debug", "myBloom") assert prev_info == cur_info - do_verify() + await do_verify() await modclient.bf().client.delete("myBloom") await modclient.bf().create("myBloom", "0.0001", "10000000") diff --git a/tests/test_asyncio/test_cluster.py b/tests/test_asyncio/test_cluster.py index 0d0ea33db2..8766cbf09b 100644 --- a/tests/test_asyncio/test_cluster.py +++ b/tests/test_asyncio/test_cluster.py @@ -11,6 +11,8 @@ if sys.version_info[0:2] == (3, 6): import pytest as pytest_asyncio + + pytestmark = pytest.mark.asyncio else: import pytest_asyncio @@ -39,8 +41,6 @@ skip_unless_arch_bits, ) -pytestmark = pytest.mark.asyncio - default_host = "127.0.0.1" default_port = 7000 default_cluster_slots = [ diff --git a/tests/test_asyncio/test_commands.py b/tests/test_asyncio/test_commands.py index e128ac40b8..913f05b3fe 100644 --- a/tests/test_asyncio/test_commands.py +++ b/tests/test_asyncio/test_commands.py @@ -12,6 +12,8 @@ if sys.version_info[0:2] == (3, 6): import pytest as pytest_asyncio + + pytestmark = pytest.mark.asyncio else: import pytest_asyncio @@ -27,11 +29,24 @@ REDIS_6_VERSION = "5.9.0" -pytestmark = pytest.mark.asyncio +@pytest_asyncio.fixture() +async def r_teardown(r: redis.Redis): + """ + A special fixture which removes the provided names from the database after use + """ + usernames = [] + + def factory(username): + usernames.append(username) + return r + + yield factory + for username in usernames: + await r.acl_deluser(username) @pytest_asyncio.fixture() -async def slowlog(r: redis.Redis, event_loop): +async def slowlog(r: redis.Redis): current_config = await r.config_get() old_slower_than_value = current_config["slowlog-log-slower-than"] old_max_legnth_value = current_config["slowlog-max-len"] @@ -94,17 +109,9 @@ async def test_acl_cat_with_category(self, r: redis.Redis): assert "get" in commands @skip_if_server_version_lt(REDIS_6_VERSION) - async def test_acl_deluser(self, r: redis.Redis, request, event_loop): + async def test_acl_deluser(self, r_teardown): username = "redis-py-user" - - def teardown(): - coro = r.acl_deluser(username) - if event_loop.is_running(): - event_loop.create_task(coro) - else: - event_loop.run_until_complete(coro) - - request.addfinalizer(teardown) + r = r_teardown(username) assert await r.acl_deluser(username) == 0 assert await r.acl_setuser(username, enabled=False, reset=True) @@ -117,18 +124,9 @@ async def test_acl_genpass(self, r: redis.Redis): @skip_if_server_version_lt(REDIS_6_VERSION) @skip_if_server_version_gte("7.0.0") - async def test_acl_getuser_setuser(self, r: redis.Redis, request, event_loop): + async def test_acl_getuser_setuser(self, r_teardown): username = "redis-py-user" - - def teardown(): - coro = r.acl_deluser(username) - if event_loop.is_running(): - event_loop.create_task(coro) - else: - event_loop.run_until_complete(coro) - - request.addfinalizer(teardown) - + r = r_teardown(username) # test enabled=False assert await r.acl_setuser(username, enabled=False, reset=True) assert await r.acl_getuser(username) == { @@ -233,17 +231,9 @@ def teardown(): @skip_if_server_version_lt(REDIS_6_VERSION) @skip_if_server_version_gte("7.0.0") - async def test_acl_list(self, r: redis.Redis, request, event_loop): + async def test_acl_list(self, r_teardown): username = "redis-py-user" - - def teardown(): - coro = r.acl_deluser(username) - if event_loop.is_running(): - event_loop.create_task(coro) - else: - event_loop.run_until_complete(coro) - - request.addfinalizer(teardown) + r = r_teardown(username) assert await r.acl_setuser(username, enabled=False, reset=True) users = await r.acl_list() @@ -251,17 +241,9 @@ def teardown(): @skip_if_server_version_lt(REDIS_6_VERSION) @pytest.mark.onlynoncluster - async def test_acl_log(self, r: redis.Redis, request, event_loop, create_redis): + async def test_acl_log(self, r_teardown, create_redis): username = "redis-py-user" - - def teardown(): - coro = r.acl_deluser(username) - if event_loop.is_running(): - event_loop.create_task(coro) - else: - event_loop.run_until_complete(coro) - - request.addfinalizer(teardown) + r = r_teardown(username) await r.acl_setuser( username, enabled=True, @@ -294,55 +276,25 @@ def teardown(): assert await r.acl_log_reset() @skip_if_server_version_lt(REDIS_6_VERSION) - async def test_acl_setuser_categories_without_prefix_fails( - self, r: redis.Redis, request, event_loop - ): + async def test_acl_setuser_categories_without_prefix_fails(self, r_teardown): username = "redis-py-user" - - def teardown(): - coro = r.acl_deluser(username) - if event_loop.is_running(): - event_loop.create_task(coro) - else: - event_loop.run_until_complete(coro) - - request.addfinalizer(teardown) + r = r_teardown(username) with pytest.raises(exceptions.DataError): await r.acl_setuser(username, categories=["list"]) @skip_if_server_version_lt(REDIS_6_VERSION) - async def test_acl_setuser_commands_without_prefix_fails( - self, r: redis.Redis, request, event_loop - ): + async def test_acl_setuser_commands_without_prefix_fails(self, r_teardown): username = "redis-py-user" - - def teardown(): - coro = r.acl_deluser(username) - if event_loop.is_running(): - event_loop.create_task(coro) - else: - event_loop.run_until_complete(coro) - - request.addfinalizer(teardown) + r = r_teardown(username) with pytest.raises(exceptions.DataError): await r.acl_setuser(username, commands=["get"]) @skip_if_server_version_lt(REDIS_6_VERSION) - async def test_acl_setuser_add_passwords_and_nopass_fails( - self, r: redis.Redis, request, event_loop - ): + async def test_acl_setuser_add_passwords_and_nopass_fails(self, r_teardown): username = "redis-py-user" - - def teardown(): - coro = r.acl_deluser(username) - if event_loop.is_running(): - event_loop.create_task(coro) - else: - event_loop.run_until_complete(coro) - - request.addfinalizer(teardown) + r = r_teardown(username) with pytest.raises(exceptions.DataError): await r.acl_setuser(username, passwords="+mypass", nopass=True) diff --git a/tests/test_asyncio/test_connection.py b/tests/test_asyncio/test_connection.py index 78a3efd2a0..8030f7e628 100644 --- a/tests/test_asyncio/test_connection.py +++ b/tests/test_asyncio/test_connection.py @@ -1,5 +1,6 @@ import asyncio import socket +import sys import types from unittest.mock import patch @@ -18,7 +19,8 @@ from .compat import mock -pytestmark = pytest.mark.asyncio +if sys.version_info[0:2] == (3, 6): + pytestmark = pytest.mark.asyncio @pytest.mark.onlynoncluster diff --git a/tests/test_asyncio/test_connection_pool.py b/tests/test_asyncio/test_connection_pool.py index 6c56558d59..c8eb918e28 100644 --- a/tests/test_asyncio/test_connection_pool.py +++ b/tests/test_asyncio/test_connection_pool.py @@ -7,6 +7,8 @@ if sys.version_info[0:2] == (3, 6): import pytest as pytest_asyncio + + pytestmark = pytest.mark.asyncio else: import pytest_asyncio @@ -15,10 +17,9 @@ from tests.conftest import skip_if_redis_enterprise, skip_if_server_version_lt from .compat import mock +from .conftest import asynccontextmanager from .test_pubsub import wait_for_message -pytestmark = pytest.mark.asyncio - @pytest.mark.onlynoncluster class TestRedisAutoReleaseConnectionPool: @@ -114,7 +115,8 @@ async def can_read(self, timeout: float = 0): class TestConnectionPool: - def get_pool( + @asynccontextmanager + async def get_pool( self, connection_kwargs=None, max_connections=None, @@ -126,71 +128,77 @@ def get_pool( max_connections=max_connections, **connection_kwargs, ) - return pool + try: + yield pool + finally: + await pool.disconnect(inuse_connections=True) async def test_connection_creation(self): connection_kwargs = {"foo": "bar", "biz": "baz"} - pool = self.get_pool( + async with self.get_pool( connection_kwargs=connection_kwargs, connection_class=DummyConnection - ) - connection = await pool.get_connection("_") - assert isinstance(connection, DummyConnection) - assert connection.kwargs == connection_kwargs + ) as pool: + connection = await pool.get_connection("_") + assert isinstance(connection, DummyConnection) + assert connection.kwargs == connection_kwargs async def test_multiple_connections(self, master_host): connection_kwargs = {"host": master_host} - pool = self.get_pool(connection_kwargs=connection_kwargs) - c1 = await pool.get_connection("_") - c2 = await pool.get_connection("_") - assert c1 != c2 + async with self.get_pool(connection_kwargs=connection_kwargs) as pool: + c1 = await pool.get_connection("_") + c2 = await pool.get_connection("_") + assert c1 != c2 async def test_max_connections(self, master_host): connection_kwargs = {"host": master_host} - pool = self.get_pool(max_connections=2, connection_kwargs=connection_kwargs) - await pool.get_connection("_") - await pool.get_connection("_") - with pytest.raises(redis.ConnectionError): + async with self.get_pool( + max_connections=2, connection_kwargs=connection_kwargs + ) as pool: + await pool.get_connection("_") await pool.get_connection("_") + with pytest.raises(redis.ConnectionError): + await pool.get_connection("_") async def test_reuse_previously_released_connection(self, master_host): connection_kwargs = {"host": master_host} - pool = self.get_pool(connection_kwargs=connection_kwargs) - c1 = await pool.get_connection("_") - await pool.release(c1) - c2 = await pool.get_connection("_") - assert c1 == c2 + async with self.get_pool(connection_kwargs=connection_kwargs) as pool: + c1 = await pool.get_connection("_") + await pool.release(c1) + c2 = await pool.get_connection("_") + assert c1 == c2 - def test_repr_contains_db_info_tcp(self): + async def test_repr_contains_db_info_tcp(self): connection_kwargs = { "host": "localhost", "port": 6379, "db": 1, "client_name": "test-client", } - pool = self.get_pool( + async with self.get_pool( connection_kwargs=connection_kwargs, connection_class=redis.Connection - ) - expected = ( - "ConnectionPool>" - ) - assert repr(pool) == expected + ) as pool: + expected = ( + "ConnectionPool>" + ) + assert repr(pool) == expected - def test_repr_contains_db_info_unix(self): + async def test_repr_contains_db_info_unix(self): connection_kwargs = {"path": "/abc", "db": 1, "client_name": "test-client"} - pool = self.get_pool( + async with self.get_pool( connection_kwargs=connection_kwargs, connection_class=redis.UnixDomainSocketConnection, - ) - expected = ( - "ConnectionPool>" - ) - assert repr(pool) == expected + ) as pool: + expected = ( + "ConnectionPool>" + ) + assert repr(pool) == expected class TestBlockingConnectionPool: - def get_pool(self, connection_kwargs=None, max_connections=10, timeout=20): + @asynccontextmanager + async def get_pool(self, connection_kwargs=None, max_connections=10, timeout=20): connection_kwargs = connection_kwargs or {} pool = redis.BlockingConnectionPool( connection_class=DummyConnection, @@ -198,7 +206,10 @@ def get_pool(self, connection_kwargs=None, max_connections=10, timeout=20): timeout=timeout, **connection_kwargs, ) - return pool + try: + yield pool + finally: + await pool.disconnect(inuse_connections=True) async def test_connection_creation(self, master_host): connection_kwargs = { @@ -207,10 +218,10 @@ async def test_connection_creation(self, master_host): "host": master_host[0], "port": master_host[1], } - pool = self.get_pool(connection_kwargs=connection_kwargs) - connection = await pool.get_connection("_") - assert isinstance(connection, DummyConnection) - assert connection.kwargs == connection_kwargs + async with self.get_pool(connection_kwargs=connection_kwargs) as pool: + connection = await pool.get_connection("_") + assert isinstance(connection, DummyConnection) + assert connection.kwargs == connection_kwargs async def test_disconnect(self, master_host): """A regression test for #1047""" @@ -220,30 +231,31 @@ async def test_disconnect(self, master_host): "host": master_host[0], "port": master_host[1], } - pool = self.get_pool(connection_kwargs=connection_kwargs) - await pool.get_connection("_") - await pool.disconnect() + async with self.get_pool(connection_kwargs=connection_kwargs) as pool: + await pool.get_connection("_") + await pool.disconnect() async def test_multiple_connections(self, master_host): connection_kwargs = {"host": master_host[0], "port": master_host[1]} - pool = self.get_pool(connection_kwargs=connection_kwargs) - c1 = await pool.get_connection("_") - c2 = await pool.get_connection("_") - assert c1 != c2 + async with self.get_pool(connection_kwargs=connection_kwargs) as pool: + c1 = await pool.get_connection("_") + c2 = await pool.get_connection("_") + assert c1 != c2 async def test_connection_pool_blocks_until_timeout(self, master_host): """When out of connections, block for timeout seconds, then raise""" connection_kwargs = {"host": master_host} - pool = self.get_pool( + async with self.get_pool( max_connections=1, timeout=0.1, connection_kwargs=connection_kwargs - ) - await pool.get_connection("_") + ) as pool: + c1 = await pool.get_connection("_") - start = asyncio.get_event_loop().time() - with pytest.raises(redis.ConnectionError): - await pool.get_connection("_") - # we should have waited at least 0.1 seconds - assert asyncio.get_event_loop().time() - start >= 0.1 + start = asyncio.get_event_loop().time() + with pytest.raises(redis.ConnectionError): + await pool.get_connection("_") + # we should have waited at least 0.1 seconds + assert asyncio.get_event_loop().time() - start >= 0.1 + await c1.disconnect() async def test_connection_pool_blocks_until_conn_available(self, master_host): """ @@ -251,26 +263,26 @@ async def test_connection_pool_blocks_until_conn_available(self, master_host): to the pool """ connection_kwargs = {"host": master_host[0], "port": master_host[1]} - pool = self.get_pool( + async with self.get_pool( max_connections=1, timeout=2, connection_kwargs=connection_kwargs - ) - c1 = await pool.get_connection("_") + ) as pool: + c1 = await pool.get_connection("_") - async def target(): - await asyncio.sleep(0.1) - await pool.release(c1) + async def target(): + await asyncio.sleep(0.1) + await pool.release(c1) - start = asyncio.get_event_loop().time() - await asyncio.gather(target(), pool.get_connection("_")) - assert asyncio.get_event_loop().time() - start >= 0.1 + start = asyncio.get_event_loop().time() + await asyncio.gather(target(), pool.get_connection("_")) + assert asyncio.get_event_loop().time() - start >= 0.1 async def test_reuse_previously_released_connection(self, master_host): connection_kwargs = {"host": master_host} - pool = self.get_pool(connection_kwargs=connection_kwargs) - c1 = await pool.get_connection("_") - await pool.release(c1) - c2 = await pool.get_connection("_") - assert c1 == c2 + async with self.get_pool(connection_kwargs=connection_kwargs) as pool: + c1 = await pool.get_connection("_") + await pool.release(c1) + c2 = await pool.get_connection("_") + assert c1 == c2 def test_repr_contains_db_info_tcp(self): pool = redis.ConnectionPool( @@ -689,6 +701,8 @@ async def test_arbitrary_command_advances_next_health_check(self, r): if r.connection: await r.get("foo") next_health_check = r.connection.next_health_check + # ensure that the event loop's `time()` advances a bit + await asyncio.sleep(0.001) await r.get("foo") assert next_health_check < r.connection.next_health_check diff --git a/tests/test_asyncio/test_encoding.py b/tests/test_asyncio/test_encoding.py index 133ea3783c..5db7187c84 100644 --- a/tests/test_asyncio/test_encoding.py +++ b/tests/test_asyncio/test_encoding.py @@ -4,14 +4,14 @@ if sys.version_info[0:2] == (3, 6): import pytest as pytest_asyncio + + pytestmark = pytest.mark.asyncio else: import pytest_asyncio import redis.asyncio as redis from redis.exceptions import DataError -pytestmark = pytest.mark.asyncio - @pytest.mark.onlynoncluster class TestEncoding: diff --git a/tests/test_asyncio/test_json.py b/tests/test_asyncio/test_json.py index a045dd7c1a..416a9f4a21 100644 --- a/tests/test_asyncio/test_json.py +++ b/tests/test_asyncio/test_json.py @@ -5,8 +5,6 @@ from redis.commands.json.path import Path from tests.conftest import skip_ifmodversion_lt -pytestmark = pytest.mark.asyncio - @pytest.mark.redismod async def test_json_setbinarykey(modclient: redis.Redis): diff --git a/tests/test_asyncio/test_lock.py b/tests/test_asyncio/test_lock.py index 8ceb3bc958..86a8d62f71 100644 --- a/tests/test_asyncio/test_lock.py +++ b/tests/test_asyncio/test_lock.py @@ -5,14 +5,14 @@ if sys.version_info[0:2] == (3, 6): import pytest as pytest_asyncio + + pytestmark = pytest.mark.asyncio else: import pytest_asyncio from redis.asyncio.lock import Lock from redis.exceptions import LockError, LockNotOwnedError -pytestmark = pytest.mark.asyncio - @pytest.mark.onlynoncluster class TestLock: diff --git a/tests/test_asyncio/test_monitor.py b/tests/test_asyncio/test_monitor.py index 783ba262b0..9185bcd2ee 100644 --- a/tests/test_asyncio/test_monitor.py +++ b/tests/test_asyncio/test_monitor.py @@ -1,10 +1,13 @@ +import sys + import pytest from tests.conftest import skip_if_redis_enterprise, skip_ifnot_redis_enterprise from .conftest import wait_for_command -pytestmark = pytest.mark.asyncio +if sys.version_info[0:2] == (3, 6): + pytestmark = pytest.mark.asyncio @pytest.mark.onlynoncluster diff --git a/tests/test_asyncio/test_pipeline.py b/tests/test_asyncio/test_pipeline.py index dfeb66464c..33391d019d 100644 --- a/tests/test_asyncio/test_pipeline.py +++ b/tests/test_asyncio/test_pipeline.py @@ -1,3 +1,5 @@ +import sys + import pytest import redis @@ -5,7 +7,8 @@ from .conftest import wait_for_command -pytestmark = pytest.mark.asyncio +if sys.version_info[0:2] == (3, 6): + pytestmark = pytest.mark.asyncio class TestPipeline: diff --git a/tests/test_asyncio/test_pubsub.py b/tests/test_asyncio/test_pubsub.py index 6c76bf334e..d6a817a61b 100644 --- a/tests/test_asyncio/test_pubsub.py +++ b/tests/test_asyncio/test_pubsub.py @@ -8,6 +8,8 @@ if sys.version_info[0:2] == (3, 6): import pytest as pytest_asyncio + + pytestmark = pytest.mark.asyncio(forbid_global_loop=True) else: import pytest_asyncio @@ -18,8 +20,6 @@ from .compat import mock -pytestmark = pytest.mark.asyncio(forbid_global_loop=True) - def with_timeout(t): def wrapper(corofunc): @@ -80,6 +80,13 @@ def make_subscribe_test_data(pubsub, type): assert False, f"invalid subscribe type: {type}" +@pytest_asyncio.fixture() +async def pubsub(r: redis.Redis): + p = r.pubsub() + yield p + await p.close() + + @pytest.mark.onlynoncluster class TestPubSubSubscribeUnsubscribe: async def _test_subscribe_unsubscribe( @@ -101,12 +108,12 @@ async def _test_subscribe_unsubscribe( i = len(keys) - 1 - i assert await wait_for_message(p) == make_message(unsub_type, key, i) - async def test_channel_subscribe_unsubscribe(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "channel") + async def test_channel_subscribe_unsubscribe(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "channel") await self._test_subscribe_unsubscribe(**kwargs) - async def test_pattern_subscribe_unsubscribe(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "pattern") + async def test_pattern_subscribe_unsubscribe(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "pattern") await self._test_subscribe_unsubscribe(**kwargs) @pytest.mark.onlynoncluster @@ -144,12 +151,12 @@ async def _test_resubscribe_on_reconnection( for channel in unique_channels: assert channel in keys - async def test_resubscribe_to_channels_on_reconnection(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "channel") + async def test_resubscribe_to_channels_on_reconnection(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "channel") await self._test_resubscribe_on_reconnection(**kwargs) - async def test_resubscribe_to_patterns_on_reconnection(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "pattern") + async def test_resubscribe_to_patterns_on_reconnection(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "pattern") await self._test_resubscribe_on_reconnection(**kwargs) async def _test_subscribed_property( @@ -199,13 +206,13 @@ async def _test_subscribed_property( # now we're finally unsubscribed assert p.subscribed is False - async def test_subscribe_property_with_channels(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "channel") + async def test_subscribe_property_with_channels(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "channel") await self._test_subscribed_property(**kwargs) @pytest.mark.onlynoncluster - async def test_subscribe_property_with_patterns(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "pattern") + async def test_subscribe_property_with_patterns(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "pattern") await self._test_subscribed_property(**kwargs) async def test_ignore_all_subscribe_messages(self, r: redis.Redis): @@ -224,9 +231,10 @@ async def test_ignore_all_subscribe_messages(self, r: redis.Redis): assert p.subscribed is True assert await wait_for_message(p) is None assert p.subscribed is False + await p.close() - async def test_ignore_individual_subscribe_messages(self, r: redis.Redis): - p = r.pubsub() + async def test_ignore_individual_subscribe_messages(self, pubsub): + p = pubsub checks = ( (p.subscribe, "foo"), @@ -243,13 +251,13 @@ async def test_ignore_individual_subscribe_messages(self, r: redis.Redis): assert message is None assert p.subscribed is False - async def test_sub_unsub_resub_channels(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "channel") + async def test_sub_unsub_resub_channels(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "channel") await self._test_sub_unsub_resub(**kwargs) @pytest.mark.onlynoncluster - async def test_sub_unsub_resub_patterns(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "pattern") + async def test_sub_unsub_resub_patterns(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "pattern") await self._test_sub_unsub_resub(**kwargs) async def _test_sub_unsub_resub( @@ -266,12 +274,12 @@ async def _test_sub_unsub_resub( assert await wait_for_message(p) == make_message(sub_type, key, 1) assert p.subscribed is True - async def test_sub_unsub_all_resub_channels(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "channel") + async def test_sub_unsub_all_resub_channels(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "channel") await self._test_sub_unsub_all_resub(**kwargs) - async def test_sub_unsub_all_resub_patterns(self, r: redis.Redis): - kwargs = make_subscribe_test_data(r.pubsub(), "pattern") + async def test_sub_unsub_all_resub_patterns(self, pubsub): + kwargs = make_subscribe_test_data(pubsub, "pattern") await self._test_sub_unsub_all_resub(**kwargs) async def _test_sub_unsub_all_resub( @@ -300,8 +308,8 @@ def message_handler(self, message): async def async_message_handler(self, message): self.async_message = message - async def test_published_message_to_channel(self, r: redis.Redis): - p = r.pubsub() + async def test_published_message_to_channel(self, r: redis.Redis, pubsub): + p = pubsub await p.subscribe("foo") assert await wait_for_message(p) == make_message("subscribe", "foo", 1) assert await r.publish("foo", "test message") == 1 @@ -310,8 +318,8 @@ async def test_published_message_to_channel(self, r: redis.Redis): assert isinstance(message, dict) assert message == make_message("message", "foo", "test message") - async def test_published_message_to_pattern(self, r: redis.Redis): - p = r.pubsub() + async def test_published_message_to_pattern(self, r: redis.Redis, pubsub): + p = pubsub await p.subscribe("foo") await p.psubscribe("f*") assert await wait_for_message(p) == make_message("subscribe", "foo", 1) @@ -340,6 +348,7 @@ async def test_channel_message_handler(self, r: redis.Redis): assert await r.publish("foo", "test message") == 1 assert await wait_for_message(p) is None assert self.message == make_message("message", "foo", "test message") + await p.close() async def test_channel_async_message_handler(self, r): p = r.pubsub(ignore_subscribe_messages=True) @@ -348,6 +357,7 @@ async def test_channel_async_message_handler(self, r): assert await r.publish("foo", "test message") == 1 assert await wait_for_message(p) is None assert self.async_message == make_message("message", "foo", "test message") + await p.close() async def test_channel_sync_async_message_handler(self, r): p = r.pubsub(ignore_subscribe_messages=True) @@ -359,6 +369,7 @@ async def test_channel_sync_async_message_handler(self, r): assert await wait_for_message(p) is None assert self.message == make_message("message", "foo", "test message") assert self.async_message == make_message("message", "bar", "test message 2") + await p.close() @pytest.mark.onlynoncluster async def test_pattern_message_handler(self, r: redis.Redis): @@ -370,6 +381,7 @@ async def test_pattern_message_handler(self, r: redis.Redis): assert self.message == make_message( "pmessage", "foo", "test message", pattern="f*" ) + await p.close() async def test_unicode_channel_message_handler(self, r: redis.Redis): p = r.pubsub(ignore_subscribe_messages=True) @@ -380,6 +392,7 @@ async def test_unicode_channel_message_handler(self, r: redis.Redis): assert await r.publish(channel, "test message") == 1 assert await wait_for_message(p) is None assert self.message == make_message("message", channel, "test message") + await p.close() @pytest.mark.onlynoncluster # see: https://redis-py-cluster.readthedocs.io/en/stable/pubsub.html @@ -395,9 +408,10 @@ async def test_unicode_pattern_message_handler(self, r: redis.Redis): assert self.message == make_message( "pmessage", channel, "test message", pattern=pattern ) + await p.close() - async def test_get_message_without_subscribe(self, r: redis.Redis): - p = r.pubsub() + async def test_get_message_without_subscribe(self, r: redis.Redis, pubsub): + p = pubsub with pytest.raises(RuntimeError) as info: await p.get_message() expect = ( @@ -427,8 +441,8 @@ def message_handler(self, message): async def r(self, create_redis): return await create_redis(decode_responses=True) - async def test_channel_subscribe_unsubscribe(self, r: redis.Redis): - p = r.pubsub() + async def test_channel_subscribe_unsubscribe(self, pubsub): + p = pubsub await p.subscribe(self.channel) assert await wait_for_message(p) == self.make_message( "subscribe", self.channel, 1 @@ -439,8 +453,8 @@ async def test_channel_subscribe_unsubscribe(self, r: redis.Redis): "unsubscribe", self.channel, 0 ) - async def test_pattern_subscribe_unsubscribe(self, r: redis.Redis): - p = r.pubsub() + async def test_pattern_subscribe_unsubscribe(self, pubsub): + p = pubsub await p.psubscribe(self.pattern) assert await wait_for_message(p) == self.make_message( "psubscribe", self.pattern, 1 @@ -451,8 +465,8 @@ async def test_pattern_subscribe_unsubscribe(self, r: redis.Redis): "punsubscribe", self.pattern, 0 ) - async def test_channel_publish(self, r: redis.Redis): - p = r.pubsub() + async def test_channel_publish(self, r: redis.Redis, pubsub): + p = pubsub await p.subscribe(self.channel) assert await wait_for_message(p) == self.make_message( "subscribe", self.channel, 1 @@ -463,8 +477,8 @@ async def test_channel_publish(self, r: redis.Redis): ) @pytest.mark.onlynoncluster - async def test_pattern_publish(self, r: redis.Redis): - p = r.pubsub() + async def test_pattern_publish(self, r: redis.Redis, pubsub): + p = pubsub await p.psubscribe(self.pattern) assert await wait_for_message(p) == self.make_message( "psubscribe", self.pattern, 1 @@ -490,6 +504,7 @@ async def test_channel_message_handler(self, r: redis.Redis): await r.publish(self.channel, new_data) assert await wait_for_message(p) is None assert self.message == self.make_message("message", self.channel, new_data) + await p.close() async def test_pattern_message_handler(self, r: redis.Redis): p = r.pubsub(ignore_subscribe_messages=True) @@ -511,6 +526,7 @@ async def test_pattern_message_handler(self, r: redis.Redis): assert self.message == self.make_message( "pmessage", self.channel, new_data, pattern=self.pattern ) + await p.close() async def test_context_manager(self, r: redis.Redis): async with r.pubsub() as pubsub: @@ -520,6 +536,7 @@ async def test_context_manager(self, r: redis.Redis): assert pubsub.connection is None assert pubsub.channels == {} assert pubsub.patterns == {} + await pubsub.close() @pytest.mark.onlynoncluster @@ -535,8 +552,8 @@ async def test_channel_subscribe(self, r: redis.Redis): class TestPubSubSubcommands: @pytest.mark.onlynoncluster @skip_if_server_version_lt("2.8.0") - async def test_pubsub_channels(self, r: redis.Redis): - p = r.pubsub() + async def test_pubsub_channels(self, r: redis.Redis, pubsub): + p = pubsub await p.subscribe("foo", "bar", "baz", "quux") for i in range(4): assert (await wait_for_message(p))["type"] == "subscribe" @@ -560,6 +577,9 @@ async def test_pubsub_numsub(self, r: redis.Redis): channels = [(b"foo", 1), (b"bar", 2), (b"baz", 3)] assert await r.pubsub_numsub("foo", "bar", "baz") == channels + await p1.close() + await p2.close() + await p3.close() @skip_if_server_version_lt("2.8.0") async def test_pubsub_numpat(self, r: redis.Redis): @@ -568,6 +588,7 @@ async def test_pubsub_numpat(self, r: redis.Redis): for i in range(3): assert (await wait_for_message(p))["type"] == "psubscribe" assert await r.pubsub_numpat() == 3 + await p.close() @pytest.mark.onlynoncluster @@ -580,6 +601,7 @@ async def test_send_pubsub_ping(self, r: redis.Redis): assert await wait_for_message(p) == make_message( type="pong", channel=None, data="", pattern=None ) + await p.close() @skip_if_server_version_lt("3.0.0") async def test_send_pubsub_ping_message(self, r: redis.Redis): @@ -589,13 +611,16 @@ async def test_send_pubsub_ping_message(self, r: redis.Redis): assert await wait_for_message(p) == make_message( type="pong", channel=None, data="hello world", pattern=None ) + await p.close() @pytest.mark.onlynoncluster class TestPubSubConnectionKilled: @skip_if_server_version_lt("3.0.0") - async def test_connection_error_raised_when_connection_dies(self, r: redis.Redis): - p = r.pubsub() + async def test_connection_error_raised_when_connection_dies( + self, r: redis.Redis, pubsub + ): + p = pubsub await p.subscribe("foo") assert await wait_for_message(p) == make_message("subscribe", "foo", 1) for client in await r.client_list(): @@ -607,8 +632,8 @@ async def test_connection_error_raised_when_connection_dies(self, r: redis.Redis @pytest.mark.onlynoncluster class TestPubSubTimeouts: - async def test_get_message_with_timeout_returns_none(self, r: redis.Redis): - p = r.pubsub() + async def test_get_message_with_timeout_returns_none(self, pubsub): + p = pubsub await p.subscribe("foo") assert await wait_for_message(p) == make_message("subscribe", "foo", 1) assert await p.get_message(timeout=0.01) is None @@ -616,15 +641,13 @@ async def test_get_message_with_timeout_returns_none(self, r: redis.Redis): @pytest.mark.onlynoncluster class TestPubSubReconnect: - # @pytest.mark.xfail @with_timeout(2) - async def test_reconnect_listen(self, r: redis.Redis): + async def test_reconnect_listen(self, r: redis.Redis, pubsub): """ Test that a loop processing PubSub messages can survive a disconnect, by issuing a connect() call. """ messages = asyncio.Queue() - pubsub = r.pubsub() interrupt = False async def loop(): @@ -698,12 +721,12 @@ async def _subscribe(self, p, *args, **kwargs): ): return - async def test_callbacks(self, r: redis.Redis): + async def test_callbacks(self, r: redis.Redis, pubsub): def callback(message): messages.put_nowait(message) messages = asyncio.Queue() - p = r.pubsub() + p = pubsub await self._subscribe(p, foo=callback) task = asyncio.get_event_loop().create_task(p.run()) await r.publish("foo", "bar") @@ -720,13 +743,13 @@ def callback(message): "type": "message", } - async def test_exception_handler(self, r: redis.Redis): + async def test_exception_handler(self, r: redis.Redis, pubsub): def exception_handler_callback(e, pubsub) -> None: assert pubsub == p exceptions.put_nowait(e) exceptions = asyncio.Queue() - p = r.pubsub() + p = pubsub await self._subscribe(p, foo=lambda x: None) with mock.patch.object(p, "get_message", side_effect=Exception("error")): task = asyncio.get_event_loop().create_task( @@ -740,26 +763,25 @@ def exception_handler_callback(e, pubsub) -> None: pass assert str(e) == "error" - async def test_late_subscribe(self, r: redis.Redis): + async def test_late_subscribe(self, r: redis.Redis, pubsub): def callback(message): messages.put_nowait(message) messages = asyncio.Queue() - p = r.pubsub() + p = pubsub task = asyncio.get_event_loop().create_task(p.run()) # wait until loop gets settled. Add a subscription await asyncio.sleep(0.1) await p.subscribe(foo=callback) # wait tof the subscribe to finish. Cannot use _subscribe() because # p.run() is already accepting messages - await asyncio.sleep(0.1) - await r.publish("foo", "bar") - message = None - try: - async with async_timeout.timeout(0.1): - message = await messages.get() - except asyncio.TimeoutError: - pass + while True: + n = await r.publish("foo", "bar") + if n == 1: + break + await asyncio.sleep(0.1) + async with async_timeout.timeout(0.1): + message = await messages.get() task.cancel() # we expect a cancelled error, not the Runtime error # ("did you forget to call subscribe()"") diff --git a/tests/test_asyncio/test_scripting.py b/tests/test_asyncio/test_scripting.py index 764525fb4a..406ab208e2 100644 --- a/tests/test_asyncio/test_scripting.py +++ b/tests/test_asyncio/test_scripting.py @@ -4,6 +4,8 @@ if sys.version_info[0:2] == (3, 6): import pytest as pytest_asyncio + + pytestmark = pytest.mark.asyncio else: import pytest_asyncio diff --git a/tests/test_asyncio/test_search.py b/tests/test_asyncio/test_search.py index 5aaa56f159..bc3a212ac9 100644 --- a/tests/test_asyncio/test_search.py +++ b/tests/test_asyncio/test_search.py @@ -1,6 +1,7 @@ import bz2 import csv import os +import sys import time from io import TextIOWrapper @@ -18,7 +19,8 @@ from redis.commands.search.suggestion import Suggestion from tests.conftest import skip_ifmodversion_lt -pytestmark = pytest.mark.asyncio +if sys.version_info[0:2] == (3, 6): + pytestmark = pytest.mark.asyncio WILL_PLAY_TEXT = os.path.abspath( diff --git a/tests/test_asyncio/test_sentinel.py b/tests/test_asyncio/test_sentinel.py index 4130e67400..e77e07f98e 100644 --- a/tests/test_asyncio/test_sentinel.py +++ b/tests/test_asyncio/test_sentinel.py @@ -5,6 +5,8 @@ if sys.version_info[0:2] == (3, 6): import pytest as pytest_asyncio + + pytestmark = pytest.mark.asyncio else: import pytest_asyncio @@ -17,8 +19,6 @@ SlaveNotFoundError, ) -pytestmark = pytest.mark.asyncio - @pytest_asyncio.fixture(scope="module") def master_ip(master_host): diff --git a/tests/test_asyncio/test_timeseries.py b/tests/test_asyncio/test_timeseries.py index ac2807fe1d..0e57c4f049 100644 --- a/tests/test_asyncio/test_timeseries.py +++ b/tests/test_asyncio/test_timeseries.py @@ -1,3 +1,4 @@ +import sys import time from time import sleep @@ -6,7 +7,8 @@ import redis.asyncio as redis from tests.conftest import skip_ifmodversion_lt -pytestmark = pytest.mark.asyncio +if sys.version_info[0:2] == (3, 6): + pytestmark = pytest.mark.asyncio @pytest.mark.redismod diff --git a/tests/test_json.py b/tests/test_json.py index 1cc448c5f9..0965a93d88 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -1411,7 +1411,8 @@ def test_set_path(client): with open(jsonfile, "w+") as fp: fp.write(json.dumps({"hello": "world"})) - open(nojsonfile, "a+").write("hello") + with open(nojsonfile, "a+") as fp: + fp.write("hello") result = {jsonfile: True, nojsonfile: False} assert client.json().set_path(Path.root_path(), root) == result diff --git a/tests/test_ssl.py b/tests/test_ssl.py index d029b80dcb..ed38a3166b 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -68,8 +68,8 @@ def test_validating_self_signed_certificate(self, request): assert r.ping() def test_validating_self_signed_string_certificate(self, request): - f = open(self.SERVER_CERT) - cert_data = f.read() + with open(self.SERVER_CERT) as f: + cert_data = f.read() ssl_url = request.config.option.redis_ssl_url p = urlparse(ssl_url)[1].split(":") r = redis.Redis( diff --git a/tox.ini b/tox.ini index 0ceb008cf6..d1aeb02ade 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ markers = asyncio: marker for async tests replica: replica tests experimental: run only experimental tests +asyncio_mode = auto [tox] minversion = 3.2.0 From 4a83d670f9ccb22ec64c332574d64ee8506424dc Mon Sep 17 00:00:00 2001 From: DvirDukhan Date: Tue, 26 Jul 2022 20:14:17 +0300 Subject: [PATCH 16/22] Graph - add counters for removed labels and properties (#2292) * grpah - add counters for removed labels and properties * added mock graph result set statistics * docstrings for graph result set statistics * format * isort * moved docstrings into functions --- redis/commands/graph/query_result.py | 24 ++++++++++++++ tests/test_graph.py | 47 ++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/redis/commands/graph/query_result.py b/redis/commands/graph/query_result.py index c560b2eb52..76ffa6bc05 100644 --- a/redis/commands/graph/query_result.py +++ b/redis/commands/graph/query_result.py @@ -9,10 +9,12 @@ from .path import Path LABELS_ADDED = "Labels added" +LABELS_REMOVED = "Labels removed" NODES_CREATED = "Nodes created" NODES_DELETED = "Nodes deleted" RELATIONSHIPS_DELETED = "Relationships deleted" PROPERTIES_SET = "Properties set" +PROPERTIES_REMOVED = "Properties removed" RELATIONSHIPS_CREATED = "Relationships created" INDICES_CREATED = "Indices created" INDICES_DELETED = "Indices deleted" @@ -21,8 +23,10 @@ STATS = [ LABELS_ADDED, + LABELS_REMOVED, NODES_CREATED, PROPERTIES_SET, + PROPERTIES_REMOVED, RELATIONSHIPS_CREATED, NODES_DELETED, RELATIONSHIPS_DELETED, @@ -323,42 +327,62 @@ def _get_stat(self, stat): @property def labels_added(self): + """Returns the number of labels added in the query""" return self._get_stat(LABELS_ADDED) + @property + def labels_removed(self): + """Returns the number of labels removed in the query""" + return self._get_stat(LABELS_REMOVED) + @property def nodes_created(self): + """Returns the number of nodes created in the query""" return self._get_stat(NODES_CREATED) @property def nodes_deleted(self): + """Returns the number of nodes deleted in the query""" return self._get_stat(NODES_DELETED) @property def properties_set(self): + """Returns the number of properties set in the query""" return self._get_stat(PROPERTIES_SET) + @property + def properties_removed(self): + """Returns the number of properties removed in the query""" + return self._get_stat(PROPERTIES_REMOVED) + @property def relationships_created(self): + """Returns the number of relationships created in the query""" return self._get_stat(RELATIONSHIPS_CREATED) @property def relationships_deleted(self): + """Returns the number of relationships deleted in the query""" return self._get_stat(RELATIONSHIPS_DELETED) @property def indices_created(self): + """Returns the number of indices created in the query""" return self._get_stat(INDICES_CREATED) @property def indices_deleted(self): + """Returns the number of indices deleted in the query""" return self._get_stat(INDICES_DELETED) @property def cached_execution(self): + """Returns whether or not the query execution plan was cached""" return self._get_stat(CACHED_EXECUTION) == 1 @property def run_time_ms(self): + """Returns the server execution time of the query""" return self._get_stat(INTERNAL_EXECUTION_TIME) diff --git a/tests/test_graph.py b/tests/test_graph.py index 76f8794c18..526308c672 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -1,7 +1,24 @@ +from unittest.mock import patch + import pytest from redis.commands.graph import Edge, Node, Path from redis.commands.graph.execution_plan import Operation +from redis.commands.graph.query_result import ( + CACHED_EXECUTION, + INDICES_CREATED, + INDICES_DELETED, + INTERNAL_EXECUTION_TIME, + LABELS_ADDED, + LABELS_REMOVED, + NODES_CREATED, + NODES_DELETED, + PROPERTIES_REMOVED, + PROPERTIES_SET, + RELATIONSHIPS_CREATED, + RELATIONSHIPS_DELETED, + QueryResult, +) from redis.exceptions import ResponseError from tests.conftest import skip_if_redis_enterprise @@ -575,3 +592,33 @@ def test_explain(client): assert result.structured_plan == expected redis_graph.delete() + + +@pytest.mark.redismod +def test_resultset_statistics(client): + with patch.object(target=QueryResult, attribute="_get_stat") as mock_get_stats: + result = client.graph().query("RETURN 1") + result.labels_added + mock_get_stats.assert_called_with(LABELS_ADDED) + result.labels_removed + mock_get_stats.assert_called_with(LABELS_REMOVED) + result.nodes_created + mock_get_stats.assert_called_with(NODES_CREATED) + result.nodes_deleted + mock_get_stats.assert_called_with(NODES_DELETED) + result.properties_set + mock_get_stats.assert_called_with(PROPERTIES_SET) + result.properties_removed + mock_get_stats.assert_called_with(PROPERTIES_REMOVED) + result.relationships_created + mock_get_stats.assert_called_with(RELATIONSHIPS_CREATED) + result.relationships_deleted + mock_get_stats.assert_called_with(RELATIONSHIPS_DELETED) + result.indices_created + mock_get_stats.assert_called_with(INDICES_CREATED) + result.indices_deleted + mock_get_stats.assert_called_with(INDICES_DELETED) + result.cached_execution + mock_get_stats.assert_called_with(CACHED_EXECUTION) + result.run_time_ms + mock_get_stats.assert_called_with(INTERNAL_EXECUTION_TIME) From 115c259765d1513f86ea0836d4bb1b376f332440 Mon Sep 17 00:00:00 2001 From: Chayim Date: Wed, 27 Jul 2022 14:05:20 +0300 Subject: [PATCH 17/22] cleaning up the readme and moving docs into readthedocs (#2291) * cleaning up the readme and moving docs into readthedocs * examples at the end as per pr comments --- README.md | 1144 ++---------------------------------- docs/advanced_features.rst | 436 ++++++++++++++ docs/clustering.rst | 242 ++++++++ docs/index.rst | 3 + docs/lua_scripting.rst | 110 ++++ redis/client.py | 2 + 6 files changed, 828 insertions(+), 1109 deletions(-) create mode 100644 docs/advanced_features.rst create mode 100644 docs/clustering.rst create mode 100644 docs/lua_scripting.rst diff --git a/README.md b/README.md index 4241da47dd..d4f324209a 100644 --- a/README.md +++ b/README.md @@ -27,18 +27,19 @@ To install redis-py, simply: $ pip install redis ``` -Looking for a high-level library to handle object mapping? See [redis-om-python](https://github.com/redis/redis-om-python)! +For faster performance, install redis with hiredis support, this provides a compiled response parser, and *for most cases* requires zero code changes. By default, if hiredis is available, redis-py will attempt to use it for response parsing. -redis-py requires a running Redis server. Assuming you have docker -```bash -docker run -p 6379:6379 -it redis/redis-stack-server:latest +``` bash +$ pip install redis[hiredis] ``` -## Getting Started +Looking for a high-level library to handle object mapping? See [redis-om-python](https://github.com/redis/redis-om-python)! -redis-py supports Python 3.7+. +## Usage -``` pycon +### Basic Example + +``` python >>> import redis >>> r = redis.Redis(host='localhost', port=6379, db=0) >>> r.set('foo', 'bar') @@ -47,34 +48,34 @@ True b'bar' ``` -By default, all responses are returned as bytes in Python -3. +The above code connects to localhost on port 6379, sets a value in Redis, and retrieves it. All responses are returned as bytes in Python, to receive decoded strings, set *decode_responses=True*. For this, and more connection options, see [these examples](https://redis.readthedocs.io/en/stable/examples.html) -If **all** string responses from a client should be decoded, the user -can specify *decode_responses=True* in -```Redis.__init__```. In this case, any Redis command that -returns a string type will be decoded with the encoding -specified. +### Connection Pools -The default encoding is utf-8, but this can be customized by specifying the -encoding argument for the redis.Redis class. -The encoding will be used to automatically encode any -strings passed to commands, such as key names and values. +By default, redis-py uses a connection pool to manage connections. Each instance of of a Redis class receives its own connection pool. You can however define your own [redis.ConnectionPool](https://redis.readthedocs.io/en/stable/connections.html#connection-pools) + +``` python +>>> pool = redis.ConnectionPool(host='localhost', port=6379, db=0) +>>> r = redis.Redis(connection_pool=pool) +``` -## API Reference +Alternatively, you might want to look at [Async connections](https://redis.readthedocs.io/en/stable/examples/asyncio_examples.html), or [Cluster connections](https://redis.readthedocs.io/en/stable/connections.html#cluster-client), or even [Async Cluster connections](https://redis.readthedocs.io/en/stable/connections.html#async-cluster-client) + +### Redis Commands + +There is built-in support for all of the [out-of-the-box Redis commands](https://redis.io/commands). They are exposed using the raw Redis command names (`HSET`, `HGETALL`, etc.) except where a work (i.e del) is reserved by the language. The complete set of commands can be found [here](https://github.com/redis/redis-py/tree/master/redis/commands), or [the documentation](https://redis.readthedocs.io/en/stable/commands.html). + +## Advanced Topics The [official Redis command documentation](https://redis.io/commands) does a great job of explaining each command in detail. redis-py attempts to adhere to the official command syntax. There are a few exceptions: -- **SELECT**: Not implemented. See the explanation in the Thread - Safety section below. -- **DEL**: *del* is a reserved keyword in the Python syntax. - Therefore redis-py uses *delete* instead. - **MULTI/EXEC**: These are implemented as part of the Pipeline class. The pipeline is wrapped with the MULTI and EXEC statements by default when it is executed, which can be disabled by specifying transaction=False. See more about Pipelines below. + - **SUBSCRIBE/LISTEN**: Similar to pipelines, PubSub is implemented as a separate class as it places the underlying connection in a state where it can\'t execute non-pubsub commands. Calling the pubsub @@ -83,1112 +84,37 @@ to adhere to the official command syntax. There are a few exceptions: PUBLISH from the Redis client (see [this comment on issue #151](https://github.com/redis/redis-py/issues/151#issuecomment-1545015) for details). -- **SCAN/SSCAN/HSCAN/ZSCAN**: The *SCAN commands are implemented as - they exist in the Redis documentation. In addition, each command has - an equivalent iterator method. These are purely for convenience so - the user doesn't have to keep track of the cursor while iterating. - Use the scan_iter/sscan_iter/hscan_iter/zscan_iter methods for this - behavior. - -## Connecting to Redis - -### Client Classes: Redis and StrictRedis - -redis-py 3.0 drops support for the legacy *Redis* client class. -*StrictRedis* has been renamed to *Redis* and an alias named -*StrictRedis* is provided so that users previously using -*StrictRedis* can continue to run unchanged. - -The 2.X *Redis* class provided alternative implementations of a few -commands. This confused users (rightfully so) and caused a number of -support issues. To make things easier going forward, it was decided to -drop support for these alternate implementations and instead focus on a -single client class. - -2.X users that are already using StrictRedis don\'t have to change the -class name. StrictRedis will continue to work for the foreseeable -future. - -2.X users that are using the Redis class will have to make changes if -they use any of the following commands: - -- SETEX: The argument order has changed. The new order is (name, time, - value). -- LREM: The argument order has changed. The new order is (name, num, - value). -- TTL and PTTL: The return value is now always an int and matches the - official Redis command (>0 indicates the timeout, -1 indicates that - the key exists but that it has no expire time set, -2 indicates that - the key does not exist) - - -### Connection Pools - -Behind the scenes, redis-py uses a connection pool to manage connections -to a Redis server. By default, each Redis instance you create will in -turn create its own connection pool. You can override this behavior and -use an existing connection pool by passing an already created connection -pool instance to the connection_pool argument of the Redis class. You -may choose to do this in order to implement client side sharding or have -fine-grain control of how connections are managed. - -``` pycon ->>> pool = redis.ConnectionPool(host='localhost', port=6379, db=0) ->>> r = redis.Redis(connection_pool=pool) -``` - -### Connections - -ConnectionPools manage a set of Connection instances. redis-py ships -with two types of Connections. The default, Connection, is a normal TCP -socket based connection. The UnixDomainSocketConnection allows for -clients running on the same device as the server to connect via a unix -domain socket. To use a UnixDomainSocketConnection connection, simply -pass the unix_socket_path argument, which is a string to the unix domain -socket file. Additionally, make sure the unixsocket parameter is defined -in your redis.conf file. It\'s commented out by default. - -``` pycon ->>> r = redis.Redis(unix_socket_path='/tmp/redis.sock') -``` - -You can create your own Connection subclasses as well. This may be -useful if you want to control the socket behavior within an async -framework. To instantiate a client class using your own connection, you -need to create a connection pool, passing your class to the -connection_class argument. Other keyword parameters you pass to the pool -will be passed to the class specified during initialization. - -``` pycon ->>> pool = redis.ConnectionPool(connection_class=YourConnectionClass, - your_arg='...', ...) -``` - -Connections maintain an open socket to the Redis server. Sometimes these -sockets are interrupted or disconnected for a variety of reasons. For -example, network appliances, load balancers and other services that sit -between clients and servers are often configured to kill connections -that remain idle for a given threshold. - -When a connection becomes disconnected, the next command issued on that -connection will fail and redis-py will raise a ConnectionError to the -caller. This allows each application that uses redis-py to handle errors -in a way that\'s fitting for that specific application. However, -constant error handling can be verbose and cumbersome, especially when -socket disconnections happen frequently in many production environments. - -To combat this, redis-py can issue regular health checks to assess the -liveliness of a connection just before issuing a command. Users can pass -`health_check_interval=N` to the Redis or ConnectionPool classes or as a -query argument within a Redis URL. The value of `health_check_interval` -must be an integer. A value of `0`, the default, disables health checks. -Any positive integer will enable health checks. Health checks are -performed just before a command is executed if the underlying connection -has been idle for more than `health_check_interval` seconds. For -example, `health_check_interval=30` will ensure that a health check is -run on any connection that has been idle for 30 or more seconds just -before a command is executed on that connection. - -If your application is running in an environment that disconnects idle -connections after 30 seconds you should set the `health_check_interval` -option to a value less than 30. - -This option also works on any PubSub connection that is created from a -client with `health_check_interval` enabled. PubSub users need to ensure -that *get_message()* or `listen()` are called more frequently than -`health_check_interval` seconds. It is assumed that most workloads -already do this. - -If your PubSub use case doesn\'t call `get_message()` or `listen()` -frequently, you should call `pubsub.check_health()` explicitly on a -regularly basis. - -### SSL Connections - -redis-py 3.0 changes the default value of the -ssl_cert_reqs option from None to -\'required\'. See [Issue -1016](https://github.com/redis/redis-py/issues/1016). This change -enforces hostname validation when accepting a cert from a remote SSL -terminator. If the terminator doesn\'t properly set the hostname on the -cert this will cause redis-py 3.0 to raise a ConnectionError. - -This check can be disabled by setting ssl_cert_reqs to -None. Note that doing so removes the security check. Do so -at your own risk. - -Example with hostname verification using a local certificate bundle -(linux): - -``` pycon ->>> import redis ->>> r = redis.Redis(host='xxxxxx.cache.amazonaws.com', port=6379, db=0, - ssl=True, - ssl_ca_certs='/etc/ssl/certs/ca-certificates.crt') ->>> r.set('foo', 'bar') -True ->>> r.get('foo') -b'bar' -``` - -Example with hostname verification using -[certifi](https://pypi.org/project/certifi/): - -``` pycon ->>> import redis, certifi ->>> r = redis.Redis(host='xxxxxx.cache.amazonaws.com', port=6379, db=0, - ssl=True, ssl_ca_certs=certifi.where()) ->>> r.set('foo', 'bar') -True ->>> r.get('foo') -b'bar' -``` - -Example turning off hostname verification (not recommended): - -``` pycon ->>> import redis ->>> r = redis.Redis(host='xxxxxx.cache.amazonaws.com', port=6379, db=0, - ssl=True, ssl_cert_reqs=None) ->>> r.set('foo', 'bar') -True ->>> r.get('foo') -b'bar' -``` - -### Sentinel support -redis-py can be used together with [Redis -Sentinel](https://redis.io/topics/sentinel) to discover Redis nodes. You -need to have at least one Sentinel daemon running in order to use -redis-py's Sentinel support. - -Connecting redis-py to the Sentinel instance(s) is easy. You can use a -Sentinel connection to discover the master and slaves network addresses: - -``` pycon ->>> from redis import Sentinel ->>> sentinel = Sentinel([('localhost', 26379)], socket_timeout=0.1) ->>> sentinel.discover_master('mymaster') -('127.0.0.1', 6379) ->>> sentinel.discover_slaves('mymaster') -[('127.0.0.1', 6380)] -``` - -To connect to a sentinel which uses SSL ([see SSL -connections](#ssl-connections) for more examples of SSL configurations): - -``` pycon ->>> from redis import Sentinel ->>> sentinel = Sentinel([('localhost', 26379)], - ssl=True, - ssl_ca_certs='/etc/ssl/certs/ca-certificates.crt') ->>> sentinel.discover_master('mymaster') -('127.0.0.1', 6379) -``` - -You can also create Redis client connections from a Sentinel instance. -You can connect to either the master (for write operations) or a slave -(for read-only operations). - -``` pycon ->>> master = sentinel.master_for('mymaster', socket_timeout=0.1) ->>> slave = sentinel.slave_for('mymaster', socket_timeout=0.1) ->>> master.set('foo', 'bar') ->>> slave.get('foo') -b'bar' -``` - -The master and slave objects are normal Redis instances with their -connection pool bound to the Sentinel instance. When a Sentinel backed -client attempts to establish a connection, it first queries the Sentinel -servers to determine an appropriate host to connect to. If no server is -found, a MasterNotFoundError or SlaveNotFoundError is raised. Both -exceptions are subclasses of ConnectionError. - -When trying to connect to a slave client, the Sentinel connection pool -will iterate over the list of slaves until it finds one that can be -connected to. If no slaves can be connected to, a connection will be -established with the master. - -See [Guidelines for Redis clients with support for Redis -Sentinel](https://redis.io/topics/sentinel-clients) to learn more about -Redis Sentinel. - --------------------------- - -### Parsers - -Parser classes provide a way to control how responses from the Redis -server are parsed. redis-py ships with two parser classes, the -PythonParser and the HiredisParser. By default, redis-py will attempt to -use the HiredisParser if you have the hiredis module installed and will -fallback to the PythonParser otherwise. - -Hiredis is a C library maintained by the core Redis team. Pieter -Noordhuis was kind enough to create Python bindings. Using Hiredis can -provide up to a 10x speed improvement in parsing responses from the -Redis server. The performance increase is most noticeable when -retrieving many pieces of data, such as from LRANGE or SMEMBERS -operations. - -Hiredis is available on PyPI, and can be installed via pip just like -redis-py. - -``` bash -$ pip install hiredis -``` - -### Response Callbacks - -The client class uses a set of callbacks to cast Redis responses to the -appropriate Python type. There are a number of these callbacks defined -on the Redis client class in a dictionary called RESPONSE_CALLBACKS. - -Custom callbacks can be added on a per-instance basis using the -set_response_callback method. This method accepts two arguments: a -command name and the callback. Callbacks added in this manner are only -valid on the instance the callback is added to. If you want to define or -override a callback globally, you should make a subclass of the Redis -client and add your callback to its RESPONSE_CALLBACKS class dictionary. - -Response callbacks take at least one parameter: the response from the -Redis server. Keyword arguments may also be accepted in order to further -control how to interpret the response. These keyword arguments are -specified during the command\'s call to execute_command. The ZRANGE -implementation demonstrates the use of response callback keyword -arguments with its \"withscores\" argument. - -### Thread Safety - -Redis client instances can safely be shared between threads. Internally, -connection instances are only retrieved from the connection pool during -command execution, and returned to the pool directly after. Command -execution never modifies state on the client instance. - -However, there is one caveat: the Redis SELECT command. The SELECT -command allows you to switch the database currently in use by the -connection. That database remains selected until another is selected or -until the connection is closed. This creates an issue in that -connections could be returned to the pool that are connected to a -different database. - -As a result, redis-py does not implement the SELECT command on client -instances. If you use multiple Redis databases within the same -application, you should create a separate client instance (and possibly -a separate connection pool) for each database. - -It is not safe to pass PubSub or Pipeline objects between threads. +For more details, please see the documentation on [advanced topics page](https://redis.readthedocs.io/en/stable/advanced_features.html). ### Pipelines -Pipelines are a subclass of the base Redis class that provide support -for buffering multiple commands to the server in a single request. They -can be used to dramatically increase the performance of groups of -commands by reducing the number of back-and-forth TCP packets between -the client and server. +The following is a basic example of a [Redis pipeline](https://redis.io/docs/manual/pipelining/), a method to optimize round-trip calls, by batching Redis commands, and receiving their results as a list. -Pipelines are quite simple to use: -``` pycon ->>> r = redis.Redis(...) ->>> r.set('bing', 'baz') ->>> # Use the pipeline() method to create a pipeline instance +``` python >>> pipe = r.pipeline() ->>> # The following SET commands are buffered ->>> pipe.set('foo', 'bar') ->>> pipe.get('bing') ->>> # the EXECUTE call sends all buffered commands to the server, returning ->>> # a list of responses, one for each command. +>>> pipe.set('foo', 5) +>>> pipe.set('bar', 18.5) +>>> pipe.set('blee', "hello world!") >>> pipe.execute() -[True, b'baz'] -``` - -For ease of use, all commands being buffered into the pipeline return -the pipeline object itself. Therefore calls can be chained like: - -``` pycon ->>> pipe.set('foo', 'bar').sadd('faz', 'baz').incr('auto_number').execute() -[True, True, 6] -``` - -In addition, pipelines can also ensure the buffered commands are -executed atomically as a group. This happens by default. If you want to -disable the atomic nature of a pipeline but still want to buffer -commands, you can turn off transactions. - -``` pycon ->>> pipe = r.pipeline(transaction=False) -``` - -A common issue occurs when requiring atomic transactions but needing to -retrieve values in Redis prior for use within the transaction. For -instance, let\'s assume that the INCR command didn\'t exist and we need -to build an atomic version of INCR in Python. - -The completely naive implementation could GET the value, increment it in -Python, and SET the new value back. However, this is not atomic because -multiple clients could be doing this at the same time, each getting the -same value from GET. - -Enter the WATCH command. WATCH provides the ability to monitor one or -more keys prior to starting a transaction. If any of those keys change -prior the execution of that transaction, the entire transaction will be -canceled and a WatchError will be raised. To implement our own -client-side INCR command, we could do something like this: - -``` pycon ->>> with r.pipeline() as pipe: -... while True: -... try: -... # put a WATCH on the key that holds our sequence value -... pipe.watch('OUR-SEQUENCE-KEY') -... # after WATCHing, the pipeline is put into immediate execution -... # mode until we tell it to start buffering commands again. -... # this allows us to get the current value of our sequence -... current_value = pipe.get('OUR-SEQUENCE-KEY') -... next_value = int(current_value) + 1 -... # now we can put the pipeline back into buffered mode with MULTI -... pipe.multi() -... pipe.set('OUR-SEQUENCE-KEY', next_value) -... # and finally, execute the pipeline (the set command) -... pipe.execute() -... # if a WatchError wasn't raised during execution, everything -... # we just did happened atomically. -... break -... except WatchError: -... # another client must have changed 'OUR-SEQUENCE-KEY' between -... # the time we started WATCHing it and the pipeline's execution. -... # our best bet is to just retry. -... continue -``` - -Note that, because the Pipeline must bind to a single connection for the -duration of a WATCH, care must be taken to ensure that the connection is -returned to the connection pool by calling the reset() method. If the -Pipeline is used as a context manager (as in the example above) reset() -will be called automatically. Of course you can do this the manual way -by explicitly calling reset(): - -``` pycon ->>> pipe = r.pipeline() ->>> while True: -... try: -... pipe.watch('OUR-SEQUENCE-KEY') -... ... -... pipe.execute() -... break -... except WatchError: -... continue -... finally: -... pipe.reset() -``` - -A convenience method named \"transaction\" exists for handling all the -boilerplate of handling and retrying watch errors. It takes a callable -that should expect a single parameter, a pipeline object, and any number -of keys to be WATCHed. Our client-side INCR command above can be written -like this, which is much easier to read: - -``` pycon ->>> def client_side_incr(pipe): -... current_value = pipe.get('OUR-SEQUENCE-KEY') -... next_value = int(current_value) + 1 -... pipe.multi() -... pipe.set('OUR-SEQUENCE-KEY', next_value) ->>> ->>> r.transaction(client_side_incr, 'OUR-SEQUENCE-KEY') -[True] +[True, True, True] ``` -Be sure to call pipe.multi() in the callable passed to -Redis.transaction prior to any write commands. +### PubSub -### Publish / Subscribe +The following example shows how to utilize [Redis Pub/Sub](https://redis.io/docs/manual/pubsub/) to subscribe to specific channels. -redis-py includes a PubSub object that subscribes to -channels and listens for new messages. Creating a PubSub -object is easy. - -``` pycon +``` python >>> r = redis.Redis(...) >>> p = r.pubsub() -``` - -Once a PubSub instance is created, channels and patterns -can be subscribed to. - -``` pycon >>> p.subscribe('my-first-channel', 'my-second-channel', ...) ->>> p.psubscribe('my-*', ...) -``` - -The PubSub instance is now subscribed to those -channels/patterns. The subscription confirmations can be seen by reading -messages from the PubSub instance. - -``` pycon >>> p.get_message() {'pattern': None, 'type': 'subscribe', 'channel': b'my-second-channel', 'data': 1} ->>> p.get_message() -{'pattern': None, 'type': 'subscribe', 'channel': b'my-first-channel', 'data': 2} ->>> p.get_message() -{'pattern': None, 'type': 'psubscribe', 'channel': b'my-*', 'data': 3} ``` -Every message read from a PubSub instance will be a -dictionary with the following keys. -- **type**: One of the following: \'subscribe\', \'unsubscribe\', - \'psubscribe\', \'punsubscribe\', \'message\', \'pmessage\' -- **channel**: The channel \[un\]subscribed to or the channel a - message was published to -- **pattern**: The pattern that matched a published message\'s - channel. Will be None in all cases except for - \'pmessage\' types. -- **data**: The message data. With \[un\]subscribe messages, this - value will be the number of channels and patterns the connection is - currently subscribed to. With \[p\]message messages, this value will - be the actual published message. - -Let\'s send a message now. - -``` pycon -# the publish method returns the number matching channel and pattern -# subscriptions. 'my-first-channel' matches both the 'my-first-channel' -# subscription and the 'my-*' pattern subscription, so this message will -# be delivered to 2 channels/patterns ->>> r.publish('my-first-channel', 'some data') -2 ->>> p.get_message() -{'channel': b'my-first-channel', 'data': b'some data', 'pattern': None, 'type': 'message'} ->>> p.get_message() -{'channel': b'my-first-channel', 'data': b'some data', 'pattern': b'my-*', 'type': 'pmessage'} -``` - -Unsubscribing works just like subscribing. If no arguments are passed to -\[p\]unsubscribe, all channels or patterns will be unsubscribed from. - -``` pycon ->>> p.unsubscribe() ->>> p.punsubscribe('my-*') ->>> p.get_message() -{'channel': b'my-second-channel', 'data': 2, 'pattern': None, 'type': 'unsubscribe'} ->>> p.get_message() -{'channel': b'my-first-channel', 'data': 1, 'pattern': None, 'type': 'unsubscribe'} ->>> p.get_message() -{'channel': b'my-*', 'data': 0, 'pattern': None, 'type': 'punsubscribe'} -``` - -redis-py also allows you to register callback functions to handle -published messages. Message handlers take a single argument, the -message, which is a dictionary just like the examples above. To -subscribe to a channel or pattern with a message handler, pass the -channel or pattern name as a keyword argument with its value being the -callback function. - -When a message is read on a channel or pattern with a message handler, -the message dictionary is created and passed to the message handler. In -this case, a None value is returned from get_message() -since the message was already handled. - -``` pycon ->>> def my_handler(message): -... print('MY HANDLER: ', message['data']) ->>> p.subscribe(**{'my-channel': my_handler}) -# read the subscribe confirmation message ->>> p.get_message() -{'pattern': None, 'type': 'subscribe', 'channel': b'my-channel', 'data': 1} ->>> r.publish('my-channel', 'awesome data') -1 -# for the message handler to work, we need tell the instance to read data. -# this can be done in several ways (read more below). we'll just use -# the familiar get_message() function for now ->>> message = p.get_message() -MY HANDLER: awesome data -# note here that the my_handler callback printed the string above. -# `message` is None because the message was handled by our handler. ->>> print(message) -None -``` - -If your application is not interested in the (sometimes noisy) -subscribe/unsubscribe confirmation messages, you can ignore them by -passing ignore_subscribe_messages=True to -r.pubsub(). This will cause all subscribe/unsubscribe -messages to be read, but they won\'t bubble up to your application. - -``` pycon ->>> p = r.pubsub(ignore_subscribe_messages=True) ->>> p.subscribe('my-channel') ->>> p.get_message() # hides the subscribe message and returns None ->>> r.publish('my-channel', 'my data') -1 ->>> p.get_message() -{'channel': b'my-channel', 'data': b'my data', 'pattern': None, 'type': 'message'} -``` - -There are three different strategies for reading messages. - -The examples above have been using pubsub.get_message(). -Behind the scenes, get_message() uses the system\'s -\'select\' module to quickly poll the connection\'s socket. If there\'s -data available to be read, get_message() will read it, -format the message and return it or pass it to a message handler. If -there\'s no data to be read, get_message() will -immediately return None. This makes it trivial to integrate into an -existing event loop inside your application. - -``` pycon ->>> while True: ->>> message = p.get_message() ->>> if message: ->>> # do something with the message ->>> time.sleep(0.001) # be nice to the system :) -``` - -Older versions of redis-py only read messages with -pubsub.listen(). listen() is a generator that blocks until -a message is available. If your application doesn\'t need to do anything -else but receive and act on messages received from redis, listen() is an -easy way to get up an running. - -``` pycon ->>> for message in p.listen(): -... # do something with the message -``` - -The third option runs an event loop in a separate thread. -pubsub.run_in_thread() creates a new thread and starts the -event loop. The thread object is returned to the caller of -[un_in_thread(). The caller can use the -thread.stop() method to shut down the event loop and -thread. Behind the scenes, this is simply a wrapper around -get_message() that runs in a separate thread, essentially -creating a tiny non-blocking event loop for you. -run_in_thread() takes an optional sleep_time -argument. If specified, the event loop will call -time.sleep() with the value in each iteration of the loop. - -Note: Since we\'re running in a separate thread, there\'s no way to -handle messages that aren\'t automatically handled with registered -message handlers. Therefore, redis-py prevents you from calling -run_in_thread() if you\'re subscribed to patterns or -channels that don\'t have message handlers attached. - -``` pycon ->>> p.subscribe(**{'my-channel': my_handler}) ->>> thread = p.run_in_thread(sleep_time=0.001) -# the event loop is now running in the background processing messages -# when it's time to shut it down... ->>> thread.stop() -``` - -run_in_thread also supports an optional exception handler, -which lets you catch exceptions that occur within the worker thread and -handle them appropriately. The exception handler will take as arguments -the exception itself, the pubsub object, and the worker thread returned -by run_in_thread. - -``` pycon ->>> p.subscribe(**{'my-channel': my_handler}) ->>> def exception_handler(ex, pubsub, thread): ->>> print(ex) ->>> thread.stop() ->>> thread.join(timeout=1.0) ->>> pubsub.close() ->>> thread = p.run_in_thread(exception_handler=exception_handler) -``` - -A PubSub object adheres to the same encoding semantics as the client -instance it was created from. Any channel or pattern that\'s unicode -will be encoded using the charset specified on the client -before being sent to Redis. If the client\'s -decode_responses flag is set the False (the default), the -\'channel\', \'pattern\' and \'data\' values in message dictionaries -will be byte strings (str on Python 2, bytes on Python 3). If the -client\'s decode_responses is True, then the \'channel\', -\'pattern\' and \'data\' values will be automatically decoded to unicode -strings using the client\'s charset. - -PubSub objects remember what channels and patterns they are subscribed -to. In the event of a disconnection such as a network error or timeout, -the PubSub object will re-subscribe to all prior channels and patterns -when reconnecting. Messages that were published while the client was -disconnected cannot be delivered. When you\'re finished with a PubSub -object, call its .close() method to shutdown the -connection. - -``` pycon ->>> p = r.pubsub() ->>> ... ->>> p.close() -``` - -The PUBSUB set of subcommands CHANNELS, NUMSUB and NUMPAT are also -supported: - -``` pycon ->>> r.pubsub_channels() -[b'foo', b'bar'] ->>> r.pubsub_numsub('foo', 'bar') -[(b'foo', 9001), (b'bar', 42)] ->>> r.pubsub_numsub('baz') -[(b'baz', 0)] ->>> r.pubsub_numpat() -1204 -``` - -### Monitor - -redis-py includes a Monitor object that streams every -command processed by the Redis server. Use listen() on the -Monitor object to block until a command is received. - -``` pycon ->>> r = redis.Redis(...) ->>> with r.monitor() as m: ->>> for command in m.listen(): ->>> print(command) -``` - -### Lua Scripting - -redis-py supports the EVAL, EVALSHA, and SCRIPT commands. However, there -are a number of edge cases that make these commands tedious to use in -real world scenarios. Therefore, redis-py exposes a Script object that -makes scripting much easier to use. (RedisClusters have limited support for -scripting.) - -To create a Script instance, use the register_script -function on a client instance passing the Lua code as the first -argument. register_script returns a Script instance that -you can use throughout your code. - -The following trivial Lua script accepts two parameters: the name of a -key and a multiplier value. The script fetches the value stored in the -key, multiplies it with the multiplier value and returns the result. - -``` pycon ->>> r = redis.Redis() ->>> lua = """ -... local value = redis.call('GET', KEYS[1]) -... value = tonumber(value) -... return value * ARGV[1]""" ->>> multiply = r.register_script(lua) -``` - -multiply is now a Script instance that is invoked by -calling it like a function. Script instances accept the following -optional arguments: - -- **keys**: A list of key names that the script will access. This - becomes the KEYS list in Lua. -- **args**: A list of argument values. This becomes the ARGV list in - Lua. -- **client**: A redis-py Client or Pipeline instance that will invoke - the script. If client isn\'t specified, the client that initially - created the Script instance (the one that - register_script was invoked from) will be used. - -Continuing the example from above: - -``` pycon ->>> r.set('foo', 2) ->>> multiply(keys=['foo'], args=[5]) -10 -``` - -The value of key \'foo\' is set to 2. When multiply is invoked, the -\'foo\' key is passed to the script along with the multiplier value of -5. Lua executes the script and returns the result, 10. - -Script instances can be executed using a different client instance, even -one that points to a completely different Redis server. - -``` pycon ->>> r2 = redis.Redis('redis2.example.com') ->>> r2.set('foo', 3) ->>> multiply(keys=['foo'], args=[5], client=r2) -15 -``` - -The Script object ensures that the Lua script is loaded into Redis\'s -script cache. In the event of a NOSCRIPT error, it will load the script -and retry executing it. - -Script objects can also be used in pipelines. The pipeline instance -should be passed as the client argument when calling the script. Care is -taken to ensure that the script is registered in Redis\'s script cache -just prior to pipeline execution. - -``` pycon ->>> pipe = r.pipeline() ->>> pipe.set('foo', 5) ->>> multiply(keys=['foo'], args=[5], client=pipe) ->>> pipe.execute() -[True, 25] -``` - - -### Scan Iterators - -The \*SCAN commands introduced in Redis 2.8 can be cumbersome to use. -While these commands are fully supported, redis-py also exposes the -following methods that return Python iterators for convenience: -scan_iter, hscan_iter, -sscan_iter and zscan_iter. - -``` pycon ->>> for key, value in (('A', '1'), ('B', '2'), ('C', '3')): -... r.set(key, value) ->>> for key in r.scan_iter(): -... print(key, r.get(key)) -A 1 -B 2 -C 3 -``` - -### Cluster Mode - -redis-py now supports cluster mode and provides a client for -[Redis Cluster](). - -The cluster client is based on Grokzen's -[redis-py-cluster](https://github.com/Grokzen/redis-py-cluster), has added bug -fixes, and now supersedes that library. Support for these changes is thanks to -his contributions. - -To learn more about Redis Cluster, see -[Redis Cluster specifications](https://redis.io/topics/cluster-spec). - -**Create RedisCluster:** - -Connecting redis-py to a Redis Cluster instance(s) requires at a minimum a -single node for cluster discovery. There are multiple ways in which a cluster -instance can be created: - -- Using 'host' and 'port' arguments: - -``` pycon ->>> from redis.cluster import RedisCluster as Redis ->>> rc = Redis(host='localhost', port=6379) ->>> print(rc.get_nodes()) - [[host=127.0.0.1,port=6379,name=127.0.0.1:6379,server_type=primary,redis_connection=Redis>>], [host=127.0.0.1,port=6378,name=127.0.0.1:6378,server_type=primary,redis_connection=Redis>>], [host=127.0.0.1,port=6377,name=127.0.0.1:6377,server_type=replica,redis_connection=Redis>>]] -``` -- Using the Redis URL specification: - -``` pycon ->>> from redis.cluster import RedisCluster as Redis ->>> rc = Redis.from_url("redis://localhost:6379/0") -``` - -- Directly, via the ClusterNode class: - -``` pycon ->>> from redis.cluster import RedisCluster as Redis ->>> from redis.cluster import ClusterNode ->>> nodes = [ClusterNode('localhost', 6379), ClusterNode('localhost', 6378)] ->>> rc = Redis(startup_nodes=nodes) -``` - -When a RedisCluster instance is being created it first attempts to establish a -connection to one of the provided startup nodes. If none of the startup nodes -are reachable, a 'RedisClusterException' will be thrown. -After a connection to the one of the cluster's nodes is established, the -RedisCluster instance will be initialized with 3 caches: -a slots cache which maps each of the 16384 slots to the node/s handling them, -a nodes cache that contains ClusterNode objects (name, host, port, redis connection) -for all of the cluster's nodes, and a commands cache contains all the server -supported commands that were retrieved using the Redis 'COMMAND' output. -See *RedisCluster specific options* below for more. - -RedisCluster instance can be directly used to execute Redis commands. When a -command is being executed through the cluster instance, the target node(s) will -be internally determined. When using a key-based command, the target node will -be the node that holds the key's slot. -Cluster management commands and other commands that are not key-based have a -parameter called 'target_nodes' where you can specify which nodes to execute -the command on. In the absence of target_nodes, the command will be executed -on the default cluster node. As part of cluster instance initialization, the -cluster's default node is randomly selected from the cluster's primaries, and -will be updated upon reinitialization. Using r.get_default_node(), you can -get the cluster's default node, or you can change it using the -'set_default_node' method. - -The 'target_nodes' parameter is explained in the following section, -'Specifying Target Nodes'. - -``` pycon ->>> # target-nodes: the node that holds 'foo1's key slot ->>> rc.set('foo1', 'bar1') ->>> # target-nodes: the node that holds 'foo2's key slot ->>> rc.set('foo2', 'bar2') ->>> # target-nodes: the node that holds 'foo1's key slot ->>> print(rc.get('foo1')) -b'bar' ->>> # target-node: default-node ->>> print(rc.keys()) -[b'foo1'] ->>> # target-node: default-node ->>> rc.ping() -``` - -**Specifying Target Nodes:** - -As mentioned above, all non key-based RedisCluster commands accept the kwarg -parameter 'target_nodes' that specifies the node/nodes that the command should -be executed on. -The best practice is to specify target nodes using RedisCluster class's node -flags: PRIMARIES, REPLICAS, ALL_NODES, RANDOM. When a nodes flag is passed -along with a command, it will be internally resolved to the relevant node/s. -If the nodes topology of the cluster changes during the execution of a command, -the client will be able to resolve the nodes flag again with the new topology -and attempt to retry executing the command. - -``` pycon ->>> from redis.cluster import RedisCluster as Redis ->>> # run cluster-meet command on all of the cluster's nodes ->>> rc.cluster_meet('127.0.0.1', 6379, target_nodes=Redis.ALL_NODES) ->>> # ping all replicas ->>> rc.ping(target_nodes=Redis.REPLICAS) ->>> # ping a random node ->>> rc.ping(target_nodes=Redis.RANDOM) ->>> # get the keys from all cluster nodes ->>> rc.keys(target_nodes=Redis.ALL_NODES) -[b'foo1', b'foo2'] ->>> # execute bgsave in all primaries ->>> rc.bgsave(Redis.PRIMARIES) -``` - -You could also pass ClusterNodes directly if you want to execute a command on a -specific node / node group that isn't addressed by the nodes flag. However, if -the command execution fails due to cluster topology changes, a retry attempt -will not be made, since the passed target node/s may no longer be valid, and -the relevant cluster or connection error will be returned. - -``` pycon ->>> node = rc.get_node('localhost', 6379) ->>> # Get the keys only for that specific node ->>> rc.keys(target_nodes=node) ->>> # get Redis info from a subset of primaries ->>> subset_primaries = [node for node in rc.get_primaries() if node.port > 6378] ->>> rc.info(target_nodes=subset_primaries) -``` - -In addition, the RedisCluster instance can query the Redis instance of a -specific node and execute commands on that node directly. The Redis client, -however, does not handle cluster failures and retries. - -``` pycon ->>> cluster_node = rc.get_node(host='localhost', port=6379) ->>> print(cluster_node) -[host=127.0.0.1,port=6379,name=127.0.0.1:6379,server_type=primary,redis_connection=Redis>>] ->>> r = cluster_node.redis_connection ->>> r.client_list() -[{'id': '276', 'addr': '127.0.0.1:64108', 'fd': '16', 'name': '', 'age': '0', 'idle': '0', 'flags': 'N', 'db': '0', 'sub': '0', 'psub': '0', 'multi': '-1', 'qbuf': '26', 'qbuf-free': '32742', 'argv-mem': '10', 'obl': '0', 'oll': '0', 'omem': '0', 'tot-mem': '54298', 'events': 'r', 'cmd': 'client', 'user': 'default'}] ->>> # Get the keys only for that specific node ->>> r.keys() -[b'foo1'] -``` - -**Multi-key commands:** - -Redis supports multi-key commands in Cluster Mode, such as Set type unions or -intersections, mset and mget, as long as the keys all hash to the same slot. -By using RedisCluster client, you can use the known functions (e.g. mget, mset) -to perform an atomic multi-key operation. However, you must ensure all keys are -mapped to the same slot, otherwise a RedisClusterException will be thrown. -Redis Cluster implements a concept called hash tags that can be used in order -to force certain keys to be stored in the same hash slot, see -[Keys hash tag](https://redis.io/topics/cluster-spec#keys-hash-tags). -You can also use nonatomic for some of the multikey operations, and pass keys -that aren't mapped to the same slot. The client will then map the keys to the -relevant slots, sending the commands to the slots' node owners. Non-atomic -operations batch the keys according to their hash value, and then each batch is -sent separately to the slot's owner. - -``` pycon -# Atomic operations can be used when all keys are mapped to the same slot ->>> rc.mset({'{foo}1': 'bar1', '{foo}2': 'bar2'}) ->>> rc.mget('{foo}1', '{foo}2') -[b'bar1', b'bar2'] -# Non-atomic multi-key operations splits the keys into different slots ->>> rc.mset_nonatomic({'foo': 'value1', 'bar': 'value2', 'zzz': 'value3') ->>> rc.mget_nonatomic('foo', 'bar', 'zzz') -[b'value1', b'value2', b'value3'] -``` - -**Cluster PubSub:** - -When a ClusterPubSub instance is created without specifying a node, a single -node will be transparently chosen for the pubsub connection on the -first command execution. The node will be determined by: - 1. Hashing the channel name in the request to find its keyslot - 2. Selecting a node that handles the keyslot: If read_from_replicas is - set to true, a replica can be selected. - -*Known limitations with pubsub:* - -Pattern subscribe and publish do not currently work properly due to key slots. -If we hash a pattern like fo* we will receive a keyslot for that string but -there are endless possibilities for channel names based on this pattern - -unknowable in advance. This feature is not disabled but the commands are not -currently recommended for use. -See [redis-py-cluster documentation](https://redis-py-cluster.readthedocs.io/en/stable/pubsub.html) - for more. - -``` pycon ->>> p1 = rc.pubsub() -# p1 connection will be set to the node that holds 'foo' keyslot ->>> p1.subscribe('foo') -# p2 connection will be set to node 'localhost:6379' ->>> p2 = rc.pubsub(rc.get_node('localhost', 6379)) -``` - -**Read Only Mode** - -By default, Redis Cluster always returns MOVE redirection response on accessing -a replica node. You can overcome this limitation and scale read commands by -triggering READONLY mode. - -To enable READONLY mode pass read_from_replicas=True to RedisCluster -constructor. When set to true, read commands will be assigned between the -primary and its replications in a Round-Robin manner. - -READONLY mode can be set at runtime by calling the readonly() method with -target_nodes='replicas', and read-write access can be restored by calling the -readwrite() method. - -``` pycon ->>> from cluster import RedisCluster as Redis -# Use 'debug' log level to print the node that the command is executed on ->>> rc_readonly = Redis(startup_nodes=startup_nodes, -... read_from_replicas=True) ->>> rc_readonly.set('{foo}1', 'bar1') ->>> for i in range(0, 4): -... # Assigns read command to the slot's hosts in a Round-Robin manner -... rc_readonly.get('{foo}1') -# set command would be directed only to the slot's primary node ->>> rc_readonly.set('{foo}2', 'bar2') -# reset READONLY flag ->>> rc_readonly.readwrite(target_nodes='replicas') -# now the get command would be directed only to the slot's primary node ->>> rc_readonly.get('{foo}1') -``` - -**Cluster Pipeline** - -ClusterPipeline is a subclass of RedisCluster that provides support for Redis -pipelines in cluster mode. -When calling the execute() command, all the commands are grouped by the node -on which they will be executed, and are then executed by the respective nodes -in parallel. The pipeline instance will wait for all the nodes to respond -before returning the result to the caller. Command responses are returned as a -list sorted in the same order in which they were sent. -Pipelines can be used to dramatically increase the throughput of Redis Cluster -by significantly reducing the the number of network round trips between the -client and the server. - -``` pycon ->>> with rc.pipeline() as pipe: -... pipe.set('foo', 'value1') -... pipe.set('bar', 'value2') -... pipe.get('foo') -... pipe.get('bar') -... print(pipe.execute()) -[True, True, b'value1', b'value2'] -... pipe.set('foo1', 'bar1').get('foo1').execute() -[True, b'bar1'] -``` - -Please note: -- RedisCluster pipelines currently only support key-based commands. -- The pipeline gets its 'read_from_replicas' value from the cluster's parameter. -Thus, if read from replications is enabled in the cluster instance, the pipeline -will also direct read commands to replicas. -- The 'transaction' option is NOT supported in cluster-mode. In non-cluster mode, -the 'transaction' option is available when executing pipelines. This wraps the -pipeline commands with MULTI/EXEC commands, and effectively turns the pipeline -commands into a single transaction block. This means that all commands are -executed sequentially without any interruptions from other clients. However, -in cluster-mode this is not possible, because commands are partitioned -according to their respective destination nodes. This means that we can not -turn the pipeline commands into one transaction block, because in most cases -they are split up into several smaller pipelines. - -**Lua Scripting in Cluster Mode** - -Cluster mode has limited support for lua scripting. - -The following commands are supported, with caveats: -- `EVAL` and `EVALSHA`: The command is sent to the relevant node, depending on -the keys (i.e., in `EVAL "