From b01b068410884da32d6fe4d353cae2a21de40ca6 Mon Sep 17 00:00:00 2001 From: Hugues Bruant Date: Tue, 22 Sep 2020 17:23:07 -0700 Subject: [PATCH 1/2] Speed up typechecking of dict, set and list expressions Typechecking of dict, set, and list literals currentlly goes through typechecking of the generic dict/set/list constructor internally. This is usually fine but becomes horrendously slow when the number of items is large: - for generic methods, `infer_arg_types_in_context` is called twice - `infer_arg_types_in_context` is O(n**2) where `n` is the number of arguments, which, in the case of a literal, is the number of items. Add an `O(n)` fast path for deriving the type of simple container literal expressions. This fast path only handle a subset of cases but it provides a tremendous speedup for the relatively common case of large literal constants. The real-world example that motivated this change is a 1889 lines long dict constant representing the parsed value of a mock JSON response from a 3rd party service, where typechecking previously took upwards of 50s and is now down to under 1s with this fast path. --- mypy/checkexpr.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index a44bc3f7ed20d..ec3d3dc9132e9 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -3175,8 +3175,32 @@ def visit_list_expr(self, e: ListExpr) -> Type: def visit_set_expr(self, e: SetExpr) -> Type: return self.check_lst_expr(e.items, 'builtins.set', '', e) + def fast_container_type( + self, items: List[Expression], container_fullname: str + ) -> Optional[Type]: + ctx = self.type_context[-1] + if ctx: + return None + values = [] # type: List[Type] + for item in items: + if isinstance(item, StarExpr): + # fallback to slow path + return None + values.append(self.accept(item)) + vt = join.join_type_list(values) + if not isinstance(vt, Instance): + return None + # TODO: update tests instead? + vt.erased = True + return self.chk.named_generic_type(container_fullname, [vt]) + def check_lst_expr(self, items: List[Expression], fullname: str, tag: str, context: Context) -> Type: + # fast path + t = self.fast_container_type(items, fullname) + if t: + return t + # Translate into type checking a generic function call. # Used for list and set expressions, as well as for tuples # containing star expressions that don't refer to a @@ -3258,6 +3282,38 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type: fallback_item = AnyType(TypeOfAny.special_form) return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) + def fast_dict_type(self, e: DictExpr) -> Optional[Type]: + ctx = self.type_context[-1] + if ctx: + return None + keys = [] # type: List[Type] + values = [] # type: List[Type] + stargs = None # type: Optional[Tuple[Type, Type]] + for key, value in e.items: + if key is None: + st = get_proper_type(self.accept(value)) + if ( + isinstance(st, Instance) + and st.type.fullname == 'builtins.dict' + and len(st.args) == 2 + ): + stargs = (st.args[0], st.args[1]) + else: + return None + else: + keys.append(self.accept(key)) + values.append(self.accept(value)) + kt = join.join_type_list(keys) + vt = join.join_type_list(values) + if not (isinstance(kt, Instance) and isinstance(vt, Instance)): + return None + if stargs and (stargs[0] != kt or stargs[1] != vt): + return None + # TODO: update tests instead? + kt.erased = True + vt.erased = True + return self.chk.named_generic_type('builtins.dict', [kt, vt]) + def visit_dict_expr(self, e: DictExpr) -> Type: """Type check a dict expression. @@ -3276,6 +3332,11 @@ def visit_dict_expr(self, e: DictExpr) -> Type: ) return typeddict_context.copy_modified() + # fast path attempt + dt = self.fast_dict_type(e) + if dt: + return dt + # Collect function arguments, watching out for **expr. args = [] # type: List[Expression] # Regular "key: value" stargs = [] # type: List[Expression] # For "**expr" From 5d470ba2bec0725c739aa974c5cb8c2fa9b8a790 Mon Sep 17 00:00:00 2001 From: Hugues Bruant Date: Sun, 18 Oct 2020 01:13:00 -0700 Subject: [PATCH 2/2] add doctstrings --- mypy/checkexpr.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ec3d3dc9132e9..cdc69186d4851 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -3178,6 +3178,16 @@ def visit_set_expr(self, e: SetExpr) -> Type: def fast_container_type( self, items: List[Expression], container_fullname: str ) -> Optional[Type]: + """ + Fast path to determine the type of a list or set literal, + based on the list of entries. This mostly impacts large + module-level constant definitions. + + Limitations: + - no active type context + - no star expressions + - the joined type of all entries must be an Instance type + """ ctx = self.type_context[-1] if ctx: return None @@ -3283,6 +3293,16 @@ def visit_tuple_expr(self, e: TupleExpr) -> Type: return TupleType(items, self.chk.named_generic_type('builtins.tuple', [fallback_item])) def fast_dict_type(self, e: DictExpr) -> Optional[Type]: + """ + Fast path to determine the type of a dict literal, + based on the list of entries. This mostly impacts large + module-level constant definitions. + + Limitations: + - no active type context + - only supported star expressions are other dict instances + - the joined types of all keys and values must be Instance types + """ ctx = self.type_context[-1] if ctx: return None