Always inline function declared with an [@inline] annotation (#422)

Author: poechsel

middle_end/flambda/inlining/inlining_decision.ml

@@ -49,14 +49,19 @@ module Function_declaration_decision = struct
         large_function_size: Code_size.t;
       }
 
-  let can_inline t =
+  type inlining_behaviour =
+    | Cannot_be_inlined
+    | Must_be_inlined
+    | Could_possibly_be_inlined
+
+  let behaviour t =
     match t with
     | Never_inline_attribute
-    | Function_body_too_large _ -> false
+    | Function_body_too_large _ -> Cannot_be_inlined
     | Stub
     | Attribute_inline
-    | Small_function _
-    | Speculatively_inlinable _-> true
+    | Small_function _ -> Must_be_inlined
+    | Speculatively_inlinable _-> Could_possibly_be_inlined
 
   let print fmt = function
     | Never_inline_attribute ->
@@ -125,7 +130,11 @@ module Function_declaration_decision = struct
   let report fmt t =
     Format.fprintf fmt "@[<v>The function %s be inlined at its use-sites@ \
                         because @[<hov>%a@]@]"
-      (if can_inline t then "can" else "cannot") report_reason t
+      (match behaviour t with
+       | Cannot_be_inlined -> "cannot"
+       | Could_possibly_be_inlined -> "could"
+       | Must_be_inlined -> "must")
+      report_reason t
 
 end
 
@@ -189,15 +198,12 @@ module Call_site_decision = struct
       }
     | Attribute_always
     | Attribute_unroll of int
+    | Definition_says_inline
     | Speculatively_inline of {
         cost_metrics: Cost_metrics.t;
         evaluated_to: float;
         threshold: float;
       }
-    | Small_function of {
-        size: Code_size.t;
-        small_function_size: Code_size.t;
-      }
 
   let print ppf t =
     match t with
@@ -213,6 +219,8 @@ module Call_site_decision = struct
       Format.fprintf ppf "Never_inline_attribute"
     | Attribute_always ->
       Format.fprintf ppf "Attribute_unroll"
+    | Definition_says_inline ->
+      Format.fprintf ppf "Definition_says_inline"
     | Attribute_unroll unroll_to ->
       Format.fprintf ppf
         "@[<hov 1>(Attribute_unroll@ \
@@ -239,14 +247,6 @@ module Call_site_decision = struct
         Cost_metrics.print cost_metrics
         evaluated_to
         threshold
-    | Small_function { size; small_function_size; } ->
-      Format.fprintf ppf
-        "@[<hov 1>(Small_function@ \
-          @[<hov 1>(size@ %a)@]@ \
-          @[<hov 1>(small_function_size@ %a)@]\
-          )@]"
-        Code_size.print size
-        Code_size.print small_function_size
 
   type can_inline =
     | Do_not_inline
@@ -261,8 +261,8 @@ module Call_site_decision = struct
     | Speculatively_not_inline _
     | Never_inline_attribute -> Do_not_inline
     | Attribute_unroll unroll_to -> Inline { unroll_to = Some unroll_to; }
+    | Definition_says_inline
     | Speculatively_inline _
-    | Small_function _
     | Attribute_always -> Inline { unroll_to = None; }
 
   let report_reason fmt t =
@@ -281,6 +281,11 @@ module Call_site_decision = struct
       Format.fprintf fmt "the@ call@ has@ an@ [@@inline always]@ attribute"
     | Attribute_unroll n ->
       Format.fprintf fmt "the@ call@ has@ an@ [@@unroll %d]@ attribute" n
+    | Definition_says_inline ->
+      Format.fprintf fmt "this@ function@ was@ decided@ to@ be@ always@ \
+                          inlined@ at@ its@ definition@ site (annotated@ by@ \
+                          [@inlined always]@ or@ determined@ to@ be@ small@ \
+                          enough)"
     | Speculatively_not_inline { cost_metrics; evaluated_to; threshold } ->
       Format.fprintf fmt
         "the@ function@ was@ not@ inlined@ after@ speculation@ as@ \
@@ -297,13 +302,6 @@ module Call_site_decision = struct
         Cost_metrics.print cost_metrics
         evaluated_to
         threshold
-    | Small_function { size; small_function_size; } ->
-      Format.fprintf fmt
-        "the@ function@ was@ classified@ as@ a@ small@ \
-          function@ and@ was@ therefore@ inlined:@ \
-          size=%a <= small function size=%a"
-        Code_size.print size
-        Code_size.print small_function_size
 
   let report fmt t =
     Format.fprintf fmt
@@ -386,28 +384,20 @@ let speculative_inlining dacc ~apply ~function_decl ~simplify_expr
 let might_inline dacc ~apply ~function_decl ~simplify_expr ~return_arity
   : Call_site_decision.t =
   let denv = DA.denv dacc in
-  let code_id = I.code_id function_decl in
-  let code = DE.find_code denv code_id in
-  let cost_metrics = Code.cost_metrics code in
-  let args =
-    Apply.inlining_arguments apply
-    |> Inlining_arguments.meet (DA.denv dacc |> DE.inlining_arguments)
-  in
-  let size = Cost_metrics.size cost_metrics in
-  let small_function_size =
-    Inlining_arguments.small_function_size args |> Code_size.of_int
-  in
-  let is_a_small_function = Code_size.(<=) size small_function_size in
   let env_prohibits_inlining = not (DE.can_inline denv) in
-  if is_a_small_function then
-    Small_function { size; small_function_size }
+  if I.must_be_inlined function_decl then
+    Definition_says_inline
   else if env_prohibits_inlining then
     Environment_says_never_inline
   else
     let cost_metrics =
       speculative_inlining ~apply dacc ~simplify_expr ~return_arity
         ~function_decl
     in
+    let args =
+      Apply.inlining_arguments apply
+      |> Inlining_arguments.meet (DA.denv dacc |> DE.inlining_arguments)
+    in
     let evaluated_to = Cost_metrics.evaluate ~args cost_metrics in
     let threshold = Inlining_arguments.threshold args in
     let is_under_inline_threshold =

middle_end/flambda/inlining/inlining_decision.mli

@@ -38,7 +38,12 @@ module Function_declaration_decision : sig
 
   val report : Format.formatter -> t -> unit
 
-  val can_inline : t -> bool
+  type inlining_behaviour = private
+    | Cannot_be_inlined
+    | Must_be_inlined
+    | Could_possibly_be_inlined
+
+  val behaviour : t -> inlining_behaviour
 end
 
 
@@ -70,15 +75,12 @@ module Call_site_decision : sig
       }
     | Attribute_always
     | Attribute_unroll of int
+    | Definition_says_inline
     | Speculatively_inline of {
         cost_metrics: Cost_metrics.t;
         evaluated_to: float;
         threshold: float;
       }
-    | Small_function of {
-        size: Code_size.t;
-        small_function_size: Code_size.t;
-      }
 
 
   val print : Format.formatter -> t -> unit

middle_end/flambda/simplify/simplify_set_of_closures.ml

@@ -36,16 +36,25 @@ let function_decl_type ~pass ~cost_metrics_source denv function_decl ?code_id
   Inlining_report.record_decision (
     At_function_declaration { code_id = Code_id.export code_id; pass; decision; })
     ~dbg:(DE.add_inlined_debuginfo' denv (FD.dbg function_decl));
-  if Inlining_decision.Function_declaration_decision.can_inline decision then
+  match Inlining_decision.Function_declaration_decision.behaviour decision with
+  | Cannot_be_inlined ->
+    T.create_non_inlinable_function_declaration
+      ~code_id
+      ~is_tupled:(FD.is_tupled function_decl)
+  | Must_be_inlined ->
     T.create_inlinable_function_declaration
       ~code_id
       ~dbg:(FD.dbg function_decl)
       ~is_tupled:(FD.is_tupled function_decl)
+      ~must_be_inlined:true
       ~rec_info
-  else
-    T.create_non_inlinable_function_declaration
+  | Could_possibly_be_inlined ->
+    T.create_inlinable_function_declaration
       ~code_id
+      ~dbg:(FD.dbg function_decl)
       ~is_tupled:(FD.is_tupled function_decl)
+      ~must_be_inlined:false
+      ~rec_info
 
 module Context_for_multiple_sets_of_closures : sig
   (* This module deals with a sub-problem of the problem of simplifying multiple

middle_end/flambda/types/flambda_type.mli

@@ -245,6 +245,7 @@ module Function_declaration_type : sig
     val dbg : t -> Debuginfo.t
     val rec_info : t -> Rec_info.t
     val is_tupled : t -> bool
+    val must_be_inlined : t -> bool
   end
 
   module Non_inlinable : sig
@@ -414,6 +415,7 @@ val create_inlinable_function_declaration
   -> dbg:Debuginfo.t
   -> rec_info:Rec_info.t
   -> is_tupled:bool
+  -> must_be_inlined:bool
   -> Function_declaration_type.t
 
 (** Create a description of a function declaration whose code is unknown.

middle_end/flambda/types/structures/function_declaration_type.rec.ml

@@ -25,35 +25,41 @@ module Inlinable = struct
     dbg : Debuginfo.t;
     rec_info : Rec_info.t;
     is_tupled : bool;
+    must_be_inlined : bool;
   }
 
-  let print ppf { code_id; dbg; rec_info; is_tupled; } =
+  let print ppf { code_id; dbg; rec_info; is_tupled; must_be_inlined } =
     Format.fprintf ppf
       "@[<hov 1>(Inlinable@ \
         @[<hov 1>(code_id@ %a)@]@ \
-        @[<hov 1>(dbg@ %a)@] \
-        @[<hov 1>(rec_info@ %a)@]\
-        @[<hov 1><is_tupled@ %b)@]\
+        @[<hov 1>(dbg@ %a)@]@ \
+        @[<hov 1>(rec_info@ %a)@]@ \
+        @[<hov 1><is_tupled@ %b)@]@ \
+        @[<hov 1><must_be_inlined@ %b)@]\
         )@]"
       Code_id.print code_id
       Debuginfo.print_compact dbg
       Rec_info.print rec_info
       is_tupled
+      must_be_inlined
 
-  let create ~code_id ~dbg ~rec_info ~is_tupled =
+  let create ~code_id ~dbg ~rec_info ~is_tupled ~must_be_inlined =
     { code_id;
       dbg;
       rec_info;
       is_tupled;
+      must_be_inlined;
     }
 
   let code_id t = t.code_id
   let dbg t = t.dbg
   let rec_info t = t.rec_info
   let is_tupled t = t.is_tupled
+  let must_be_inlined t = t.must_be_inlined
 
   let apply_renaming
-        ({ code_id; dbg = _; rec_info = _; is_tupled = _; } as t) renaming =
+        ({ code_id; dbg = _; rec_info = _; is_tupled = _;
+           must_be_inlined = _ } as t) renaming =
     let code_id' = Renaming.apply_code_id renaming code_id in
     if code_id == code_id' then t
     else { t with code_id = code_id'; }
@@ -109,15 +115,17 @@ let print ppf t =
 let free_names (t : t) =
   match t with
   | Bottom | Unknown -> Name_occurrences.empty
-  | Ok (Inlinable { code_id; dbg = _; rec_info = _; is_tupled = _; })
+  | Ok (Inlinable { code_id; dbg = _; rec_info = _; is_tupled = _;
+                    must_be_inlined = _; })
   | Ok (Non_inlinable { code_id; is_tupled = _; }) ->
     Name_occurrences.add_code_id Name_occurrences.empty code_id
       Name_mode.in_types
 
 let all_ids_for_export (t : t) =
   match t with
   | Bottom | Unknown -> Ids_for_export.empty
-  | Ok (Inlinable { code_id; dbg = _; rec_info = _; is_tupled = _; })
+  | Ok (Inlinable { code_id; dbg = _; rec_info = _; is_tupled = _;
+                    must_be_inlined = _; })
   | Ok (Non_inlinable { code_id; is_tupled = _; }) ->
     Ids_for_export.add_code_id Ids_for_export.empty code_id
 
@@ -171,24 +179,28 @@ let meet (env : Meet_env.t) (t1 : t) (t2 : t)
       dbg = dbg1;
       rec_info = _rec_info1;
       is_tupled = is_tupled1;
+      must_be_inlined = must_be_inlined1;
     }),
     Ok (Inlinable {
       code_id = code_id2;
       dbg = dbg2;
       rec_info = _rec_info2;
       is_tupled = is_tupled2;
+      must_be_inlined = must_be_inlined2;
     }) ->
     let typing_env = Meet_env.env env in
     let target_code_age_rel = TE.code_age_relation typing_env in
     let resolver = TE.code_age_relation_resolver typing_env in
     let check_other_things_and_return code_id : (t * TEE.t) Or_bottom.t =
       assert (Int.equal (Debuginfo.compare dbg1 dbg2) 0);
       assert (Bool.equal is_tupled1 is_tupled2);
+      assert (Bool.equal must_be_inlined1 must_be_inlined2);
       Ok (Ok (Inlinable {
           code_id;
           dbg = dbg1;
           rec_info = _rec_info1;
           is_tupled = is_tupled1;
+          must_be_inlined = must_be_inlined1;
         }),
         TEE.empty ())
     in
@@ -243,24 +255,28 @@ let join (env : Join_env.t) (t1 : t) (t2 : t) : t =
       dbg = dbg1;
       rec_info = _rec_info1;
       is_tupled = is_tupled1;
+      must_be_inlined = must_be_inlined1;
     }),
     Ok (Inlinable {
       code_id = code_id2;
       dbg = dbg2;
       rec_info = _rec_info2;
       is_tupled = is_tupled2;
+      must_be_inlined = must_be_inlined2;
     }) ->
     let typing_env = Join_env.target_join_env env in
     let target_code_age_rel = TE.code_age_relation typing_env in
     let resolver = TE.code_age_relation_resolver typing_env in
     let check_other_things_and_return code_id : t =
       assert (Int.equal (Debuginfo.compare dbg1 dbg2) 0);
       assert (Bool.equal is_tupled1 is_tupled2);
+      assert (Bool.equal must_be_inlined1 must_be_inlined2);
       Ok (Inlinable {
         code_id;
         dbg = dbg1;
         rec_info = _rec_info1;
         is_tupled = is_tupled1;
+        must_be_inlined = must_be_inlined1;
       })
     in
     (* CR mshinwell: What about [rec_info]? *)
@@ -280,12 +296,14 @@ let join (env : Join_env.t) (t1 : t) (t2 : t) : t =
 
 let apply_rec_info (t : t) rec_info : t Or_bottom.t =
   match t with
-  | Ok (Inlinable { code_id; dbg; rec_info = rec_info'; is_tupled; }) ->
+  | Ok (Inlinable { code_id; dbg; rec_info = rec_info'; is_tupled;
+                    must_be_inlined }) ->
     let rec_info = Rec_info.merge rec_info' ~newer:rec_info in
     Ok (Ok (Inlinable { code_id;
       dbg;
       rec_info;
       is_tupled;
+      must_be_inlined;
     }))
   | Ok (Non_inlinable { code_id = _; is_tupled = _; }) -> Ok t
   | Unknown | Bottom -> Ok t

middle_end/flambda/types/structures/function_declaration_type.rec.mli

@@ -24,12 +24,14 @@ module Inlinable : sig
     -> dbg:Debuginfo.t
     -> rec_info:Rec_info.t
     -> is_tupled:bool
+    -> must_be_inlined:bool
     -> t
 
   val code_id : t -> Code_id.t
   val dbg : t -> Debuginfo.t
   val rec_info : t -> Rec_info.t
   val is_tupled : t -> bool
+  val must_be_inlined : t -> bool
 end
 
 module Non_inlinable : sig

middle_end/flambda/types/type_grammar.rec.ml

@@ -605,9 +605,10 @@ let any_boxed_int64 () = box_int64 (any_naked_int64 ())
 let any_boxed_nativeint () = box_nativeint (any_naked_nativeint ())
 
 let create_inlinable_function_declaration ~code_id ~dbg ~rec_info ~is_tupled
-      : Function_declaration_type.t =
+      ~must_be_inlined : Function_declaration_type.t =
   Ok (Inlinable (
-    Function_declaration_type.Inlinable.create ~code_id ~dbg ~rec_info ~is_tupled))
+    Function_declaration_type.Inlinable.create ~code_id ~dbg ~rec_info
+      ~is_tupled ~must_be_inlined))
 
 let create_non_inlinable_function_declaration ~code_id ~is_tupled
       : Function_declaration_type.t =