feat: implement per-module library dependency filtering (issue #4572)

Use ocamldep output to filter cross-library file dependencies per module.
Instead of every module depending on all libraries, each module now depends
only on libraries whose entry modules it actually references.

- Lib_index maps library entry module names back to libraries, computed
  once per stanza and stored in Compilation_context
- read_immediate_deps_raw_of returns raw (unresolved) module names from
  ocamldep, including cross-library references previously discarded
- build_cm filters library deps using the index; conservative fallbacks
  for root module references and unresolvable names
- deps_of_entries extended to support per-file deps for unwrapped libraries
  via Lib_index.entry (Lib.t * Module.t option)
- Dep_graph.dir accessor added for link-time module detection

Signed-off-by: Robin Bate Boerop <me@robinbb.com>

Author: robinbb

src/dune_rules/compilation_context.ml

@@ -5,8 +5,8 @@ module Includes = struct
   type t = Command.Args.without_targets Command.Args.t Lib_mode.Cm_kind.Map.t
 
   (* Note: Library file dependencies (Hidden_deps) are now added per-module
-     in module_compilation.ml. This allows future work to depend only on
-     libraries that a module actually uses.
+     in module_compilation.ml based on ocamldep output. This allows us to
+     only depend on libraries that a module actually uses.
      See issue #4572: Finer dependency analysis between libraries. *)
   let make ~project ~direct_requires ~hidden_requires lib_config
     : _ Lib_mode.Cm_kind.Map.t
@@ -68,6 +68,7 @@ type t =
   ; parameters : Module_name.t list Resolve.Memo.t
   ; instances : Parameterised_instances.t Resolve.Memo.t option
   ; includes : Includes.t
+  ; lib_index : Lib_file_deps.Lib_index.t Resolve.Memo.t
   ; preprocessing : Pp_spec.t
   ; opaque : bool
   ; js_of_ocaml : Js_of_ocaml.In_context.t option Js_of_ocaml.Mode.Pair.t
@@ -95,6 +96,7 @@ let requires_hidden t = t.requires_hidden
 let requires_link t = Memo.Lazy.force t.requires_link
 let parameters t = t.parameters
 let includes t = t.includes
+let lib_index t = t.lib_index
 let preprocessing t = t.preprocessing
 let opaque t = t.opaque
 let js_of_ocaml t = t.js_of_ocaml
@@ -218,6 +220,11 @@ let create
   ; implements
   ; parameters
   ; includes = Includes.make ~project ~direct_requires ~hidden_requires ocaml.lib_config
+  ; lib_index =
+      (let open Resolve.Memo.O in
+       let* direct_libs = direct_requires
+       and* hidden_libs = hidden_requires in
+       Lib_file_deps.Lib_index.create super_context (direct_libs @ hidden_libs) ~for_)
   ; preprocessing
   ; opaque
   ; js_of_ocaml
@@ -318,6 +325,7 @@ let for_module_generated_at_link_time cctx ~requires ~module_ =
   ; flags = Ocaml_flags.empty
   ; requires_link = Memo.lazy_ (fun () -> requires)
   ; requires_compile = requires
+  ; lib_index = Resolve.Memo.return Lib_file_deps.Lib_index.empty
   ; includes
   ; modules
   }

src/dune_rules/compilation_context.mli

@@ -62,6 +62,7 @@ val requires_hidden : t -> Lib.t list Resolve.Memo.t
 val requires_compile : t -> Lib.t list Resolve.Memo.t
 val parameters : t -> Module_name.t list Resolve.Memo.t
 val includes : t -> Command.Args.without_targets Command.Args.t Lib_mode.Cm_kind.Map.t
+val lib_index : t -> Lib_file_deps.Lib_index.t Resolve.Memo.t
 val preprocessing : t -> Pp_spec.t
 val opaque : t -> bool
 val js_of_ocaml : t -> Js_of_ocaml.In_context.t option Js_of_ocaml.Mode.Pair.t

src/dune_rules/dep_graph.ml

@@ -7,6 +7,7 @@ type t =
   }
 
 let make ~dir ~per_module = { dir; per_module }
+let dir t = t.dir
 
 let deps_of t (m : Module.t) =
   match Module_name.Unique.Map.find t.per_module (Module.obj_name m) with

src/dune_rules/dep_graph.mli

@@ -9,6 +9,7 @@ val make
   -> per_module:Module.t list Action_builder.t Module_name.Unique.Map.t
   -> t
 
+val dir : t -> Path.Build.t
 val deps_of : t -> Module.t -> Module.t list Action_builder.t
 val top_closed_implementations : t -> Module.t list -> Module.t list Action_builder.t
 

src/dune_rules/lib_file_deps.ml

@@ -48,23 +48,39 @@ let deps_of_lib (lib : Lib.t) ~groups =
   |> Dep.Set.of_list
 ;;
 
+let deps_of_module (lib : Lib.t) (m : Module.t) ~cm_kinds =
+  let obj_dir = Lib.info lib |> Lib_info.obj_dir in
+  List.filter_map cm_kinds ~f:(fun kind ->
+    Obj_dir.Module.cm_file obj_dir m ~kind |> Option.map ~f:(fun p -> Dep.file p))
+  |> Dep.Set.of_list
+;;
+
 let deps libs ~groups = Dep.Set.union_map libs ~f:(deps_of_lib ~groups)
 
-let deps_of_entries ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) (libs : Lib.t list) =
-  let groups =
+let deps_of_entries ~opaque ~(cm_kind : Lib_mode.Cm_kind.t) entries =
+  let groups, cm_kinds =
     match cm_kind with
-    | Ocaml Cmi | Ocaml Cmo -> [ Group.Ocaml Cmi ]
-    | Melange Cmi -> [ Group.Melange Cmi ]
-    | Melange Cmj -> [ Group.Melange Cmi; Melange Cmj ]
-    | Ocaml Cmx -> [ Group.Ocaml Cmi; Ocaml Cmx ]
+    | Ocaml Cmi | Ocaml Cmo -> [ Group.Ocaml Cmi ], [ Lib_mode.Cm_kind.Ocaml Cmi ]
+    | Melange Cmi -> [ Group.Melange Cmi ], [ Melange Cmi ]
+    | Melange Cmj -> [ Group.Melange Cmi; Melange Cmj ], [ Melange Cmi; Melange Cmj ]
+    | Ocaml Cmx -> [ Group.Ocaml Cmi; Ocaml Cmx ], [ Ocaml Cmi; Ocaml Cmx ]
   in
-  Dep.Set.union_map libs ~f:(fun lib ->
-    let groups =
+  List.map entries ~f:(fun ((lib : Lib.t), mod_opt) ->
+    let is_opaque_local =
       match cm_kind with
-      | Ocaml Cmx when opaque && Lib.is_local lib -> [ Group.Ocaml Cmi ]
-      | _ -> groups
+      | Ocaml Cmx -> opaque && Lib.is_local lib
+      | _ -> false
     in
-    deps_of_lib lib ~groups)
+    match mod_opt with
+    | Some m ->
+      let cm_kinds =
+        if is_opaque_local then [ Lib_mode.Cm_kind.Ocaml Cmi ] else cm_kinds
+      in
+      deps_of_module lib m ~cm_kinds
+    | None ->
+      let groups = if is_opaque_local then [ Group.Ocaml Cmi ] else groups in
+      deps_of_lib lib ~groups)
+  |> List.fold_left ~init:Dep.Set.empty ~f:Dep.Set.union
 ;;
 
 type path_specification =
@@ -98,3 +114,92 @@ let eval ~loc ~expander ~paths:path_spec (deps : Dep_conf.t list) =
        | Some _ -> raise_disallowed_external_path ~loc lib_name path));
   paths
 ;;
+
+module Lib_index = struct
+  (** Each entry pairs a library with an optional Module.t: None means use
+      glob deps (wrapped libs, external unwrapped), Some m means use
+      per-file deps via Obj_dir.Module.cm_file (local unwrapped). *)
+  type entry = Lib.t * Module.t option
+
+  type t =
+    { by_module_name : entry list Module_name.Map.t
+    ; unresolved : Lib.t list
+    }
+
+  let module_names_of_lib sctx (lib : Lib.t) ~for_
+    : (Module_name.t * Module.t option) list option Resolve.Memo.t
+    =
+    let open Resolve.Memo.O in
+    let* main_module = Lib.main_module_name lib in
+    match main_module with
+    | Some name -> Resolve.Memo.return (Some [ name, None ])
+    | None ->
+      let info = Lib.info lib in
+      (match Lib_info.entry_modules info ~for_ with
+       | Lib_info.Source.External (Ok names) ->
+         Resolve.Memo.return (Some (List.map names ~f:(fun n -> n, None)))
+       | Lib_info.Source.External (Error _) -> Resolve.Memo.return None
+       | Lib_info.Source.Local ->
+         Resolve.Memo.lift_memo
+           (let open Memo.O in
+            let+ modules_opt = Dir_contents.modules_of_lib sctx lib ~for_ in
+            match modules_opt with
+            | None -> None
+            | Some modules_with_vlib ->
+              let modules = Modules.With_vlib.drop_vlib modules_with_vlib in
+              let entry_modules = Modules.entry_modules modules in
+              Some (List.map entry_modules ~f:(fun m -> Module.name m, Some m))))
+  ;;
+
+  let empty = { by_module_name = Module_name.Map.empty; unresolved = [] }
+
+  let create sctx (libs : Lib.t list) ~for_ : t Resolve.Memo.t =
+    let open Resolve.Memo.O in
+    let+ entries =
+      Resolve.Memo.List.map libs ~f:(fun lib ->
+        let+ names_opt = module_names_of_lib sctx lib ~for_ in
+        lib, names_opt)
+    in
+    let by_module_name, unresolved =
+      List.fold_left
+        entries
+        ~init:(Module_name.Map.empty, [])
+        ~f:(fun (by_name, unresolved) (lib, names_opt) ->
+          match names_opt with
+          | None -> by_name, lib :: unresolved
+          | Some named_modules ->
+            let by_name =
+              List.fold_left named_modules ~init:by_name ~f:(fun acc (name, mod_opt) ->
+                Module_name.Map.update acc name ~f:(function
+                  | None -> Some [ lib, mod_opt ]
+                  | Some entries -> Some ((lib, mod_opt) :: entries)))
+            in
+            by_name, unresolved)
+    in
+    { by_module_name; unresolved }
+  ;;
+
+  let filter_libs (index : t) ~(referenced_modules : Module_name.Set.t) : entry list =
+    let from_refs =
+      Module_name.Set.fold referenced_modules ~init:[] ~f:(fun name acc ->
+        match Module_name.Map.find index.by_module_name name with
+        | None -> acc
+        | Some entries -> List.rev_append entries acc)
+    in
+    let unresolved = List.map index.unresolved ~f:(fun lib -> lib, None) in
+    (* Entries are unique per (lib, module name). Two modules with the
+       same name can't appear for the same library since module names are
+       unique within a library's entry modules. *)
+    let compare (a_lib, a_mod) (b_lib, b_mod) =
+      match Lib.compare a_lib b_lib with
+      | (Lt | Gt) as c -> c
+      | Eq ->
+        (match a_mod, b_mod with
+         | None, None -> Eq
+         | None, Some _ -> Lt
+         | Some _, None -> Gt
+         | Some a, Some b -> Module_name.compare (Module.name a) (Module.name b))
+    in
+    List.rev_append unresolved from_refs |> List.sort_uniq ~compare
+  ;;
+end

src/dune_rules/lib_file_deps.mli

@@ -15,10 +15,6 @@ end
     with extension [files] of libraries [libs]. *)
 val deps : Lib.t list -> groups:Group.t list -> Dep.Set.t
 
-(** Compute library file dependencies for all [libs] for the given [cm_kind].
-    When [opaque] is true, local libraries only depend on .cmi (not .cmx). *)
-val deps_of_entries : opaque:bool -> cm_kind:Lib_mode.Cm_kind.t -> Lib.t list -> Dep.Set.t
-
 type path_specification =
   | Allow_all
   | Disallow_external of Lib_name.t
@@ -31,3 +27,27 @@ val eval
   -> paths:path_specification
   -> Dep_conf.t list
   -> Path.Set.t Memo.t
+
+module Lib_index : sig
+  type entry = Lib.t * Module.t option
+  type t
+
+  val empty : t
+
+  val create
+    :  Super_context.t
+    -> Lib.t list
+    -> for_:Compilation_mode.t
+    -> t Resolve.Memo.t
+
+  val filter_libs : t -> referenced_modules:Module_name.Set.t -> entry list
+end
+
+(** Compute library file dependencies for the given entries and cm_kind.
+    Entries with [Some module_] use per-file deps; [None] uses glob deps.
+    When [opaque] is true, local libraries only depend on .cmi (not .cmx). *)
+val deps_of_entries
+  :  opaque:bool
+  -> cm_kind:Lib_mode.Cm_kind.t
+  -> Lib_index.entry list
+  -> Dep.Set.t

src/dune_rules/module_compilation.ml

@@ -286,28 +286,86 @@ let build_cm
         | Some All | None -> Hidden_targets [ obj ])
    in
    let opaque = Compilation_context.opaque cctx in
-   (* Library file dependencies, added per-module. Alias and Wrapped_compat
-      modules are compiled with Includes.empty and need no library file deps.
-      All other modules depend on all libraries in the stanza's requires. *)
+   (* Library file dependencies - filtered per-module based on ocamldep output.
+      Issue #4572: Finer dependency analysis between libraries. *)
    let lib_cm_deps =
+     let requires_compile = Compilation_context.requires_compile cctx in
+     let requires_hidden = Compilation_context.requires_hidden cctx in
+     let for_ = Compilation_context.for_ cctx in
+     let stanza_modules = Compilation_context.modules cctx in
+     (* Alias modules are compiled with Includes.empty (no -I flags for
+        libraries) by for_alias_module, and Wrapped_compat modules are
+        compiled with Includes.empty by for_wrapped_compat. Neither needs
+        library file deps — they can't access library .cmi files regardless.
+        Skipping them also avoids a cascade where wrapper .cmi changes
+        would trigger recompilation of all stanza modules.
+        Other special modules (Root, etc.) and synthetic contexts (dir
+        mismatch) fall back to all-library glob deps. *)
      let skip_lib_deps =
        match Module.kind m with
        | Alias _ | Wrapped_compat -> true
        | _ -> false
      in
+     let can_filter =
+       let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in
+       let dep_graph_dir = Dep_graph.dir dep_graph in
+       (not skip_lib_deps)
+       && Path.Build.equal dep_graph_dir (Obj_dir.dir obj_dir)
+       &&
+       match Module.kind m with
+       | Root | Wrapped_compat | Impl_vmodule | Virtual | Parameter -> false
+       | _ -> Module.has m ~ml_kind
+     in
      Action_builder.dyn_deps
        (if skip_lib_deps
         then Action_builder.return ((), Dep.Set.empty)
         else
           let open Action_builder.O in
-          let+ all_libs =
+          let* all_libs, lib_index_opt =
             Resolve.Memo.read
               (let open Resolve.Memo.O in
-               let+ direct_libs = Compilation_context.requires_compile cctx
-               and+ hidden_libs = Compilation_context.requires_hidden cctx in
-               direct_libs @ hidden_libs)
+               let* direct_libs = requires_compile
+               and* hidden_libs = requires_hidden in
+               let all_libs = direct_libs @ hidden_libs in
+               if can_filter && not (List.is_empty all_libs)
+               then
+                 let+ lib_index = Compilation_context.lib_index cctx in
+                 all_libs, Some lib_index
+               else Resolve.Memo.return (all_libs, None))
           in
-          (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind all_libs)
+          let all_entries = List.map all_libs ~f:(fun lib -> lib, None) in
+          match lib_index_opt with
+          | None ->
+            Action_builder.return
+              ((), Lib_file_deps.deps_of_entries ~opaque ~cm_kind all_entries)
+          | Some lib_index ->
+            let+ referenced_modules =
+              Ocamldep.read_immediate_deps_raw_of ~obj_dir ~ml_kind ~for_ m
+            in
+            let references_root_module =
+              Module_name.Set.exists referenced_modules ~f:(fun name ->
+                match Modules.With_vlib.find stanza_modules name with
+                | Some m -> Module.kind m = Root
+                | None -> false)
+            in
+            let external_modules =
+              Module_name.Set.filter referenced_modules ~f:(fun name ->
+                Option.is_none (Modules.With_vlib.find stanza_modules name))
+            in
+            let used_entries =
+              if references_root_module
+              then all_entries
+              else (
+                let used =
+                  Lib_file_deps.Lib_index.filter_libs
+                    lib_index
+                    ~referenced_modules:external_modules
+                in
+                if List.is_empty used && not (Module_name.Set.is_empty external_modules)
+                then all_entries
+                else used)
+            in
+            (), Lib_file_deps.deps_of_entries ~opaque ~cm_kind used_entries)
    in
    let other_cm_files =
      let dep_graph = Ml_kind.Dict.get (Compilation_context.dep_graphs cctx) ml_kind in