feature: use diff rather (mode promote) for generating opam files

Signed-off-by: Rudi Grinberg <me@rgrinberg.com>

Author: rgrinberg

src/dune_rules/install_rules.ml

@@ -17,12 +17,13 @@ let need_odoc_config (pkg : Package.t) =
 module Package_paths = struct
   let opam_file (ctx : Build_context.t) (pkg : Package.t) =
     let opam_file = Package.opam_file pkg in
-    let exists =
-      match Package.has_opam_file pkg with
-      | Exists b -> b
-      | Generated -> true
-    in
-    if exists then Some (Path.Build.append_source ctx.build_dir opam_file) else None
+    match Package.has_opam_file pkg with
+    | Exists false -> Memo.return None
+    | Exists true -> Memo.return (Some (Path.Build.append_source ctx.build_dir opam_file))
+    | Generated ->
+      let dir = Path.Build.append_source ctx.build_dir (Package.dir pkg) in
+      let+ project = Dune_load.find_project ~dir in
+      Some (Opam_create.build_path ~build_dir:ctx.build_dir ~project pkg)
   ;;
 
   let meta_fn pkg = "META." ^ Package.Name.to_string pkg
@@ -579,7 +580,7 @@ end = struct
     let* packages = Dune_load.packages () in
     let+ init =
       Package_map_traversals.parallel_map packages ~f:(fun _name (pkg : Package.t) ->
-        let opam_file = Package_paths.opam_file ctx pkg in
+        let* opam_file = Package_paths.opam_file ctx pkg in
         let init =
           let file section local_file dst =
             Install.Entry.Unexpanded.make

src/dune_rules/opam_create.ml

@@ -337,6 +337,13 @@ let opam_fields project (package : Package.t) =
 
 let template_file = Path.extend_basename ~suffix:".template"
 
+let build_path ~build_dir ~project pkg =
+  let opam_path = Path.Build.append_source build_dir (Package.opam_file pkg) in
+  if Dune_project.dune_version project < (3, 23)
+  then opam_path
+  else Path.Build.extend_basename opam_path ~suffix:".generated"
+;;
+
 let opam_template ~opam_path =
   let open Action_builder.O in
   let opam_template_path = template_file opam_path in
@@ -377,32 +384,60 @@ let generate project pkg ~template =
 let add_alias_rule (ctx : Build_context.t) ~project ~pkg =
   let build_dir = ctx.build_dir in
   let dir = Path.Build.append_source build_dir (Dune_project.root project) in
+  let source_opam_path = Package.opam_file pkg |> Path.source in
   let opam_path = Path.Build.append_source build_dir (Package.opam_file pkg) in
+  let generated_opam_path = build_path ~build_dir ~project pkg in
   let aliases =
     [ Alias.make Alias0.install ~dir
     ; Alias.make Alias0.runtest ~dir
     ; Alias.make Alias0.check ~dir (* check doesn't pick up the promote target? *)
     ]
   in
-  let deps = Path.Set.singleton (Path.build opam_path) in
+  let deps = Path.Set.singleton (Path.build generated_opam_path) in
   Memo.parallel_iter aliases ~f:(fun alias ->
-    (* TODO slow. we should be calling these functions only once, rather than
-       once per package *)
-    Rules.Produce.Alias.add_deps alias (Action_builder.path_set deps))
+    let* () =
+      (* TODO slow. we should be calling these functions only once, rather than
+         once per package *)
+      Rules.Produce.Alias.add_deps alias (Action_builder.path_set deps)
+    in
+    if Dune_project.dune_version project < (3, 23)
+    then Memo.return ()
+    else
+      Rules.Produce.Alias.add_action
+        alias
+        ~loc:(Loc.in_file source_opam_path)
+        (let open Action_builder.O in
+         let+ () = Action_builder.path (Path.build generated_opam_path)
+         and+ () =
+           Action_builder.if_file_exists
+             source_opam_path
+             ~then_:(Action_builder.path (Path.build opam_path))
+             ~else_:(Action_builder.return ())
+         in
+         Action.Full.make (Action.diff (Path.build opam_path) generated_opam_path)))
 ;;
 
 let add_opam_file_rule sctx ~project ~pkg =
-  let open Action_builder.O in
   let build_dir = Super_context.context sctx |> Context.build_dir in
   let opam_path = Path.Build.append_source build_dir (Package.opam_file pkg) in
+  let generated_opam_path = build_path ~build_dir ~project pkg in
   let opam_rule =
+    let open Action_builder.O in
     (let+ template = opam_template ~opam_path:(Path.build opam_path) in
      generate project pkg ~template)
-    |> Action_builder.write_file_dyn opam_path
+    |> Action_builder.write_file_dyn generated_opam_path
   in
   let dir = Path.Build.append_source build_dir (Dune_project.root project) in
-  let mode = Rule.Mode.Promote { lifetime = Unlimited; into = None; only = None } in
-  Super_context.add_rule sctx ~mode ~dir opam_rule
+  if Dune_project.dune_version project < (3, 23)
+  then (
+    let mode = Rule.Mode.Promote { lifetime = Unlimited; into = None; only = None } in
+    Super_context.add_rule sctx ~mode ~dir opam_rule)
+  else
+    let* () = Super_context.add_rule sctx ~dir opam_rule in
+    let visible_opam_rule =
+      Action_builder.copy ~src:(Path.build generated_opam_path) ~dst:opam_path
+    in
+    Super_context.add_rule sctx ~mode:Fallback ~dir visible_opam_rule
 ;;
 
 let add_opam_file_rules sctx project =

src/dune_rules/opam_create.mli

@@ -5,6 +5,13 @@ open Import
 (** Given an opam filename, returns the filename of the template file *)
 val template_file : Path.t -> Path.t
 
+(** Path to the generated opam file in the build directory. *)
+val build_path
+  :  build_dir:Path.Build.t
+  -> project:Dune_project.t
+  -> Package.t
+  -> Path.Build.t
+
 (** Generate the contents of an opam file. [template] is the filename and
     contents of the template file. *)
 val generate : Dune_project.t -> Package.t -> template:(Path.t * string) option -> string

test/blackbox-tests/test-cases/dune-init/dune-init.t/run.t

@@ -376,38 +376,6 @@ We can build the project:
 And the opam file will be generated as expected
 
   $ cat new_exec_proj/new_exec_proj.opam | sed 's/"dune"/$dune/' | sed -E 's/3.[0-9]+/3.XX/g'
-  # This file is generated by dune, edit dune-project instead
-  opam-version: "2.0"
-  synopsis: "A short synopsis"
-  description: "A longer description"
-  maintainer: ["Maintainer Name <maintainer@example.com>"]
-  authors: ["Author Name <author@example.com>"]
-  license: "LICENSE"
-  tags: ["add topics" "to describe" "your" "project"]
-  homepage: "https://github.com/username/reponame"
-  doc: "https://url/to/documentation"
-  bug-reports: "https://github.com/username/reponame/issues"
-  depends: [
-    $dune {>= "3.XX"}
-    "ocaml"
-    "odoc" {with-doc}
-  ]
-  build: [
-    [$dune "subst"] {dev}
-    [
-      $dune
-      "build"
-      "-p"
-      name
-      "-j"
-      jobs
-      "@install"
-      "@runtest" {with-test}
-      "@doc" {with-doc}
-    ]
-  ]
-  dev-repo: "git+https://github.com/username/reponame.git"
-  x-maintenance-intent: ["(latest)"]
 
 We can build and run the resulting executable:
 
@@ -486,49 +454,91 @@ We can build and install the project:
 
   $ dune build --root new_lib_proj @install
   Entering directory 'new_lib_proj'
+  File "new_lib_proj.opam", line 1, characters 0-0:
+  --- new_lib_proj.opam
+  +++ new_lib_proj.opam.generated
+  @@ -0,0 +1,32 @@
+  +# This file is generated by dune, edit dune-project instead
+  +opam-version: "2.0"
+  +synopsis: "A short synopsis"
+  +description: "A longer description"
+  +maintainer: ["Maintainer Name <maintainer@example.com>"]
+  +authors: ["Author Name <author@example.com>"]
+  +license: "LICENSE"
+  +tags: ["add topics" "to describe" "your" "project"]
+  +homepage: "https://github.com/username/reponame"
+  +doc: "https://url/to/documentation"
+  +bug-reports: "https://github.com/username/reponame/issues"
+  +depends: [
+  +  "dune" {>= "3.23"}
+  +  "ocaml"
+  +  "odoc" {with-doc}
+  +]
+  +build: [
+  +  ["dune" "subst"] {dev}
+  +  [
+  +    "dune"
+  +    "build"
+  +    "-p"
+  +    name
+  +    "-j"
+  +    jobs
+  +    "@install"
+  +    "@runtest" {with-test}
+  +    "@doc" {with-doc}
+  +  ]
+  +]
+  +dev-repo: "git+https://github.com/username/reponame.git"
+  +x-maintenance-intent: ["(latest)"]
   Leaving directory 'new_lib_proj'
+  [1]
 
 And the opam file will be generated as expected
 
   $ cat new_lib_proj/new_lib_proj.opam | sed -E 's/3.[0-9]+/3.XX/g'
-  # This file is generated by dune, edit dune-project instead
-  opam-version: "2.0"
-  synopsis: "A short synopsis"
-  description: "A longer description"
-  maintainer: ["Maintainer Name <maintainer@example.com>"]
-  authors: ["Author Name <author@example.com>"]
-  license: "LICENSE"
-  tags: ["add topics" "to describe" "your" "project"]
-  homepage: "https://github.com/username/reponame"
-  doc: "https://url/to/documentation"
-  bug-reports: "https://github.com/username/reponame/issues"
-  depends: [
-    "dune" {>= "3.XX"}
-    "ocaml"
-    "odoc" {with-doc}
-  ]
-  build: [
-    ["dune" "subst"] {dev}
-    [
-      "dune"
-      "build"
-      "-p"
-      name
-      "-j"
-      jobs
-      "@install"
-      "@runtest" {with-test}
-      "@doc" {with-doc}
-    ]
-  ]
-  dev-repo: "git+https://github.com/username/reponame.git"
-  x-maintenance-intent: ["(latest)"]
 
 And we we can run the tests:
 
   $ dune runtest --root new_lib_proj
   Entering directory 'new_lib_proj'
+  File "new_lib_proj.opam", line 1, characters 0-0:
+  --- new_lib_proj.opam
+  +++ new_lib_proj.opam.generated
+  @@ -0,0 +1,32 @@
+  +# This file is generated by dune, edit dune-project instead
+  +opam-version: "2.0"
+  +synopsis: "A short synopsis"
+  +description: "A longer description"
+  +maintainer: ["Maintainer Name <maintainer@example.com>"]
+  +authors: ["Author Name <author@example.com>"]
+  +license: "LICENSE"
+  +tags: ["add topics" "to describe" "your" "project"]
+  +homepage: "https://github.com/username/reponame"
+  +doc: "https://url/to/documentation"
+  +bug-reports: "https://github.com/username/reponame/issues"
+  +depends: [
+  +  "dune" {>= "3.23"}
+  +  "ocaml"
+  +  "odoc" {with-doc}
+  +]
+  +build: [
+  +  ["dune" "subst"] {dev}
+  +  [
+  +    "dune"
+  +    "build"
+  +    "-p"
+  +    name
+  +    "-j"
+  +    jobs
+  +    "@install"
+  +    "@runtest" {with-test}
+  +    "@doc" {with-doc}
+  +  ]
+  +]
+  +dev-repo: "git+https://github.com/username/reponame.git"
+  +x-maintenance-intent: ["(latest)"]
   Leaving directory 'new_lib_proj'
+  [1]
 
 
 Initializing Projects Using the PATH Argument

test/blackbox-tests/test-cases/dune-project-meta/fresh-build.t

@@ -22,3 +22,19 @@ source tree if (generate_opam_files true) is enabled.
   $ dune build foo.install
   $ grep opam _build/default/foo.install
     "_build/install/default/lib/foo/opam"
+
+The same should hold once opam generation switches to diff actions.
+
+  $ rm -f foo.opam
+  $ cat >dune-project <<EOF
+  > (lang dune 3.23)
+  > (package
+  >  (name foo))
+  > (generate_opam_files true)
+  > EOF
+
+  $ dune build foo.install
+  $ grep opam _build/default/foo.install
+    "_build/install/default/lib/foo/opam"
+  $ if test -e foo.opam; then echo present; else echo absent; fi
+  absent

test/blackbox-tests/test-cases/package-runtest-alias.t

@@ -9,6 +9,14 @@ a package:
   >  (dir foo))
   > EOF
 
-  $ dune build @check
+  $ dune build @check 2>&1 | sed -n '1,3p'
+  File "foo.opam", line 1, characters 0-0:
+  --- foo.opam
+  +++ foo.opam.generated
+  [1]
+  $ dune promotion list
+  foo.opam
+  $ dune promote
+  Promoting _build/default/foo.opam.generated to foo.opam.
   $ cat foo.opam | grep -o '"@runtest/foo" {with-test}'
   "@runtest/foo" {with-test}