feat: image build improvements

backend/provisioner/oci_image_provisioner.go

@@ -17,13 +17,13 @@ import (
 	"github.com/block/ftl/internal/oci"
 )
 
-func NewOCIImageProvisioner(storage *oci.ImageService, defaultImage string) *InMemProvisioner {
+func NewOCIImageProvisioner(storage *oci.ImageService, astorage *oci.ArtefactService, defaultImage string) *InMemProvisioner {
 	return NewEmbeddedProvisioner(map[schema.ResourceType]InMemResourceProvisionerFn{
-		schema.ResourceTypeImage: provisionOCIImage(storage, defaultImage),
+		schema.ResourceTypeImage: provisionOCIImage(storage, astorage, defaultImage),
 	}, map[schema.ResourceType]InMemResourceProvisionerFn{})
 }
 
-func provisionOCIImage(storage *oci.ImageService, defaultImage string) InMemResourceProvisionerFn {
+func provisionOCIImage(storage *oci.ImageService, astorage *oci.ArtefactService, defaultImage string) InMemResourceProvisionerFn {
 	return func(ctx context.Context, changeset key.Changeset, deployment key.Deployment, rc schema.Provisioned, moduleSch *schema.Module) (*schema.RuntimeElement, error) {
 		logger := log.FromContext(ctx)
 		variants := goslices.Collect(slices.FilterVariants[*schema.MetadataArtefact](moduleSch.Metadata))
@@ -55,15 +55,15 @@ func provisionOCIImage(storage *oci.ImageService, defaultImage string) InMemReso
 			tag = git.Commit
 		}
 
-		target := string(storage.Image(deployment.Payload.Realm, deployment.Payload.Module, tag))
-		err = storage.BuildOCIImageFromRemote(ctx, image, target, tempDir, moduleSch, deployment, variants, oci.WithRemotePush())
+		target := storage.Image(deployment.Payload.Realm, deployment.Payload.Module, tag)
+		err = storage.BuildOCIImageFromRemote(ctx, astorage, image, target, tempDir, moduleSch, deployment, variants, oci.WithRemotePush())
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to build image")
 		}
 		return &schema.RuntimeElement{
 			Deployment: deployment,
 			Element: &schema.ModuleRuntimeImage{
-				Image: target,
+				Image: string(target),
 			},
 		}, nil
 	}

backend/provisioner/registry.go

@@ -70,15 +70,15 @@ func (reg *ProvisionerRegistry) listBindings() []*ProvisionerBinding {
 
 type pluginProcesses map[string]Plugin
 
-func registryFromConfig(ctx context.Context, workingDir string, cfg *provisionerPluginConfig, runnerScaling scaling.RunnerScaling, adminClient adminpbconnect.AdminServiceClient, imageService *oci.ImageService) (*ProvisionerRegistry, error) {
+func registryFromConfig(ctx context.Context, workingDir string, cfg *provisionerPluginConfig, runnerScaling scaling.RunnerScaling, adminClient adminpbconnect.AdminServiceClient, imageService *oci.ImageService, artifactService *oci.ArtefactService) (*ProvisionerRegistry, error) {
 	logger := log.FromContext(ctx)
 	result := &ProvisionerRegistry{}
 	if err := cfg.Validate(); err != nil {
 		return nil, errors.Wrap(err, "error validating provisioner config")
 	}
 	processes := pluginProcesses{}
 	for _, plugin := range cfg.Plugins {
-		provisioner, err := provisionerIDToProvisioner(ctx, plugin.ID, workingDir, runnerScaling, adminClient, imageService, processes)
+		provisioner, err := provisionerIDToProvisioner(ctx, plugin.ID, workingDir, runnerScaling, adminClient, imageService, artifactService, processes)
 		if err != nil {
 			return nil, errors.WithStack(err)
 		}
@@ -95,6 +95,7 @@ func provisionerIDToProvisioner(
 	scaling scaling.RunnerScaling,
 	adminClient adminpbconnect.AdminServiceClient,
 	imageService *oci.ImageService,
+	artifactService *oci.ArtefactService,
 	processes pluginProcesses,
 ) (Plugin, error) {
 	switch id {
@@ -107,7 +108,7 @@ func provisionerIDToProvisioner(
 	case "noop":
 		return NewPluginClient(&NoopProvisioner{}), nil
 	case "oci-image":
-		return NewOCIImageProvisioner(imageService, "ftl0/ftl-runner"), nil
+		return NewOCIImageProvisioner(imageService, artifactService, "ftl0/ftl-runner"), nil
 	default:
 		if _, ok := processes[id]; ok {
 			return processes[id], nil

backend/provisioner/service.go

@@ -198,7 +198,7 @@ func Start(
 	return nil
 }
 
-func RegistryFromConfigFile(ctx context.Context, workingDir string, file *os.File, scaling scaling.RunnerScaling, adminClient adminpbconnect.AdminServiceClient, imageService *oci.ImageService) (*ProvisionerRegistry, error) {
+func RegistryFromConfigFile(ctx context.Context, workingDir string, file *os.File, scaling scaling.RunnerScaling, adminClient adminpbconnect.AdminServiceClient, imageService *oci.ImageService, artifactService *oci.ArtefactService) (*ProvisionerRegistry, error) {
 	config := provisionerPluginConfig{}
 	bytes, err := io.ReadAll(bufio.NewReader(file))
 	if err != nil {
@@ -208,7 +208,7 @@ func RegistryFromConfigFile(ctx context.Context, workingDir string, file *os.Fil
 		return nil, errors.Wrap(err, "error parsing plugin configuration")
 	}
 
-	registry, err := registryFromConfig(ctx, workingDir, &config, scaling, adminClient, imageService)
+	registry, err := registryFromConfig(ctx, workingDir, &config, scaling, adminClient, imageService, artifactService)
 	if err != nil {
 		return nil, errors.Wrap(err, "error creating provisioner registry")
 	}

cmd/ftl-provisioner/main.go

@@ -59,13 +59,13 @@ func main() {
 	artefactService, err := oci.NewArtefactService(ctx, cli.ArtefactConfig)
 	kctx.FatalIfErrorf(err, "failed to create OCI registry storage")
 
-	imageService, err := oci.NewImageService(ctx, artefactService, &cli.ImageConfig)
+	imageService, err := oci.NewImageService(ctx, &cli.ImageConfig)
 	kctx.FatalIfErrorf(err, "failed to create image service")
 
 	scaling := k8sscaling.NewK8sScaling(false, cli.Realm, mapper, cli.KubeConfig.RouteTemplate(), cli.CronServiceAccount, cli.AdminServiceAccount, cli.ConsoleServiceAccount, cli.HTTPServiceAccount)
 	err = scaling.Start(ctx)
 	kctx.FatalIfErrorf(err, "error starting k8s scaling")
-	registry, err := provisioner.RegistryFromConfigFile(ctx, cli.ProvisionerConfig.WorkingDir, cli.ProvisionerConfig.PluginConfigFile, scaling, adminClient, imageService)
+	registry, err := provisioner.RegistryFromConfigFile(ctx, cli.ProvisionerConfig.WorkingDir, cli.ProvisionerConfig.PluginConfigFile, scaling, adminClient, imageService, artefactService)
 	kctx.FatalIfErrorf(err, "failed to create provisioner registry")
 
 	// Use in mem sql-migration provisioner as fallback for sql-migration provisioning if no other provisioner is registered
@@ -82,7 +82,7 @@ func main() {
 	if _, ok := slices.Find(registry.Bindings, func(binding *provisioner.ProvisionerBinding) bool {
 		return slices.Contains(binding.Types, schema.ResourceTypeImage)
 	}); !ok {
-		ociProvisioner := provisioner.NewOCIImageProvisioner(imageService, cli.DefaultRunnerImage)
+		ociProvisioner := provisioner.NewOCIImageProvisioner(imageService, artefactService, cli.DefaultRunnerImage)
 		runnerBinding := registry.Register("oci-image", ociProvisioner, schema.ResourceTypeImage)
 		logger.Debugf("Registered provisioner %s as fallback for image", runnerBinding)
 	}

cmd/ftl/app.go

@@ -45,28 +45,28 @@ type SharedCLI struct {
 	AdminEndpoint  *url.URL              `help:"Admin endpoint." env:"FTL_ENDPOINT" default:"http://127.0.0.1:8892"`
 	Trace          string                `help:"File to write golang runtime/trace output to." hidden:""`
 
-	Ping       pingCmd       `cmd:"" help:"Ping the FTL cluster."`
-	Init       initCmd       `cmd:"" help:"Initialize a new FTL project."`
-	Profile    profileCmd    `cmd:"" help:"Manage profiles."`
-	Module     moduleCmd     `cmd:"" help:"Manage modules."`
-	PS         psCmd         `cmd:"" help:"List deployments."`
-	Call       callCmd       `cmd:"" help:"Call an FTL verb."`
-	Changeset  changesetCmd  `cmd:"" help:"Work with changesets."`
-	Bench      benchCmd      `cmd:"" help:"Benchmark an FTL verb."`
-	Replay     replayCmd     `cmd:"" help:"Call an FTL verb with the same request body as the last invocation."`
-	Update     updateCmd     `cmd:"" help:"Update a deployment."`
-	Kill       killCmd       `cmd:"" help:"Kill a deployment."`
-	Schema     schemaCmd     `cmd:"" help:"FTL schema commands."`
-	Download   downloadCmd   `cmd:"" help:"Download a deployment."`
-	Secret     secretCmd     `cmd:"" help:"Manage secrets."`
-	Config     configCmd     `cmd:"" help:"Manage configuration."`
-	Pubsub     pubsubCmd     `cmd:"" help:"Manage pub/sub."`
-	Goose      gooseCmd      `cmd:"" help:"Run a goose command."`
-	Mysql      mySQLCmd      `cmd:"" help:"Manage MySQL databases."`
-	Postgres   postgresCmd   `cmd:"" help:"Manage PostgreSQL databases."`
-	Edit       editCmd       `cmd:"" help:"Edit a declaration in an IDE."`
-	Realm      realmCmd      `cmd:"" help:"Manage realms." hidden:""`
-	BuildImage buildImageCmd `cmd:"" help:"Build (and optionally push) a FTL OCI image"`
+	Ping      pingCmd      `cmd:"" help:"Ping the FTL cluster."`
+	Init      initCmd      `cmd:"" help:"Initialize a new FTL project."`
+	Profile   profileCmd   `cmd:"" help:"Manage profiles."`
+	Module    moduleCmd    `cmd:"" help:"Manage modules."`
+	PS        psCmd        `cmd:"" help:"List deployments."`
+	Call      callCmd      `cmd:"" help:"Call an FTL verb."`
+	Changeset changesetCmd `cmd:"" help:"Work with changesets."`
+	Bench     benchCmd     `cmd:"" help:"Benchmark an FTL verb."`
+	Replay    replayCmd    `cmd:"" help:"Call an FTL verb with the same request body as the last invocation."`
+	Update    updateCmd    `cmd:"" help:"Update a deployment."`
+	Kill      killCmd      `cmd:"" help:"Kill a deployment."`
+	Schema    schemaCmd    `cmd:"" help:"FTL schema commands."`
+	Download  downloadCmd  `cmd:"" help:"Download a deployment."`
+	Secret    secretCmd    `cmd:"" help:"Manage secrets."`
+	Config    configCmd    `cmd:"" help:"Manage configuration."`
+	Pubsub    pubsubCmd    `cmd:"" help:"Manage pub/sub."`
+	Goose     gooseCmd     `cmd:"" help:"Run a goose command."`
+	Mysql     mySQLCmd     `cmd:"" help:"Manage MySQL databases."`
+	Postgres  postgresCmd  `cmd:"" help:"Manage PostgreSQL databases."`
+	Edit      editCmd      `cmd:"" help:"Edit a declaration in an IDE."`
+	Realm     realmCmd     `cmd:"" help:"Manage realms." hidden:""`
+	Image     imageCmd     `cmd:"" help:"Image related commands."`
 }
 
 type BuildAndDeploy struct {

cmd/ftl/cmd_image.go

@@ -0,0 +1,6 @@
+package main
+
+type imageCmd struct {
+	Build   imageBuildCmd   `cmd:"" help:"Build an image from the modules in the project."`
+	Inspect imageInspectCmd `cmd:"" help:"Inspect the schema of an image."`
+}

cmd/ftl/cmd_buildimage.go → cmd/ftl/cmd_image_build.go

@@ -18,19 +18,19 @@ import (
 	"github.com/block/ftl/internal/schema/schemaeventsource"
 )
 
-type buildImageCmd struct {
-	Parallelism     int                `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"`
-	Dirs            []string           `arg:"" help:"Base directories containing modules (defaults to modules in project config)." type:"existingdir" optional:""`
-	BuildEnv        []string           `help:"Environment variables to set for the build."`
-	ArtefactConfig  oci.ArtefactConfig `embed:""`
-	ImageConfig     oci.ImageConfig    `embed:""`
-	Tag             string             `help:"The image tag" default:"latest"`
-	RunnerImage     string             `help:"An override of the runner base image"`
-	Push            bool               `help:"Push the image to the registry after building." default:"false"`
-	SkipLocalDaemon bool               `help:"Skip pushing to the local docker daemon." default:"false"`
+type imageBuildCmd struct {
+	Parallelism     int             `short:"j" help:"Number of modules to build in parallel." default:"${numcpu}"`
+	Dirs            []string        `arg:"" help:"Base directories containing modules (defaults to modules in project config)." type:"existingdir" optional:""`
+	BuildEnv        []string        `help:"Environment variables to set for the build."`
+	ImageConfig     oci.ImageConfig `embed:""`
+	Tag             string          `help:"The image tag" default:"latest"`
+	RunnerImage     string          `help:"An override of the runner base image"`
+	Push            bool            `help:"Push the image to the registry after building." default:"false"`
+	SkipLocalDaemon bool            `help:"Skip pushing to the local docker daemon." default:"false"`
+	TarFile         string          `help:"File system path to push the image to"`
 }
 
-func (b *buildImageCmd) Run(
+func (b *imageBuildCmd) Run(
 	ctx context.Context,
 	adminClient adminpbconnect.AdminServiceClient,
 	schemaSource *schemaeventsource.EventSource,
@@ -64,11 +64,7 @@ func (b *buildImageCmd) Run(
 		logger.Warnf("No modules were found to build")
 		return nil
 	}
-	artefactService, err := oci.NewArtefactService(ctx, b.ArtefactConfig)
-	if err != nil {
-		return errors.Wrapf(err, "failed to init artefact service")
-	}
-	imageService, err := oci.NewImageService(ctx, artefactService, &b.ImageConfig)
+	imageService, err := oci.NewImageService(ctx, &b.ImageConfig)
 	if err != nil {
 		return errors.Wrapf(err, "failed to init OCI")
 	}
@@ -104,16 +100,18 @@ func (b *buildImageCmd) Run(
 				image += "latest"
 			}
 		}
-		tgt := string(b.ArtefactConfig.Repository)
-		tgt += ":"
-		tgt += b.Tag
+		tgt := imageService.Image(projConfig.Name, moduleSch.Name, b.Tag)
+		moduleSch.Metadata = append(moduleSch.Metadata, &schema.MetadataImage{Image: string(tgt)})
 		targets := []oci.ImageTarget{}
 		if !b.SkipLocalDaemon {
 			targets = append(targets, oci.WithLocalDeamon())
 		}
 		if b.Push {
 			targets = append(targets, oci.WithRemotePush())
 		}
+		if b.TarFile != "" {
+			targets = append(targets, oci.WithDiskImage(b.TarFile))
+		}
 		// TODO: we need to properly sync the deployment with the actual deployment key
 		// this is just a hack to get the module and realm to the runner
 		deployment := key.NewDeploymentKey(projConfig.Name, moduleSch.Name)

cmd/ftl/cmd_image_inspect.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"context"
+
+	"github.com/alecthomas/errors"
+
+	"github.com/block/ftl/common/log"
+	"github.com/block/ftl/internal/oci"
+)
+
+type imageInspectCmd struct {
+	ImageConfig oci.ImageConfig `embed:""`
+	Image       string          `arg:"" help:"The image to inspect" default:"latest"`
+}
+
+func (b *imageInspectCmd) Run(
+	ctx context.Context,
+) error {
+	logger := log.FromContext(ctx)
+
+	imageService, err := oci.NewImageService(ctx, &b.ImageConfig)
+	if err != nil {
+		return errors.Wrapf(err, "failed to init OCI")
+	}
+
+	sch, err := imageService.PullSchema(ctx, b.Image)
+	if err != nil {
+		return errors.Wrapf(err, "failed to pull image schema %s", b.Image)
+	}
+	logger.Infof("%s", sch.String()) //nolint
+	return nil
+}

cmd/ftl/cmd_serve.go

@@ -272,14 +272,14 @@ func (s *serveCommonConfig) run(
 		},
 	}
 
-	imageService, err := oci.NewImageService(ctx, artefactService, &s.ImageConfig)
+	imageService, err := oci.NewImageService(ctx, &s.ImageConfig)
 	if err != nil {
 		return errors.Wrap(err, "failed to create image service")
 	}
 
 	// read provisioners from a config file if provided
 	if s.PluginConfigFile != nil {
-		r, err := provisioner.RegistryFromConfigFile(provisionerCtx, s.WorkingDir, s.PluginConfigFile, runnerScaling, adminClient, imageService)
+		r, err := provisioner.RegistryFromConfigFile(provisionerCtx, s.WorkingDir, s.PluginConfigFile, runnerScaling, adminClient, imageService, artefactService)
 		if err != nil {
 			return errors.Wrap(err, "failed to create provisioner registry")
 		}