From bdeaf1399a7cc5871369e4d1b1059879aa387418 Mon Sep 17 00:00:00 2001 From: Christoph Langguth Date: Wed, 13 May 2026 09:35:16 +0200 Subject: [PATCH 01/10] SED-4692 Refactor YAML patching logic --- .editorconfig | 2 +- .../src/main/java/step/ap_ide/StepUp.java | 2 + .../src/test/resources/step.properties | 3 +- .../resources/expected/Hello World Plan.yml | 12 +- .../resources/expected/descriptorAfterAdd.yml | 12 +- .../keywordsAfterCompositeModification.yml | 14 +- .../test/resources/expected/plan1AfterAdd.yml | 54 ++-- .../expected/plan1AfterModification.yml | 46 +-- .../expected/plan1AfterModifyAndAdd.yml | 58 ++-- .../resources/expected/plan1AfterRemove.yml | 11 +- .../resources/expected/plan1AfterRename.yml | 44 +-- .../packages/AutomationPackageReader.java | 2 +- .../AutomationPackageDescriptorReader.java | 23 +- .../AutomationPackageYamlFragmentManager.java | 27 +- ...AutomationPackageFragmentDeserializer.java | 8 +- ...tomationPackageDescriptorDeserializer.java | 3 +- ...AutomationPackageFragmentDeserializer.java | 12 +- .../YamlKeywordDeserializer.java | 16 +- ...AbstractAutomationPackageFragmentYaml.java | 25 +- .../AutomationPackageDescriptorYamlImpl.java | 5 +- .../model/AutomationPackageFragmentYaml.java | 3 +- .../AutomationPackageFragmentYamlImpl.java | 5 +- .../step/core/yaml/PatchingContextTest.java | 234 +++++++++++++++ .../yaml/descriptors/completeDescriptor2.yml | 74 +++++ .../step/core/yaml/PatchableYamlModel.java | 20 +- .../core/yaml/PatchableYamlModelBase.java | 66 ++-- .../java/step/core/yaml/PatchingContext.java | 281 ++++++++++++++++++ .../deserialization/PatchableYamlList.java | 137 ++++----- .../PatchableYamlListDeserializer.java | 3 +- .../PatchableYamlModelDeserializer.java | 7 +- .../yaml/deserialization/PatchingContext.java | 166 ----------- .../PatchingParserDelegate.java | 3 +- .../AutomationPackageParameter.java | 7 +- .../model/YamlAutomationPackageKeyword.java | 4 +- .../plans/parser/yaml/VersionedYamlPlan.java | 2 +- .../java/step/plans/parser/yaml/YamlPlan.java | 8 +- .../automation/AutomationPackageSchedule.java | 7 +- .../plans/automation/YamlPlainTextPlan.java | 4 +- .../plans/parser/yaml/YamlPlanReader.java | 17 +- 39 files changed, 910 insertions(+), 517 deletions(-) create mode 100644 step-automation-packages/step-automation-packages-yaml/src/test/java/step/core/yaml/PatchingContextTest.java create mode 100644 step-automation-packages/step-automation-packages-yaml/src/test/resources/step/automation/packages/yaml/descriptors/completeDescriptor2.yml create mode 100644 step-core-model/src/main/java/step/core/yaml/PatchingContext.java delete mode 100644 step-core-model/src/main/java/step/core/yaml/deserialization/PatchingContext.java diff --git a/.editorconfig b/.editorconfig index 9293c5b4f5..781452eac0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -36,7 +36,7 @@ indent_size = 2 [*.{yml,yaml}] indent_style = space indent_size = 2 - +ij_yaml_spaces_within_brackets = false ######################################## # Markdown diff --git a/step-ap-ide/src/main/java/step/ap_ide/StepUp.java b/step-ap-ide/src/main/java/step/ap_ide/StepUp.java index 8ec7ca6d76..680405e907 100644 --- a/step-ap-ide/src/main/java/step/ap_ide/StepUp.java +++ b/step-ap-ide/src/main/java/step/ap_ide/StepUp.java @@ -27,6 +27,8 @@ public class StepUp { private static final String initialDirName = "src/main/resources/work-initial"; public static void main(String[] args) throws Exception { + // Use an IntelliJ run configuration that uses '%MODULE_WORKING_DIR%' (verbatim) as the working directory, and use + // -config="src/test/resources/step.properties" as program argument ControllerServer.main(args); initWorkdir(); App.main(args); // this will never return diff --git a/step-ap-ide/src/test/resources/step.properties b/step-ap-ide/src/test/resources/step.properties index a0f0f27121..9681de1e36 100644 --- a/step-ap-ide/src/test/resources/step.properties +++ b/step-ap-ide/src/test/resources/step.properties @@ -23,13 +23,14 @@ plugins.BookmarkPlugin.enabled=true plugins.ParameterManagerControllerPlugin.enabled=true plugins.ExecutionPlugin.enabled=true plugins.ExecutionTypeControllerPlugin.enabled=true +#Required functionality +plugins.CompositeFunctionTypeControllerPlugin.enabled=true #Disable all other plugins for now plugins.ArtifactRepositoryPlugin.enabled=false plugins.AutomationPackageParameterPlugin.enabled=false plugins.AutomationPackageRepositoriesPlugin.enabled=false plugins.AutomationPackageSchedulerPlugin.enabled=false plugins.BaseExecutionTypeControllerPlugin.enabled=false -plugins.CompositeFunctionTypeControllerPlugin.enabled=false plugins.DataPoolPlugin.enabled=false plugins.EncryptionManagerDependencyPlugin.enabled=false plugins.EntityLockingPlugin.enabled=false diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/Hello World Plan.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/Hello World Plan.yml index c1c6fe467e..d7d707f7eb 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/Hello World Plan.yml +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/Hello World Plan.yml @@ -1,8 +1,8 @@ --- plans: -- name: "Hello World Plan" - root: - sequence: - children: - - echo: - text: "Hello World" + - name: "Hello World Plan" + root: + sequence: + children: + - echo: + text: "Hello World" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterAdd.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterAdd.yml index 9db0b00a0e..395e245be3 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterAdd.yml +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterAdd.yml @@ -13,12 +13,12 @@ alertingRules: BindingValueEqualsPredicate: value: "myValue" plans: -- name: "New Name" - root: - sequence: - children: - - echo: - text: "Hello World" + - name: "New Name" + root: + sequence: + children: + - echo: + text: "Hello World" parameters: - key: "paramInMainAP" value: "once" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/keywordsAfterCompositeModification.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/keywordsAfterCompositeModification.yml index 4fadfecdad..475e4c8b80 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/keywordsAfterCompositeModification.yml +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/keywordsAfterCompositeModification.yml @@ -12,13 +12,13 @@ keywords: root: testCase: children: - - echo: - text: "Modified Echo" - - return: - output: - - output1: "value" - - output2: - expression: "'some thing dynamic'" + - echo: + text: "Modified Echo" + - return: + output: + - output1: "value" + - output2: + expression: "'some thing dynamic'" - GeneralScript: name: "GeneralScript keyword from AP" scriptLanguage: javascript diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterAdd.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterAdd.yml index c6cb8b9ebc..6dca1e6e92 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterAdd.yml +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterAdd.yml @@ -2,31 +2,31 @@ fragments: [] keywords: [] plans: -- name: "Test Plan" - root: - testCase: - children: - - echo: - text: "Just echo" - - echo: - text: - expression: "mySimpleKey" - - callKeyword: - nodeName: "CallMyKeyword2" - inputs: - - myInput: "myValue" - keyword: "MyKeyword2" - categories: - - "Yaml Plan" -- name: "New Name" - root: - sequence: - children: - - echo: - text: "Hello World" + - name: "Test Plan" + root: + testCase: + children: + - echo: + text: "Just echo" + - echo: + text: + expression: "mySimpleKey" + - callKeyword: + nodeName: "CallMyKeyword2" + inputs: + - myInput: "myValue" + keyword: "MyKeyword2" + categories: + - "Yaml Plan" + - name: "New Name" + root: + sequence: + children: + - echo: + text: "Hello World" plansPlainText: -- name: "Plain text plan" - rootType: "Sequence" - categories: - - "PlainTextPlan" - file: "plans/plan2.plan" + - name: "Plain text plan" + rootType: "Sequence" + categories: + - "PlainTextPlan" + file: "plans/plan2.plan" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterModification.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterModification.yml index 81cc315f24..9a4a97b961 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterModification.yml +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterModification.yml @@ -2,27 +2,27 @@ fragments: [] keywords: [] plans: -- name: "Test Plan" - root: - testCase: - nodeName: "Test Plan" - children: - - echo: - text: - expression: "new Date().toString();" - - echo: - text: - expression: "mySimpleKey" - - callKeyword: - nodeName: "CallMyKeyword2" - inputs: - - myInput: "myValue" - keyword: "MyKeyword2" - categories: - - "Yaml Plan" + - name: "Test Plan" + root: + testCase: + nodeName: "Test Plan" + children: + - echo: + text: + expression: "new Date().toString();" + - echo: + text: + expression: "mySimpleKey" + - callKeyword: + nodeName: "CallMyKeyword2" + inputs: + - myInput: "myValue" + keyword: "MyKeyword2" + categories: + - "Yaml Plan" plansPlainText: -- name: "Plain text plan" - rootType: "Sequence" - categories: - - "PlainTextPlan" - file: "plans/plan2.plan" + - name: "Plain text plan" + rootType: "Sequence" + categories: + - "PlainTextPlan" + file: "plans/plan2.plan" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterModifyAndAdd.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterModifyAndAdd.yml index 237084b8b7..d6ee24d376 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterModifyAndAdd.yml +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterModifyAndAdd.yml @@ -2,33 +2,33 @@ fragments: [] keywords: [] plans: -- name: "Test Plan" - root: - testCase: - nodeName: "Test Plan" - children: - - echo: - text: - expression: "new Date().toString();" - - echo: - text: - expression: "mySimpleKey" - - callKeyword: - nodeName: "CallMyKeyword2" - inputs: - - myInput: "myValue" - keyword: "MyKeyword2" - categories: - - "Yaml Plan" -- name: "New Name" - root: - sequence: - children: - - echo: - text: "Hello World" + - name: "Test Plan" + root: + testCase: + nodeName: "Test Plan" + children: + - echo: + text: + expression: "new Date().toString();" + - echo: + text: + expression: "mySimpleKey" + - callKeyword: + nodeName: "CallMyKeyword2" + inputs: + - myInput: "myValue" + keyword: "MyKeyword2" + categories: + - "Yaml Plan" + - name: "New Name" + root: + sequence: + children: + - echo: + text: "Hello World" plansPlainText: -- name: "Plain text plan" - rootType: "Sequence" - categories: - - "PlainTextPlan" - file: "plans/plan2.plan" + - name: "Plain text plan" + rootType: "Sequence" + categories: + - "PlainTextPlan" + file: "plans/plan2.plan" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterRemove.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterRemove.yml index f01060a03f..cbb10acc18 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterRemove.yml +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterRemove.yml @@ -1,9 +1,10 @@ --- fragments: [] keywords: [] +plans: [] plansPlainText: -- name: "Plain text plan" - rootType: "Sequence" - categories: - - "PlainTextPlan" - file: "plans/plan2.plan" + - name: "Plain text plan" + rootType: "Sequence" + categories: + - "PlainTextPlan" + file: "plans/plan2.plan" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterRename.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterRename.yml index bc4856bdc6..91fc8259f6 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterRename.yml +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/plan1AfterRename.yml @@ -2,26 +2,26 @@ fragments: [] keywords: [] plans: -- name: "New Plan Name" - root: - testCase: - nodeName: "Test Plan" - children: - - echo: - text: "Just echo" - - echo: - text: - expression: "mySimpleKey" - - callKeyword: - nodeName: "CallMyKeyword2" - inputs: - - myInput: "myValue" - keyword: "MyKeyword2" - categories: - - "Yaml Plan" + - name: "New Plan Name" + root: + testCase: + nodeName: "Test Plan" + children: + - echo: + text: "Just echo" + - echo: + text: + expression: "mySimpleKey" + - callKeyword: + nodeName: "CallMyKeyword2" + inputs: + - myInput: "myValue" + keyword: "MyKeyword2" + categories: + - "Yaml Plan" plansPlainText: -- name: "Plain text plan" - rootType: "Sequence" - categories: - - "PlainTextPlan" - file: "plans/plan2.plan" + - name: "Plain text plan" + rootType: "Sequence" + categories: + - "PlainTextPlan" + file: "plans/plan2.plan" diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java index 73171c5cd5..4fead6c631 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java @@ -205,7 +205,7 @@ private void fillAutomationPackageWithImportedFragments(AutomationPackageContent List resources = archive.getResourcesByPattern(importedFragmentReference); for (URL resource : resources) { try (InputStream fragmentYamlStream = resource.openStream()) { - fragment = getOrCreateDescriptorReader().readAutomationPackageFragment(fragmentYamlStream, importedFragmentReference, archive.getAutomationPackageName()); + fragment = getOrCreateDescriptorReader().readAutomationPackageFragment(fragmentYamlStream, resource.toString(), archive.getAutomationPackageName()); fragmentYamlMap.put(resource.toString(), fragment); fragment.setFragmentUrl(resource); fillAutomationPackageWithImportedFragments(targetPackage, fragment, archive, fragmentYamlMap); diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageDescriptorReader.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageDescriptorReader.java index dac6eabcab..4e139641cc 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageDescriptorReader.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageDescriptorReader.java @@ -18,11 +18,9 @@ ******************************************************************************/ package step.automation.packages.yaml; -import com.fasterxml.jackson.databind.*; -import com.fasterxml.jackson.databind.deser.BeanDeserializerModifier; -import com.fasterxml.jackson.databind.deser.std.CollectionDeserializer; +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; -import com.fasterxml.jackson.databind.type.CollectionType; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import org.apache.commons.lang3.StringUtils; @@ -37,11 +35,10 @@ import step.automation.packages.yaml.model.AutomationPackageFragmentYaml; import step.automation.packages.yaml.model.AutomationPackageFragmentYamlImpl; import step.core.accessors.DefaultJacksonMapperProvider; -import step.core.yaml.PatchableYamlModel; -import step.core.yaml.deserialization.*; -import step.plans.parser.yaml.YamlPlan; +import step.core.yaml.PatchingContext; +import step.core.yaml.deserialization.PatchableYamlList; +import step.core.yaml.deserialization.PatchingParserDelegate; import step.plans.parser.yaml.YamlPlanReader; -import step.plans.parser.yaml.deserializers.UpgradableYamlPlanDeserializer; import step.plans.parser.yaml.model.YamlPlanVersions; import step.plans.parser.yaml.schema.YamlPlanValidationException; @@ -49,7 +46,6 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.List; import java.util.Map; public class AutomationPackageDescriptorReader { @@ -78,7 +74,7 @@ public AutomationPackageDescriptorReader(String jsonSchemaPath, AutomationPackag public AutomationPackageDescriptorYaml readAutomationPackageDescriptor(InputStream yamlDescriptor, String packageName) throws AutomationPackageReadingException { log.info("Reading automation package descriptor..."); - return readAutomationPackageYamlFile(yamlDescriptor, getDescriptorClass(), packageName); + return readAutomationPackageYamlFile("automation-package.yml", yamlDescriptor, getDescriptorClass(), packageName); } protected Class getDescriptorClass() { @@ -87,14 +83,14 @@ protected Class getDescriptorClass() public AutomationPackageFragmentYaml readAutomationPackageFragment(InputStream yamlFragment, String fragmentName, String packageName) throws AutomationPackageReadingException { log.info("Reading automation package descriptor fragment ({})...", fragmentName); - return readAutomationPackageYamlFile(yamlFragment, getFragmentClass(), packageName); + return readAutomationPackageYamlFile(fragmentName, yamlFragment, getFragmentClass(), packageName); } protected Class getFragmentClass() { return AutomationPackageFragmentYamlImpl.class; } - protected T readAutomationPackageYamlFile(InputStream yaml, Class targetClass, String packageName) throws AutomationPackageReadingException { + protected T readAutomationPackageYamlFile(String location, InputStream yaml, Class targetClass, String packageName) throws AutomationPackageReadingException { try { String yamlDescriptorString = new String(yaml.readAllBytes(), StandardCharsets.UTF_8); String version = null; @@ -110,7 +106,7 @@ protected T readAutomationPackageYamlF throw new YamlPlanValidationException(message, ex); } } - PatchingContext context = new PatchingContext(yamlDescriptorString, yamlObjectMapper); + PatchingContext context = new PatchingContext(location, yamlDescriptorString, yamlObjectMapper); PatchingParserDelegate parser = new PatchingParserDelegate(yamlObjectMapper.createParser(yamlDescriptorString), context); Map, Object> injections = new HashMap<>(); @@ -170,6 +166,7 @@ private ObjectMapper createYamlObjectMapper() { // Disable native type id to enable conversion to generic Documents yamlFactory.disable(YAMLGenerator.Feature.USE_NATIVE_TYPE_ID); + yamlFactory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR); ObjectMapper yamlMapper = DefaultJacksonMapperProvider.getObjectMapper(yamlFactory); diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java index 5f05a50670..866f8ca711 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java @@ -27,10 +27,10 @@ import step.core.plans.Plan; import step.core.yaml.PatchableYamlModel; import step.core.yaml.PatchableYamlModelBase; +import step.core.yaml.PatchingContext; import step.core.yaml.deserialization.AutomationPackagePerObjectSaveUnsupportedException; import step.core.yaml.deserialization.AutomationPackageUpdateException; import step.core.yaml.deserialization.PatchableYamlList; -import step.core.yaml.deserialization.PatchingContext; import step.functions.Function; import step.parameter.Parameter; import step.parameter.automation.AutomationPackageParameter; @@ -51,8 +51,8 @@ public class AutomationPackageYamlFragmentManager { - private final Path apRoot; - private final StagingAutomationPackageContext stagingContext; + protected final Path apRoot; + protected final StagingAutomationPackageContext stagingContext; public enum NewObjectFragmentMode { /** @@ -67,14 +67,14 @@ public enum NewObjectFragmentMode { public static final String PROPERTY_NEW_OBJECT_FRAGMENT_MODE = "newFragmentPaths.%s.mode"; public static final String PROPERTY_NEW_OBJECT_FRAGMENT_PATH = "newFragmentPaths.%s.path"; - private final AutomationPackageDescriptorReader descriptorReader; + protected final AutomationPackageDescriptorReader descriptorReader; - private final Map patchableMap = new ConcurrentHashMap<>(); - private final Map fragmentMap = new ConcurrentHashMap<>(); - private final Map pathToYamlFragment; + protected final Map patchableMap = new ConcurrentHashMap<>(); + protected final Map fragmentMap = new ConcurrentHashMap<>(); + protected final Map pathToYamlFragment; - private Properties properties = new Properties(); - private final AutomationPackageFragmentYaml descriptorYaml; + protected Properties properties = new Properties(); + protected final AutomationPackageFragmentYaml descriptorYaml; public AutomationPackageYamlFragmentManager(AutomationPackageDescriptorYaml descriptorYaml, Map fragmentMap, AutomationPackageDescriptorReader descriptorReader, StagingAutomationPackageContext stagingContext) { @@ -146,8 +146,8 @@ public synchronized Plan savePlan(Plan plan) { pathToYamlFragment.put(fragment.getFragmentUrl().toString(), fragment); addFragmentEntity(fragment, fragment.getPlans(), newYamlPlan); } else { - YamlPlan yamlPlan = (YamlPlan) patchableMap.get(plan); - modifyFragmentEntity(fragment, fragment.getPlans(), yamlPlan, newYamlPlan); + YamlPlan oldYamlPlan = (YamlPlan) patchableMap.get(plan); + modifyFragmentEntity(fragment, fragment.getPlans(), oldYamlPlan, newYamlPlan); } patchableMap.put(plan, newYamlPlan); @@ -161,14 +161,11 @@ public synchronized step.functions.Function saveFunction(step.functions.Function fragment = fragmentForNewObject(function, YamlPlan.PLANS_ENTITY_NAME); fragmentMap.put(function, fragment); pathToYamlFragment.put(fragment.getFragmentUrl().toString(), fragment); - //addFragmentEntity(fragment, fragment.getKeywords(), newYamlKeyword); } else { YamlAutomationPackageKeyword yamlKeyword = (YamlAutomationPackageKeyword) patchableMap.get(function); yamlKeyword.getYamlKeyword().updateFromFunction(function); modifyFragmentEntity(fragment, fragment.getKeywords(), yamlKeyword, yamlKeyword); } - //patchableMap.put(function, y); - return function; } @@ -217,7 +214,7 @@ private AutomationPackageFragmentYaml fragmentForNewObject(AbstractOrganizableOb if (pathToYamlFragment.containsKey(url.toString())) { return pathToYamlFragment.get(url.toString()); } - PatchingContext context = new PatchingContext("---", descriptorYaml.getPatchingContext().getMapper()); + PatchingContext context = new PatchingContext(url.toString(), "---", descriptorYaml.getPatchingContext().getMapper()); AutomationPackageFragmentYaml fragment = new AutomationPackageFragmentYamlImpl(context); fragment.setFragmentUrl(url); return fragment; diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/AbstractYamlAutomationPackageFragmentDeserializer.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/AbstractYamlAutomationPackageFragmentDeserializer.java index d22526a3f1..815cfaaa2f 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/AbstractYamlAutomationPackageFragmentDeserializer.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/AbstractYamlAutomationPackageFragmentDeserializer.java @@ -19,19 +19,19 @@ package step.automation.packages.yaml.deserialization; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.deser.BeanDeserializer; import com.fasterxml.jackson.databind.deser.ContextualDeserializer; import com.fasterxml.jackson.databind.deser.ResolvableDeserializer; import step.automation.packages.deserialization.AutomationPackageSerializationRegistry; import step.automation.packages.yaml.model.AbstractAutomationPackageFragmentYaml; import step.automation.packages.yaml.model.AutomationPackageFragmentYamlImpl; +import step.core.yaml.PatchingContext; import step.core.yaml.deserialization.PatchableYamlList; -import step.core.yaml.deserialization.PatchingContext; -import step.core.yaml.deserializers.StepYamlDeserializerAddOn; import java.io.IOException; -import java.util.List; public abstract class AbstractYamlAutomationPackageFragmentDeserializer extends BeanDeserializer implements ContextualDeserializer, ResolvableDeserializer { diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlAutomationPackageDescriptorDeserializer.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlAutomationPackageDescriptorDeserializer.java index c010ebdd43..d009a93eb9 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlAutomationPackageDescriptorDeserializer.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlAutomationPackageDescriptorDeserializer.java @@ -25,8 +25,7 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.deser.BeanDeserializer; import step.automation.packages.yaml.model.AutomationPackageDescriptorYamlImpl; -import step.automation.packages.yaml.model.AutomationPackageFragmentYamlImpl; -import step.core.yaml.deserialization.PatchingContext; +import step.core.yaml.PatchingContext; import step.core.yaml.deserializers.StepYamlDeserializerAddOn; import java.io.IOException; diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlAutomationPackageFragmentDeserializer.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlAutomationPackageFragmentDeserializer.java index 857dc8644c..521d7a4137 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlAutomationPackageFragmentDeserializer.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlAutomationPackageFragmentDeserializer.java @@ -19,18 +19,16 @@ package step.automation.packages.yaml.deserialization; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.BeanProperty; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.deser.BeanDeserializer; -import com.fasterxml.jackson.databind.deser.ContextualDeserializer; -import com.fasterxml.jackson.databind.deser.ResolvableDeserializer; -import step.automation.packages.deserialization.AutomationPackageSerializationRegistry; -import step.automation.packages.yaml.model.AbstractAutomationPackageFragmentYaml; import step.automation.packages.yaml.model.AutomationPackageFragmentYamlImpl; -import step.core.yaml.deserialization.PatchingContext; +import step.core.yaml.PatchingContext; import step.core.yaml.deserializers.StepYamlDeserializerAddOn; import java.io.IOException; -import java.util.List; @StepYamlDeserializerAddOn(targetClasses = {AutomationPackageFragmentYamlImpl.class}) public class YamlAutomationPackageFragmentDeserializer extends AbstractYamlAutomationPackageFragmentDeserializer { diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlKeywordDeserializer.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlKeywordDeserializer.java index 686d00d367..9fd9baabbd 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlKeywordDeserializer.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/YamlKeywordDeserializer.java @@ -18,11 +18,12 @@ ******************************************************************************/ package step.automation.packages.yaml.deserialization; -import com.fasterxml.jackson.core.JsonLocation; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import step.automation.packages.model.AbstractYamlFunction; import step.automation.packages.model.YamlAutomationPackageKeyword; import step.core.yaml.AutomationPackageKeywordsLookuper; @@ -34,6 +35,7 @@ @StepYamlDeserializerAddOn(targetClasses = {YamlAutomationPackageKeyword.class}) public class YamlKeywordDeserializer extends StepYamlDeserializer { + private static final Logger logger = LoggerFactory.getLogger(YamlKeywordDeserializer.class); private final AutomationPackageKeywordsLookuper keywordsLookuper = new AutomationPackageKeywordsLookuper(); @@ -48,15 +50,19 @@ public YamlAutomationPackageKeyword deserialize(JsonParser jsonParser, Deseriali try { PatchingParserDelegate patchingParser = (PatchingParserDelegate) jsonParser; Class clazz = Class.forName(keywordsLookuper.yamlKeywordClassToJava(yamlName)); - JsonLocation startItem = patchingParser.currentLocation(); - jsonParser.nextToken(); + swallowToken(jsonParser); + logger.debug("{} About to deserialize YamlAutomationPackageKeyword (yamlName={} -> class={})", this, yamlName, clazz.getName()); YamlAutomationPackageKeyword keyword = new YamlAutomationPackageKeyword((AbstractYamlFunction) deserializationContext.readValue(jsonParser, clazz), patchingParser.getPatchingContext()); - keyword.setPatchingBounds(startItem, patchingParser.getLastDistinctLocation()); - jsonParser.nextToken(); + swallowToken(jsonParser); return keyword; } catch (ClassNotFoundException e) { throw new RuntimeException(e); } } + private void swallowToken(JsonParser jsonParser) throws IOException { + var tk = jsonParser.nextToken(); + logger.debug("{} Swallowed token: {}", this, tk); + } + } diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java index 284bf5155b..c4891c8311 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java @@ -18,16 +18,17 @@ ******************************************************************************/ package step.automation.packages.yaml.model; -import com.fasterxml.jackson.annotation.*; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.Nulls; import org.apache.commons.io.FileUtils; -import step.automation.packages.deserialization.AutomationPackageSerializationRegistry; import step.automation.packages.model.YamlAutomationPackageKeyword; import step.automation.packages.yaml.AutomationPackageWriteToDiskException; +import step.core.yaml.PatchingContext; import step.core.yaml.deserialization.AutomationPackageConcurrentEditException; import step.core.yaml.deserialization.PatchableYamlList; -import step.core.yaml.deserialization.PatchingContext; import step.plans.automation.YamlPlainTextPlan; import step.plans.parser.yaml.YamlPlan; @@ -37,14 +38,17 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.text.MessageFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @JsonInclude(JsonInclude.Include.NON_EMPTY) public abstract class AbstractAutomationPackageFragmentYaml implements AutomationPackageFragmentYaml { private List fragments = new ArrayList<>(); private PatchableYamlList keywords; private PatchableYamlList plans; - private List plansPlainText = new ArrayList<>(); + private PatchableYamlList plansPlainText; private final Map> additionalFields = new HashMap<>(); private PatchingContext context; @@ -54,6 +58,7 @@ public AbstractAutomationPackageFragmentYaml(PatchingContext patchingContext) { context = patchingContext; plans = new PatchableYamlList<>(patchingContext, YamlPlan.PLANS_ENTITY_NAME); keywords = new PatchableYamlList<>(patchingContext, YamlAutomationPackageKeyword.KEYWORDS_ENTITY_NAME); + plansPlainText = new PatchableYamlList<>(patchingContext, "plansPlainText"); } @JsonIgnore @@ -97,7 +102,7 @@ public Map> getAdditionalFields() { @Override public void setAdditionalFields(String key, PatchableYamlList list) { - additionalFields.put(key, list); + additionalFields.put(key, list); } @Override @@ -106,7 +111,7 @@ public List getPlansPlainText() { } @JsonSetter(nulls = Nulls.AS_EMPTY) - public void setPlansPlainText(List plansPlainText) { + public void setPlansPlainText(PatchableYamlList plansPlainText) { this.plansPlainText = plansPlainText; } @@ -145,7 +150,7 @@ public void writeToDisk() { if (file.exists() && file.lastModified() > fileLastModified) { throw new AutomationPackageConcurrentEditException(MessageFormat.format("Automation package fragment {0} was edited outside the editor.", url)); } - FileUtils.writeStringToFile(file, context.getYaml(), StandardCharsets.UTF_8); + FileUtils.writeStringToFile(file, context.getCurrentYaml(), StandardCharsets.UTF_8); resetLastModified(); } catch (IOException | URISyntaxException e) { throw new AutomationPackageWriteToDiskException(MessageFormat.format("Error when writing automation package fragment {0} back to disk.", url), e); diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageDescriptorYamlImpl.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageDescriptorYamlImpl.java index e7be231568..32fbed50a6 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageDescriptorYamlImpl.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageDescriptorYamlImpl.java @@ -19,11 +19,8 @@ package step.automation.packages.yaml.model; import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.OptBoolean; -import com.fasterxml.jackson.databind.ObjectMapper; -import step.automation.packages.deserialization.AutomationPackageSerializationRegistry; -import step.core.yaml.deserialization.PatchingContext; +import step.core.yaml.PatchingContext; import java.util.HashMap; import java.util.Map; diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java index 69f09fe8ee..7cf2be19b3 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java @@ -18,10 +18,9 @@ ******************************************************************************/ package step.automation.packages.yaml.model; -import com.fasterxml.jackson.databind.JsonNode; import step.automation.packages.model.YamlAutomationPackageKeyword; +import step.core.yaml.PatchingContext; import step.core.yaml.deserialization.PatchableYamlList; -import step.core.yaml.deserialization.PatchingContext; import step.plans.automation.YamlPlainTextPlan; import step.plans.parser.yaml.YamlPlan; diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYamlImpl.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYamlImpl.java index 6d4aecc70a..d1c87063fe 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYamlImpl.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYamlImpl.java @@ -19,11 +19,8 @@ package step.automation.packages.yaml.model; import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.OptBoolean; -import com.fasterxml.jackson.databind.ObjectMapper; -import step.automation.packages.deserialization.AutomationPackageSerializationRegistry; -import step.core.yaml.deserialization.PatchingContext; +import step.core.yaml.PatchingContext; public class AutomationPackageFragmentYamlImpl extends AbstractAutomationPackageFragmentYaml { diff --git a/step-automation-packages/step-automation-packages-yaml/src/test/java/step/core/yaml/PatchingContextTest.java b/step-automation-packages/step-automation-packages-yaml/src/test/java/step/core/yaml/PatchingContextTest.java new file mode 100644 index 0000000000..d4279eddfb --- /dev/null +++ b/step-automation-packages/step-automation-packages-yaml/src/test/java/step/core/yaml/PatchingContextTest.java @@ -0,0 +1,234 @@ +package step.core.yaml; + +import org.junit.Assert; +import org.junit.Test; +import step.automation.packages.deserialization.AutomationPackageSerializationRegistry; +import step.automation.packages.yaml.AutomationPackageDescriptorReader; +import step.automation.packages.yaml.YamlAutomationPackageVersions; +import step.automation.packages.yaml.model.AutomationPackageDescriptorYaml; +import step.core.scheduler.automation.AutomationPackageSchedule; +import step.core.scheduler.automation.AutomationPackageScheduleRegistration; +import step.core.yaml.deserialization.PatchableYamlList; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class PatchingContextTest { + + private static final String EXPECTED_UNMODIFIED = """ + schemaVersion: 1.0.0 + name: "complete-package" + fragments: + - "importPlans.yml" + - "importKeywords.yml" + keywords: + - JMeter: + name: "JMeter keyword from automation package" + description: "JMeter keyword 1" + executeLocally: false + useCustomTemplate: true + callTimeout: 1000 + jmeterTestplan: "jmeterProject1/jmeterProject1.xml" + routing: + criteriaA: valueA + criteriaB: valueB + schema: + type: object + properties: + firstName: + type: string + lastName: + type: string + required: [ "firstName", "lastName" ] + - Composite: + name: "Composite1" + plan: + root: + testCase: + children: + - echo: + text: "Just echo" + - return: + output: + - output1: "value" + - output2: + expression: "'some thing dynamic'" + plans: + - name: First Plan + root: + sequence: + continueOnError: false + children: + - if: + nodeName: IfBlock + condition: + expression: "controllerSettings.getSettingByKey('housekeeping_enabled').getValue()=='true'" + description: "my description" + children: + - assert: + actual: + expression: "'status'" + operator: "EQUALS" + doNegate: false + expected: + expression: "'ok'" + customErrorMessage: "my custom error" + - name: Second Plan + root: + testCase: + children: + - echo: + text: "Just echo" + schedules: + - name: "My first task" + cron: "*/5 * * * *" + executionParameters: + environment: "TEST" + planName: "First Plan" + - name: "My second task" + cron: "0 * * * *" + executionParameters: + environment: "PROD" + planName: "Second Plan" + """; + + private static final String EXPECTED_REMOVED_1 = """ + schemaVersion: 1.0.0 + name: "complete-package" + fragments: + - "importPlans.yml" + - "importKeywords.yml" + keywords: + - Composite: + name: "Composite1" + plan: + root: + testCase: + children: + - echo: + text: "Just echo" + - return: + output: + - output1: "value" + - output2: + expression: "'some thing dynamic'" + plans: + - name: First Plan + root: + sequence: + continueOnError: false + children: + - if: + nodeName: IfBlock + condition: + expression: "controllerSettings.getSettingByKey('housekeeping_enabled').getValue()=='true'" + description: "my description" + children: + - assert: + actual: + expression: "'status'" + operator: "EQUALS" + doNegate: false + expected: + expression: "'ok'" + customErrorMessage: "my custom error" + schedules: + - name: "My first task" + cron: "*/5 * * * *" + executionParameters: + environment: "TEST" + planName: "First Plan" + """; + + private static final String EXPECTED_REMOVED_2_MODIFIED = """ + schemaVersion: 1.0.0 + name: "complete-package" + fragments: + - "importPlans.yml" + - "importKeywords.yml" + keywords: [] + plans: [] + schedules: + - name: "This is now the new schedule name which is rather long, really long in fact" + active: false + cron: "*/5 * * * *" + planName: "First Plan" + executionParameters: {} + """; + + + private final AutomationPackageDescriptorReader reader; + + public PatchingContextTest() { + AutomationPackageSerializationRegistry serializationRegistry = new AutomationPackageSerializationRegistry(); + AutomationPackageScheduleRegistration.registerSerialization(serializationRegistry); + reader = new AutomationPackageDescriptorReader(YamlAutomationPackageVersions.ACTUAL_JSON_SCHEMA_PATH, serializationRegistry); + } + + @Test + public void testLowLevelDeletionModification() throws Exception { + File file = new File("src/test/resources/step/automation/packages/yaml/descriptors/completeDescriptor2.yml"); + try (InputStream is = new FileInputStream(file)) { + AutomationPackageDescriptorYaml descriptor = reader.readAutomationPackageDescriptor(is, ""); + PatchingContext patchingContext = descriptor.getPatchingContext(); + String current = ensureValid(patchingContext.getCurrentYaml()); + Assert.assertEquals(EXPECTED_UNMODIFIED, current); + + // we know that the claimed chunks are in the order [keywords, plans, schedules] + List lists = patchingContext.chunks.values().stream() + .filter(m -> m instanceof PatchableYamlList) + .map(x -> (PatchableYamlList) x) + .toList(); + assertEquals(3, lists.size()); + // remove first keyword, second plan, second schedule + lists.get(0).remove(0); + lists.get(1).remove(1); + lists.get(2).remove(1); + + current = ensureValid(patchingContext.getCurrentYaml()); + Assert.assertEquals(EXPECTED_REMOVED_1, current); + + // empty first two lists, modify schedule a little + AutomationPackageSchedule schedule1 = (AutomationPackageSchedule) lists.get(2).getFirst(); + schedule1.setName("This is now the new schedule name which is rather long, really long in fact"); + schedule1.setActive(false); + schedule1.getExecutionParameters().clear(); + // we have to tell the entity explicitly that it was modified + schedule1.setModified(); + lists.subList(0, 2).forEach(ArrayList::removeFirst); + + current = ensureValid(patchingContext.getCurrentYaml()); + Assert.assertEquals(EXPECTED_REMOVED_2_MODIFIED, current); + + lists.get(2).removeFirst(); + current = ensureValid(patchingContext.getCurrentYaml()); + + Assert.assertEquals(""" + schemaVersion: 1.0.0 + name: "complete-package" + fragments: + - "importPlans.yml" + - "importKeywords.yml" + keywords: [] + plans: [] + schedules: [] + """, current); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private String ensureValid(String yaml) throws Exception { + InputStream in = new ByteArrayInputStream(yaml.getBytes()); + // we're simply reading it and expecting it not to fail. + reader.readAutomationPackageDescriptor(in, ""); + return yaml; + } +} diff --git a/step-automation-packages/step-automation-packages-yaml/src/test/resources/step/automation/packages/yaml/descriptors/completeDescriptor2.yml b/step-automation-packages/step-automation-packages-yaml/src/test/resources/step/automation/packages/yaml/descriptors/completeDescriptor2.yml new file mode 100644 index 0000000000..df780f8314 --- /dev/null +++ b/step-automation-packages/step-automation-packages-yaml/src/test/resources/step/automation/packages/yaml/descriptors/completeDescriptor2.yml @@ -0,0 +1,74 @@ +schemaVersion: 1.0.0 +name: "complete-package" +fragments: + - "importPlans.yml" + - "importKeywords.yml" +keywords: + - JMeter: + name: "JMeter keyword from automation package" + description: "JMeter keyword 1" + executeLocally: false + useCustomTemplate: true + callTimeout: 1000 + jmeterTestplan: "jmeterProject1/jmeterProject1.xml" + routing: + criteriaA: valueA + criteriaB: valueB + schema: + type: object + properties: + firstName: + type: string + lastName: + type: string + required: [ "firstName", "lastName" ] + - Composite: + name: "Composite1" + plan: + root: + testCase: + children: + - echo: + text: "Just echo" + - return: + output: + - output1: "value" + - output2: + expression: "'some thing dynamic'" +plans: + - name: First Plan + root: + sequence: + continueOnError: false + children: + - if: + nodeName: IfBlock + condition: + expression: "controllerSettings.getSettingByKey('housekeeping_enabled').getValue()=='true'" + description: "my description" + children: + - assert: + actual: + expression: "'status'" + operator: "EQUALS" + doNegate: false + expected: + expression: "'ok'" + customErrorMessage: "my custom error" + - name: Second Plan + root: + testCase: + children: + - echo: + text: "Just echo" +schedules: + - name: "My first task" + cron: "*/5 * * * *" + executionParameters: + environment: "TEST" + planName: "First Plan" + - name: "My second task" + cron: "0 * * * *" + executionParameters: + environment: "PROD" + planName: "Second Plan" diff --git a/step-core-model/src/main/java/step/core/yaml/PatchableYamlModel.java b/step-core-model/src/main/java/step/core/yaml/PatchableYamlModel.java index 2dcfad3a80..0303ecbeef 100644 --- a/step-core-model/src/main/java/step/core/yaml/PatchableYamlModel.java +++ b/step-core-model/src/main/java/step/core/yaml/PatchableYamlModel.java @@ -19,23 +19,23 @@ package step.core.yaml; import com.fasterxml.jackson.core.JsonLocation; -import step.core.yaml.deserialization.PatchingContext; public interface PatchableYamlModel { - void setPatchingBounds(JsonLocation startLocation, JsonLocation endLocation); + void setPatchingContext(PatchingContext context); - int getStartOffset(); + PatchingContext getPatchingContext(); - int getIndent(); + String getCurrentYaml(String contextIndent); - int getEndOffset(); + void setModified(); - void setStartOffset(int startOffset); + enum StartingLineDeterminationStrategy { + SAME_LINE, + NEXT_CONTENT_LINE + } - void setEndOffset(int endOffset); + StartingLineDeterminationStrategy getStartingLineDeterminationStrategy(); - void setIndent(int indent); - - void setContext(PatchingContext context); + void onParsed(JsonLocation startLocation, JsonLocation endLocation); } diff --git a/step-core-model/src/main/java/step/core/yaml/PatchableYamlModelBase.java b/step-core-model/src/main/java/step/core/yaml/PatchableYamlModelBase.java index 4146a38d8b..ce6f3bbd53 100644 --- a/step-core-model/src/main/java/step/core/yaml/PatchableYamlModelBase.java +++ b/step-core-model/src/main/java/step/core/yaml/PatchableYamlModelBase.java @@ -20,68 +20,62 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonLocation; -import step.core.yaml.deserialization.PatchingContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PatchableYamlModelBase extends AbstractYamlModel implements PatchableYamlModel { + private static final Logger logger = LoggerFactory.getLogger(PatchableYamlModelBase.class); @JsonIgnore private PatchingContext context; - @JsonIgnore - private int startOffset = -1; - - @JsonIgnore - private int indent = -1; - - @JsonIgnore - private int endOffset = -1; - public PatchableYamlModelBase(PatchingContext context) { this.context = context; } + @Override @JsonIgnore - public void setPatchingBounds(JsonLocation startLocation, JsonLocation endLocation) { - startOffset = (int) startLocation.getCharOffset(); - endOffset = context.ensureNextEndOfLineOffset((int) endLocation.getCharOffset()); - indent = startLocation.getColumnNr() -1; - context.getPatchables().add(this); + public final PatchingContext getPatchingContext() { + return context; } @JsonIgnore - public int getStartOffset(){ - return startOffset; - } + private boolean modified = false; - @JsonIgnore - public int getIndent() { - return indent; + public void setModified() { + this.modified = true; } + @Override @JsonIgnore - public int getEndOffset() { - return endOffset; + public final void setPatchingContext(PatchingContext context) { + this.context = context; } - @Override - public void setStartOffset(int startOffset) { - this.startOffset = startOffset; + @JsonIgnore + public String getCurrentYaml(String contextIndent) { + if (!modified) { + String chunk = context.getChunk(this); + if (chunk != null) { + // use the original chunk, but still re-indent appropriately if needed + return context.reindent(chunk, contextIndent); + } + // fallthrough in case of any problem? + throw new IllegalStateException(); + } + return context.serialize(this, contextIndent); } @Override - public void setEndOffset(int endOffset) { - this.endOffset = endOffset; - } - - @JsonIgnore - @Override - public void setIndent(int indent) { - this.indent = indent; + public StartingLineDeterminationStrategy getStartingLineDeterminationStrategy() { + // Let's hope that this really is true for all subclasses :-) + return StartingLineDeterminationStrategy.SAME_LINE; } @Override - public void setContext(PatchingContext context) { - this.context = context; + public void onParsed(JsonLocation startLocation, JsonLocation endLocation) { + context.claimChunk(startLocation, endLocation, this); } + } diff --git a/step-core-model/src/main/java/step/core/yaml/PatchingContext.java b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java new file mode 100644 index 0000000000..2a0e4f084a --- /dev/null +++ b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java @@ -0,0 +1,281 @@ +package step.core.yaml; + +import com.fasterxml.jackson.core.JsonLocation; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import step.core.yaml.deserialization.AutomationPackageUpdateException; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +public class PatchingContext { + private static final Logger logger = LoggerFactory.getLogger(PatchingContext.class); + private final String sourceLocation; + private final CopyOnWriteArrayList initialLines; + + private final ObjectMapper mapper; + + protected final ConcurrentNavigableMap chunks = new ConcurrentSkipListMap<>(); + + public PatchingContext() { + this(new ObjectMapper()); + } + + public PatchingContext(ObjectMapper mapper) { + this(null, "", mapper); + } + + public PatchingContext(String sourceLocation, String yaml, ObjectMapper mapper) { + this.sourceLocation = sourceLocation; + this.initialLines = new CopyOnWriteArrayList<>(yaml.lines().toList()); + this.mapper = mapper; + } + + public ObjectMapper getMapper() { + return mapper; + } + + public String getChunk(PatchableYamlModel entity) { + return getChunkBounds(entity).map(this::getChunk) + .orElse(null); + } + + private Optional getChunkBounds(PatchableYamlModel entity) { + // this is not terribly efficient as it will be O(n), but we're talking small amounts of data + return chunks.entrySet().stream() + .filter(entry -> entry.getValue().equals(entity)) + .findFirst() + .map(Map.Entry::getKey); + } + + private String getChunk(ChunkBounds bounds) { + return String.join("\n", getLines(bounds)) + "\n"; + } + + private List getLines(ChunkBounds bounds) { + return initialLines.subList(bounds.startLineNumber - 1, bounds.endLineNumber); + } + + public record ChunkBounds(int startLineNumber, int endLineNumber) implements Comparable { + private static final Comparator COMPARATOR = Comparator + .comparingInt(ChunkBounds::startLineNumber) // lower startLine first + .thenComparing(Comparator.comparingInt(ChunkBounds::endLineNumber).reversed()); // larger endLine (i.e. larger chunk) first + + @Override + public int compareTo(ChunkBounds that) { + return COMPARATOR.compare(this, that); + } + + public boolean encompasses(ChunkBounds inner) { + return inner.startLineNumber >= this.startLineNumber && inner.endLineNumber <= this.endLineNumber; + } + } + + public String getCurrentYaml() { + List allBounds = getAllOuterBounds(); + StringBuilder yaml = new StringBuilder(); + for (ChunkBounds bound : allBounds) { + PatchableYamlModel patchableYamlModel = chunks.get(bound); + if (patchableYamlModel == null) { + // unclaimed, return original lines + yaml.append(getChunk(bound)); + } else { + // claimed, return whatever that patchable currently thinks its content is, + // using its original indentation + String indent = detectIndent(initialLines.get(bound.startLineNumber - 1)); // we only need the first line + yaml.append(patchableYamlModel.getCurrentYaml(indent)); + } + } + return yaml.toString(); + } + + /** + * @return a sorted list of bounds, encompassing the entire original + * file. Note this only returns top-level bounds, not bounds that are + * encompassed by other bounds. For instance: returns list bounds, + * but not bounds of items inside those lists. This also returns + * bounds for lines that are NOT claimed by anything. + */ + private List getAllOuterBounds() { + List allBounds = getClaimedOuterBounds(); + List unclaimedBounds = new ArrayList<>(); + // claimedBounds is sorted, we need to fill the gaps + int startLineNumber = 1; + for (ChunkBounds bound : allBounds) { + if (bound.startLineNumber > startLineNumber) { + unclaimedBounds.add(new ChunkBounds(startLineNumber, bound.startLineNumber - 1)); + } + startLineNumber = bound.endLineNumber + 1; + } + allBounds.addAll(unclaimedBounds); + unclaimedBounds.clear(); // not needed anymore, might as well free it + allBounds.sort(ChunkBounds.COMPARATOR); + if (!allBounds.isEmpty() && allBounds.getLast().endLineNumber < initialLines.size()) { + allBounds.add(new ChunkBounds(allBounds.getLast().endLineNumber + 1, initialLines.size())); + } + return allBounds; + } + + /** + * + * @return a sorted list of (only) the outer bounds of claimed lines, i.e. the chunk bounds + * that are owned by some PatchableYamlModel. Inner claims (e.g. actual objects inside a list) are disregarded + */ + private List getClaimedOuterBounds() { + List allBounds = chunks.keySet().stream().toList(); + ChunkBounds lastBound = null; + List outerBounds = new ArrayList<>(); + for (ChunkBounds bound : allBounds) { + if (lastBound != null) { + if (lastBound.encompasses(bound)) { + continue; + } + } + lastBound = bound; + outerBounds.add(bound); + } + return outerBounds; + } + + private String serializeUnindented(PatchableYamlModel entity) { + try { + return mapper.writeValueAsString(entity) + .replaceAll("---\n", "") + .trim(); + } catch (JsonProcessingException e) { + throw new AutomationPackageUpdateException("Error Serializing YAML object", e); + } + } + + /* + Note for this and following methods: because of how YAML works, contextIndent could + be simply a string of spaces (" "), but it also could contain a dash if the item + is contained in a list (" - "). Only the first line needs the list marker, all others + need to be aligned and consist only of spaces. + */ + public String serialize(PatchableYamlModel entity, String contextIndent) { + return indent(serializeUnindented(entity), contextIndent); + } + + String indent(String chunk, String contextIndent) { + if (chunk == null || chunk.isEmpty()) { + return chunk; + } + String onlyIndent = contextIndent.replace('-', ' '); + AtomicBoolean firstLine = new AtomicBoolean(true); + return chunk.lines() + .map(line -> firstLine.getAndSet(false) ? contextIndent + line : onlyIndent + line) + .collect(Collectors.joining("\n", "", "\n")); + } + + public String reindent(String chunk, String contextIndent) { + if (chunk == null || chunk.isEmpty()) { + return chunk; + } + String existingIndent = detectIndent(chunk); + if (existingIndent.equals(contextIndent)) { + return chunk; + } + String unindented = stripIndent(chunk, existingIndent.length()); + return indent(unindented, contextIndent); + } + + private String stripIndent(String chunk, int length) { + return chunk.lines() + .map(l -> { + // handle potential comments which may not be properly aligned. + if (l.trim().startsWith("#")) { + if (l.indexOf('#') < length) { + return l.trim(); + } + } + // short/empty lines + if (l.length() <= length) { + return ""; + } + // any other line - chop off indent + return l.substring(length); + }) + .collect(Collectors.joining("\n", "", "\n")); + } + + private String detectIndent(String chunk) { + // chunk is guaranteed to be non-empty; we don't care how many lines this has, we only need to look at the first one. + int pos = 0; + int len = chunk.length(); + + // Leading spaces + while (pos < len && chunk.charAt(pos) == ' ') { + pos++; + } + + // Optional list indicator (dash) and following spaces + if (pos < len && chunk.charAt(pos) == '-') { + pos++; + // 3. Consume spaces after the dash + while (pos < len && chunk.charAt(pos) == ' ') { + pos++; + } + } + return chunk.substring(0, pos); + } + + + public ChunkBounds claimChunk(JsonLocation startLocation, JsonLocation endLocation, PatchableYamlModel entity) { + int startLineNumber = startLocation.getLineNr(); + int endLineNumber = endLocation.getLineNr(); + String startLine = initialLines.get(startLineNumber - 1); + if (startLocation.getColumnNr() >= startLine.length()) { + // sometimes the start is at the correct line, sometimes it's at the end (in fact, AFTER the end) of some previous line + PatchableYamlModel.StartingLineDeterminationStrategy strategy = entity.getStartingLineDeterminationStrategy(); + logger.debug("{}: entity of {} seems to start at end of line {} (columnNr={}, lineLength={}), determining correct entity start line (strategy={})", sourceLocation, + entity.getClass(), startLocation.getLineNr(), startLocation.getColumnNr(), startLine.length(), strategy); + if (strategy == PatchableYamlModel.StartingLineDeterminationStrategy.NEXT_CONTENT_LINE) { + boolean found = false; + for (int l = startLineNumber; l < initialLines.size(); l++) { + String candidate = initialLines.get(l).trim(); + if (!candidate.isEmpty() && !candidate.startsWith("#")) { + startLineNumber = l + 1; + logger.debug("Using first non-empty, non-comment line (lineNumber={}, line content={})", startLineNumber, candidate); + found = true; + break; + } + } + if (!found) { + logger.debug("Unable to determine correct start line for entity, keeping original line number!"); + } + } + } + ChunkBounds bounds = new ChunkBounds(startLineNumber, endLineNumber); + chunks.put(bounds, entity); + return bounds; + } + + public ChunkBounds appendAndClaim(PatchableYamlModel entity, String chunk) { + List lines = chunk.lines().toList(); + // yes, initialLines won't be so "initial" anymore ;-) + initialLines.addAll(lines); + int endLine = initialLines.size(); + int startLine = endLine - lines.size() + 1; + ChunkBounds bounds = new ChunkBounds(startLine, endLine); + chunks.put(bounds, entity); + return bounds; + } + + + public void replaceEntity(PatchableYamlModel oldPatchable, PatchableYamlModel newPatchable) { + newPatchable.setPatchingContext(this); + newPatchable.setModified(); + getChunkBounds(oldPatchable).ifPresent(bounds -> chunks.put(bounds, newPatchable)); + } +} diff --git a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlList.java b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlList.java index 7ba84bb908..99cd566165 100644 --- a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlList.java +++ b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlList.java @@ -21,80 +21,76 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonLocation; import step.core.yaml.PatchableYamlModel; +import step.core.yaml.PatchingContext; import java.util.ArrayList; import java.util.Collection; +import java.util.Objects; import java.util.function.UnaryOperator; -public class PatchableYamlList extends ArrayList implements PatchableYamlModel{ +public class PatchableYamlList extends ArrayList implements PatchableYamlModel { - private PatchingContext context; + private final PatchingContext patchingContext; private final String fieldName; + private volatile PatchingContext.ChunkBounds bounds; - @JsonIgnore - private int startOffset = -1; - - @JsonIgnore - private int indent = -1; - - @JsonIgnore - private int endOffset = -1; + public PatchableYamlList(PatchingContext patchingContext, String fieldName) { - public PatchableYamlList(PatchingContext context, String fieldName) { - - this(new ArrayList<>(), context, fieldName); + this(new ArrayList<>(), patchingContext, fieldName); } - protected PatchableYamlList(Collection delegate, PatchingContext context, String fieldName) { - super(delegate); - this.context = context; - this.fieldName = fieldName; + protected PatchableYamlList(Collection content, PatchingContext patchingContext, String fieldName) { + super(content); + this.patchingContext = Objects.requireNonNull(patchingContext); + this.fieldName = Objects.requireNonNull(fieldName); } @Override - public boolean remove(Object item) { - if (super.remove(item)) { - PatchableYamlModel patchableItem = (PatchableYamlModel) item; - context.removePatchable(patchableItem); + @JsonIgnore + public PatchingContext getPatchingContext() { + return patchingContext; + } - if (super.isEmpty()) { - context.removePatchable(this); - } - return true; + @Override + @JsonIgnore + public String getCurrentYaml(String contextIndent) { + if (isEmpty()) { + return contextIndent + fieldName + ": []\n"; } - return false; + String childIndent = " ".repeat(contextIndent.length()) + " - "; + // Simply return a concatenated list of the current items; they're responsible for their own serialization + // Note that in theory we could even try to preserve the original comments between items (if there are any), + // but this could become complicated if entries get deleted, so we omit it for now. + StringBuilder sb = new StringBuilder(); + sb.append(contextIndent).append(fieldName).append(":").append("\n"); + stream().map(item -> (PatchableYamlModel) item).forEach(item -> { + sb.append(item.getCurrentYaml(childIndent)); + }); + return sb.toString(); + } @Override public boolean add(T item) { - if (!context.contains(this)) { - context.appendEmptyPatchable(this); + if (bounds == null) { + // This list has not been registered with the context yet, meaning it hasn't been parsed from a file, but manually added. + // We'll need to add it to the context as a new object. + synchronized (this) { + if (bounds == null) { + // FIXME: For now, this assumes all lists are top-level (see the hardcoded indent below) + bounds = patchingContext.appendAndClaim(this, getCurrentYaml("")); + } + } } - PatchableYamlModel patchableItem = (PatchableYamlModel) item; - if (super.isEmpty()) { - patchableItem.setIndent(getListItemMarker().length()); - context.replaceContainerPatchable(this, patchableItem, fieldName + ":\n" + getListItemMarker()); - } else { - PatchableYamlModel last = (PatchableYamlModel) get(size()-1); - - context.addPatchableAfter(last, patchableItem, getListItemMarker(last)); - } + PatchableYamlModel patchableItem = (PatchableYamlModel) item; + patchableItem.setPatchingContext(this.getPatchingContext()); + patchableItem.setModified(); super.add(item); return true; } - private String getListItemMarker(PatchableYamlModel last) { - String yaml = context.getYaml(); - int listItemMarkerStartOffset = yaml.lastIndexOf("\n", last.getStartOffset()); - return yaml.substring( listItemMarkerStartOffset, last.getStartOffset()); - } - - private String getListItemMarker() { - return " ".repeat(indent) + "- "; - } - @Override public void replaceAll(UnaryOperator operator) { super.replaceAll(item -> { @@ -123,54 +119,29 @@ public void clear() { super.forEach(this::remove); } - public void replaceItem(PatchableYamlModel oldEntity, PatchableYamlModel newEntity) { + public void replaceItem(PatchableYamlModel oldEntity, PatchableYamlModel newEntity) { replaceAll(item -> item == oldEntity ? (T) newEntity : item); - context.replacePatchable(oldEntity, newEntity); - } - - @JsonIgnore - public void setPatchingBounds(JsonLocation startLocation, JsonLocation endLocation) { - startOffset = (int) startLocation.getCharOffset(); - endOffset = context.ensureNextEndOfLineOffset((int) endLocation.getCharOffset()); - indent = startLocation.getColumnNr() -1; - context.getPatchables().add(this); - } - - @Override - public int getStartOffset(){ - return startOffset; - } - - @Override - public int getIndent() { - return indent; + patchingContext.replaceEntity(oldEntity, newEntity); } @Override - public int getEndOffset() { - return endOffset; - } - - - @Override - public void setStartOffset(int startOffset) { - this.startOffset = startOffset; + @JsonIgnore + public void setPatchingContext(PatchingContext context) { + throw new UnsupportedOperationException(); } @Override - public void setEndOffset(int endOffset) { - this.endOffset = endOffset; + public StartingLineDeterminationStrategy getStartingLineDeterminationStrategy() { + return StartingLineDeterminationStrategy.NEXT_CONTENT_LINE; } - @JsonIgnore @Override - public void setIndent(int indent) { - this.indent = indent; + public void setModified() { + throw new UnsupportedOperationException(); } @Override - @JsonIgnore - public void setContext(PatchingContext context) { - this.context = context; + public void onParsed(JsonLocation startLocation, JsonLocation endLocation) { + bounds = patchingContext.claimChunk(startLocation, endLocation, this); } } diff --git a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlListDeserializer.java b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlListDeserializer.java index 192ec9d849..f5b81d0f2d 100644 --- a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlListDeserializer.java +++ b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlListDeserializer.java @@ -48,7 +48,8 @@ public Collection deserialize(JsonParser p, DeserializationContext ctxt) JsonLocation startLocation = patchingParser.getLastLocationForToken(JsonToken.FIELD_NAME); Collection entity = delegate.deserialize(p, ctxt, new ArrayList<>()); PatchableYamlList patchableYamlList = new PatchableYamlList<>(entity, patchingParser.getPatchingContext(), patchingParser.currentName()); - patchableYamlList.setPatchingBounds(startLocation, patchingParser.getLastDistinctLocation()); + patchableYamlList.onParsed(startLocation, patchingParser.getLastDistinctLocation()); + //patchableYamlList.getPatchingContext().claimChunk(, patchableYamlList); return patchableYamlList; } return super.deserialize(p, ctxt); diff --git a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlModelDeserializer.java b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlModelDeserializer.java index af2993db76..4a96e32eed 100644 --- a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlModelDeserializer.java +++ b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlModelDeserializer.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.JsonLocation; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; @@ -32,7 +31,6 @@ import java.io.IOException; public class PatchableYamlModelDeserializer extends JsonDeserializer implements ContextualDeserializer { - private final JsonDeserializer delegate; public PatchableYamlModelDeserializer(JsonDeserializer delegate) { @@ -41,11 +39,10 @@ public PatchableYamlModelDeserializer(JsonDeserializer delegate) { @Override public T deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - if (p instanceof PatchingParserDelegate) { - PatchingParserDelegate patchingParser = (PatchingParserDelegate) p; + if (p instanceof PatchingParserDelegate patchingParser) { JsonLocation startItem = patchingParser.currentLocation(); T entity = delegate.deserialize(p, ctxt); - entity.setPatchingBounds(startItem, patchingParser.getLastDistinctLocation()); + entity.onParsed(startItem, patchingParser.getLastDistinctLocation()); return entity; } return delegate.deserialize(p, ctxt); diff --git a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchingContext.java b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchingContext.java deleted file mode 100644 index 8c759474e7..0000000000 --- a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchingContext.java +++ /dev/null @@ -1,166 +0,0 @@ -/******************************************************************************* - * Copyright (C) 2026, exense GmbH - * - * This file is part of STEP - * - * STEP is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * STEP is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with STEP. If not, see . - ******************************************************************************/ -package step.core.yaml.deserialization; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import step.core.yaml.PatchableYamlModel; - -import java.util.ArrayList; -import java.util.List; - -public class PatchingContext { - private String yaml; - - private final List patchables = new ArrayList<>(); - private final ObjectMapper mapper; - - public PatchingContext() { - yaml = ""; - mapper = new ObjectMapper(); - } - - public PatchingContext(String yaml, ObjectMapper mapper) { - this.yaml = yaml; - this.mapper = mapper; - } - - public String getYaml() { - return yaml; - } - - public ObjectMapper getMapper() { - return mapper; - } - - public List getPatchables() { - return patchables; - } - - public void setYaml(String yaml) { - this.yaml = yaml; - } - - private String entityStringWithIndent(PatchableYamlModel entity, int indent) { - try { - String indentString = " ".repeat(indent); - return mapper - .writeValueAsString(entity) - .replaceAll("---\n", "") - .trim() - .replaceAll("\n", "\n" + indentString); - } catch (JsonProcessingException e) { - throw new AutomationPackageUpdateException("Error Serializing new object", e); - } - } - - public void replacePatchable(PatchableYamlModel oldPatchable, PatchableYamlModel newPatchable) { - String entityString = entityStringWithIndent(newPatchable, oldPatchable.getIndent()); - - int endOffset = oldPatchable.getEndOffset(); - int delta = entityString.length() - (endOffset - oldPatchable.getStartOffset()); - - - yaml = yaml.substring(0, oldPatchable.getStartOffset()) - + entityString - + yaml.substring(oldPatchable.getEndOffset()); - - newPatchable.setStartOffset(oldPatchable.getStartOffset()); - newPatchable.setIndent(oldPatchable.getIndent()); - newPatchable.setEndOffset(oldPatchable.getEndOffset() + delta); - - patchables.replaceAll(p -> p == oldPatchable ? newPatchable : p); - - updatePatchableOffsetsAfter(newPatchable, endOffset, delta); - } - - public void removePatchable(PatchableYamlModel patchable) { - if (!patchables.contains(patchable)) return; - - int previousLineEnd = ensurePreviousEndOfLineOffset(patchable.getStartOffset()); - int delta = previousLineEnd - patchable.getEndOffset(); - yaml = yaml.substring(0, previousLineEnd) + yaml.substring(patchable.getEndOffset()); - updatePatchableOffsetsAfter(patchable, patchable.getEndOffset(), delta); - patchables.remove(patchable); - } - - - - public void addPatchableAfter(PatchableYamlModel last, PatchableYamlModel patchableItem, String entityPrefix) { - - String entityString = entityStringWithIndent(patchableItem, last.getIndent()); - - yaml = yaml.substring(0, last.getEndOffset()) + entityPrefix + entityString + yaml.substring(last.getEndOffset()); - - patchableItem.setStartOffset(last.getEndOffset() + entityPrefix.length()); - patchableItem.setEndOffset(last.getEndOffset() + entityPrefix.length() + entityString.length()); - patchableItem.setIndent(last.getIndent()); - patchableItem.setContext(this); - - patchables.add(patchables.indexOf(last)+1, patchableItem); - - updatePatchableOffsetsAfter(patchableItem, patchableItem.getEndOffset(), entityString.length() + entityPrefix.length()); - } - - private void updatePatchableOffsetsAfter(PatchableYamlModel patchable, int endOffset, int delta) { - for(int i = patchables.indexOf(patchable) + 1; i < patchables.size(); i++) { - PatchableYamlModel successor = patchables.get(i); - if (successor.getStartOffset() >= endOffset) { - successor.setStartOffset(successor.getStartOffset()+delta); - } - successor.setEndOffset(successor.getEndOffset()+delta); - } - } - - public boolean contains(PatchableYamlModel patchable) { - return patchables.contains(patchable); - } - - public int ensureNextEndOfLineOffset(int offset) { - return Math.max(yaml.indexOf("\n", offset), offset); - } - - public int ensurePreviousEndOfLineOffset(int offset) { - return Math.max(yaml.lastIndexOf("\n", offset), 0); - } - - public void replaceContainerPatchable(PatchableYamlModel containerPatchable, PatchableYamlModel child, String containerPrefix) { - String childString = entityStringWithIndent(child, child.getIndent()); - yaml = yaml.substring(0, containerPatchable.getStartOffset()) + containerPrefix + childString + yaml.substring(containerPatchable.getEndOffset()); - - containerPatchable.setEndOffset(containerPatchable.getStartOffset() + containerPrefix.length() + childString.length()); - child.setStartOffset(containerPatchable.getStartOffset() + containerPrefix.length()); - child.setEndOffset(containerPatchable.getEndOffset()); - - patchables.add(patchables.indexOf(containerPatchable), child); - - int delta = containerPrefix.length() + childString.length() - (containerPatchable.getEndOffset() - containerPatchable.getStartOffset()); - updatePatchableOffsetsAfter(containerPatchable, containerPatchable.getEndOffset(), delta); - } - - public void appendEmptyPatchable(PatchableYamlModel patchable) { - patchable.setIndent(0); - yaml += "\n"; - patchable.setStartOffset(yaml.length()); - patchable.setEndOffset(yaml.length()); - patchable.setIndent(0); - patchables.add(patchable); - yaml += "\n"; - } -} diff --git a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchingParserDelegate.java b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchingParserDelegate.java index dedf45b116..eef97c33c9 100644 --- a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchingParserDelegate.java +++ b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchingParserDelegate.java @@ -22,10 +22,9 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.core.util.JsonParserDelegate; +import step.core.yaml.PatchingContext; import java.io.IOException; -import java.util.ArrayDeque; -import java.util.Deque; import java.util.HashMap; import java.util.Map; diff --git a/step-core-model/src/main/java/step/parameter/automation/AutomationPackageParameter.java b/step-core-model/src/main/java/step/parameter/automation/AutomationPackageParameter.java index a07a308c81..aecc711f00 100644 --- a/step-core-model/src/main/java/step/parameter/automation/AutomationPackageParameter.java +++ b/step-core-model/src/main/java/step/parameter/automation/AutomationPackageParameter.java @@ -18,13 +18,16 @@ ******************************************************************************/ package step.parameter.automation; -import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.OptBoolean; import step.commons.activation.Expression; import step.core.dynamicbeans.DynamicValue; import step.core.yaml.PatchableYamlModelBase; +import step.core.yaml.PatchingContext; import step.core.yaml.YamlFieldCustomCopy; import step.core.yaml.YamlModel; -import step.core.yaml.deserialization.PatchingContext; import step.parameter.Parameter; import step.parameter.ParameterScope; diff --git a/step-core/src/main/java/step/automation/packages/model/YamlAutomationPackageKeyword.java b/step-core/src/main/java/step/automation/packages/model/YamlAutomationPackageKeyword.java index ed114c6cb9..e1e76a63c9 100644 --- a/step-core/src/main/java/step/automation/packages/model/YamlAutomationPackageKeyword.java +++ b/step-core/src/main/java/step/automation/packages/model/YamlAutomationPackageKeyword.java @@ -20,8 +20,7 @@ import step.automation.packages.StagingAutomationPackageContext; import step.core.yaml.PatchableYamlModelBase; -import step.core.yaml.YamlModelUtils; -import step.core.yaml.deserialization.PatchingContext; +import step.core.yaml.PatchingContext; import step.functions.Function; public class YamlAutomationPackageKeyword extends PatchableYamlModelBase implements AutomationPackageKeyword { @@ -29,7 +28,6 @@ public class YamlAutomationPackageKeyword extends PatchableYamlModelBase impleme private AbstractYamlFunction yamlKeyword; - public YamlAutomationPackageKeyword(AbstractYamlFunction yamlKeyword, PatchingContext context) { super(context); this.yamlKeyword = yamlKeyword; diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/VersionedYamlPlan.java b/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/VersionedYamlPlan.java index 9f45f796e8..ba5a297424 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/VersionedYamlPlan.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/VersionedYamlPlan.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import step.core.yaml.deserialization.PatchingContext; +import step.core.yaml.PatchingContext; @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder(VersionedYamlPlan.VERSION_FIELD_NAME) diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/YamlPlan.java b/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/YamlPlan.java index 25f0b1d250..74ba1898bb 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/YamlPlan.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/YamlPlan.java @@ -28,7 +28,7 @@ import step.core.plans.agents.configuration.AgentProvisioningConfigurationDeserializer; import step.core.plans.agents.configuration.AgentProvisioningConfigurationSerializer; import step.core.yaml.PatchableYamlModelBase; -import step.core.yaml.deserialization.PatchingContext; +import step.core.yaml.PatchingContext; import step.core.yaml.model.NamedYamlArtefact; import java.util.List; @@ -81,7 +81,7 @@ public List getCategories() { return categories; } - public void setCategories(List categories) { - this.categories = categories; - } + public void setCategories(List categories) { + this.categories = categories; + } } diff --git a/step-plans/step-plans-core/src/main/java/step/core/scheduler/automation/AutomationPackageSchedule.java b/step-plans/step-plans-core/src/main/java/step/core/scheduler/automation/AutomationPackageSchedule.java index 0a3b4fc65f..32f85099f4 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/scheduler/automation/AutomationPackageSchedule.java +++ b/step-plans/step-plans-core/src/main/java/step/core/scheduler/automation/AutomationPackageSchedule.java @@ -20,12 +20,13 @@ import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.OptBoolean; import step.core.yaml.PatchableYamlModelBase; -import step.core.yaml.deserialization.PatchingContext; +import step.core.yaml.PatchingContext; -import java.util.Map; import java.util.List; +import java.util.Map; public class AutomationPackageSchedule extends PatchableYamlModelBase { @@ -35,8 +36,10 @@ public class AutomationPackageSchedule extends PatchableYamlModelBase { private Boolean active = true; private String cron; + @JsonInclude(JsonInclude.Include.NON_NULL) // otherwise fails schema check after re-reading serialized value private List cronExclusions; private String planName; + @JsonInclude(JsonInclude.Include.NON_NULL) private String assertionPlanName; private Map executionParameters; diff --git a/step-plans/step-plans-parser/src/main/java/step/plans/automation/YamlPlainTextPlan.java b/step-plans/step-plans-parser/src/main/java/step/plans/automation/YamlPlainTextPlan.java index f9bfe54608..33607d5e5a 100644 --- a/step-plans/step-plans-parser/src/main/java/step/plans/automation/YamlPlainTextPlan.java +++ b/step-plans/step-plans-parser/src/main/java/step/plans/automation/YamlPlainTextPlan.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.OptBoolean; import step.core.yaml.PatchableYamlModelBase; -import step.core.yaml.deserialization.PatchingContext; +import step.core.yaml.PatchingContext; import step.plans.nl.RootArtefactType; import java.util.List; @@ -38,7 +38,7 @@ public class YamlPlainTextPlan extends PatchableYamlModelBase { private String file; @JsonCreator - public YamlPlainTextPlan(@JacksonInject(useInput = OptBoolean.FALSE, optional = OptBoolean.TRUE) PatchingContext context) { + public YamlPlainTextPlan(@JacksonInject(useInput = OptBoolean.FALSE, optional = OptBoolean.TRUE) PatchingContext context) { super(context); } diff --git a/step-plans/step-plans-yaml-parser/src/main/java/step/plans/parser/yaml/YamlPlanReader.java b/step-plans/step-plans-yaml-parser/src/main/java/step/plans/parser/yaml/YamlPlanReader.java index d88f06da38..063df0d9ed 100644 --- a/step-plans/step-plans-yaml-parser/src/main/java/step/plans/parser/yaml/YamlPlanReader.java +++ b/step-plans/step-plans-yaml-parser/src/main/java/step/plans/parser/yaml/YamlPlanReader.java @@ -43,12 +43,14 @@ import step.core.scanner.AnnotationScanner; import step.core.scanner.CachedAnnotationScanner; import step.core.yaml.PatchableYamlModel; +import step.core.yaml.PatchingContext; import step.core.yaml.deserialization.PatchableYamlList; import step.core.yaml.deserialization.PatchableYamlListDeserializer; import step.core.yaml.deserialization.PatchableYamlModelDeserializer; -import step.core.yaml.deserialization.PatchingContext; import step.core.yaml.deserializers.StepYamlDeserializer; import step.core.yaml.deserializers.StepYamlDeserializersScanner; +import step.core.yaml.model.AbstractYamlArtefact; +import step.core.yaml.model.NamedYamlArtefact; import step.core.yaml.serializers.StepYamlSerializersScanner; import step.migration.MigrationManager; import step.plans.nl.RootArtefactType; @@ -56,8 +58,6 @@ import step.plans.parser.yaml.deserializers.UpgradableYamlPlanDeserializer; import step.plans.parser.yaml.migrations.AbstractYamlPlanMigrationTask; import step.plans.parser.yaml.migrations.YamlPlanMigration; -import step.core.yaml.model.AbstractYamlArtefact; -import step.core.yaml.model.NamedYamlArtefact; import step.plans.parser.yaml.model.YamlPlanVersions; import step.plans.parser.yaml.schema.YamlPlanValidationException; import step.repositories.parser.StepsParser; @@ -67,7 +67,11 @@ import java.io.OutputStream; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import static step.core.scanner.Classes.newInstanceAs; @@ -216,6 +220,7 @@ public static ObjectMapper createDefaultYamlMapper() { YAMLFactory yamlFactory = new YAMLFactory(); // Disable native type id to enable conversion to generic Documents yamlFactory.disable(YAMLGenerator.Feature.USE_NATIVE_TYPE_ID); + yamlFactory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR); return DefaultJacksonMapperProvider.getObjectMapper(yamlFactory); } @@ -330,13 +335,13 @@ public Plan yamlPlanToPlan(YamlPlan yamlPlan) { } public VersionedYamlPlan planToVersionedYamlPlan(Plan plan) { - VersionedYamlPlan yamlPlan = new VersionedYamlPlan(new PatchingContext("", yamlMapper), currentVersion.toString()); + VersionedYamlPlan yamlPlan = new VersionedYamlPlan(new PatchingContext(yamlMapper), currentVersion.toString()); setYamlPlanFieldsFromPlan(yamlPlan, plan); return yamlPlan; } public YamlPlan planToYamlPlan(Plan plan) { - YamlPlan yamlPlan = new YamlPlan(new PatchingContext("", yamlMapper)); + YamlPlan yamlPlan = new YamlPlan(new PatchingContext(yamlMapper)); setYamlPlanFieldsFromPlan(yamlPlan, plan); return yamlPlan; } From 654121f60d5ac4a01e23936f1a8efd751d3633d2 Mon Sep 17 00:00:00 2001 From: Christoph Langguth Date: Fri, 15 May 2026 06:49:39 +0200 Subject: [PATCH 02/10] SED-4429 bugfixes and other improvements --- step-ap-ide/pom.xml | 91 +++++++++++++++++-- .../java/step/ap_ide/{App.java => FXApp.java} | 7 +- .../src/main/java/step/ap_ide/StepUp.java | 79 +++++++++++----- .../{test => main}/resources/step.properties | 0 .../work-initial/automation-package.yml | 3 + .../AutomationPackageYamlFragmentManager.java | 66 ++++++++++++-- .../java/step/core/yaml/PatchingContext.java | 6 +- .../automation/YamlCompositeFunction.java | 11 ++- 8 files changed, 212 insertions(+), 51 deletions(-) rename step-ap-ide/src/main/java/step/ap_ide/{App.java => FXApp.java} (95%) rename step-ap-ide/src/{test => main}/resources/step.properties (100%) diff --git a/step-ap-ide/pom.xml b/step-ap-ide/pom.xml index 5e3d894e10..4e32a62cc3 100644 --- a/step-ap-ide/pom.xml +++ b/step-ap-ide/pom.xml @@ -11,6 +11,10 @@ 0.0.0-SNAPSHOT + + 21.0.2 + + ch.exense.step @@ -35,25 +39,92 @@ org.openjfx javafx-controls - 21.0.2 + ${javafx.version} + win + + + org.openjfx + javafx-web + ${javafx.version} + win + + + + org.openjfx + javafx-controls + ${javafx.version} + linux + + + org.openjfx + javafx-web + ${javafx.version} + linux + + + + org.openjfx + javafx-controls + ${javafx.version} + mac org.openjfx javafx-web - 21.0.2 + ${javafx.version} + mac + + + org.openjfx + javafx-controls + ${javafx.version} + mac-aarch64 + + + org.openjfx + javafx-web + ${javafx.version} + mac-aarch64 + + - - - - - - - - + + org.apache.maven.plugins + maven-shade-plugin + 3.5.2 + + + package + + shade + + + false + + + + + step.ap_ide.StepUp + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + diff --git a/step-ap-ide/src/main/java/step/ap_ide/App.java b/step-ap-ide/src/main/java/step/ap_ide/FXApp.java similarity index 95% rename from step-ap-ide/src/main/java/step/ap_ide/App.java rename to step-ap-ide/src/main/java/step/ap_ide/FXApp.java index f9962cc9ff..3c37d5aaaf 100644 --- a/step-ap-ide/src/main/java/step/ap_ide/App.java +++ b/step-ap-ide/src/main/java/step/ap_ide/FXApp.java @@ -18,9 +18,9 @@ import java.io.File; import java.nio.file.Files; -public class App extends Application { +public class FXApp extends Application { - private static final Logger logger = LoggerFactory.getLogger(App.class); + private static final Logger logger = LoggerFactory.getLogger(FXApp.class); private final WebView webView = new WebView(); private final MenuBar menuBar = new MenuBar(); @@ -65,7 +65,6 @@ private void initToolBar() { Button reloadButton = new Button("Reload"); reloadButton.setOnAction(e -> webView.getEngine().reload()); - // NEW: Button to dump the HTML so we can see what's actually there Button dumpHtmlButton = new Button("Dump HTML"); dumpHtmlButton.setOnAction(e -> { try { @@ -82,6 +81,8 @@ private void initToolBar() { newEmptyApButton.setOnAction(e -> { try { File workDir = Files.createTempDirectory("automationPackageCollectionTest").toFile(); + System.err.println("CREATED AND NOW USING NEW WORKDIR: " + workDir.getAbsolutePath()); + StepUp.useAutomationPackageDirectory(workDir); webView.getEngine().reload(); } catch (Exception ex) { logger.error("error", ex); diff --git a/step-ap-ide/src/main/java/step/ap_ide/StepUp.java b/step-ap-ide/src/main/java/step/ap_ide/StepUp.java index 680405e907..938961b7f2 100644 --- a/step-ap-ide/src/main/java/step/ap_ide/StepUp.java +++ b/step-ap-ide/src/main/java/step/ap_ide/StepUp.java @@ -5,6 +5,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import step.automation.packages.AutomationPackageHookRegistry; +import step.automation.packages.JavaAutomationPackageArchive; import step.automation.packages.JavaAutomationPackageReader; import step.automation.packages.deserialization.AutomationPackageSerializationRegistry; import step.automation.packages.yaml.AutomationPackageYamlFragmentManager; @@ -16,6 +17,11 @@ import step.plans.parser.yaml.YamlPlan; import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Objects; import java.util.Properties; @@ -23,40 +29,43 @@ public class StepUp { private static final Logger logger = LoggerFactory.getLogger(StepUp.class); + // These are meant for development to have something to play with without having to recreate everything from scratch private static final String workDirName = "work"; private static final String initialDirName = "src/main/resources/work-initial"; public static void main(String[] args) throws Exception { - // Use an IntelliJ run configuration that uses '%MODULE_WORKING_DIR%' (verbatim) as the working directory, and use - // -config="src/test/resources/step.properties" as program argument - ControllerServer.main(args); + Configuration configuration = new Configuration(); + // this uses the existing methods, we could also refactor the implementation to make it a little easier. + InputStream propsStream = StepUp.class.getClassLoader().getResourceAsStream("step.properties"); + configuration.getUnderlyingPropertyObject().load(propsStream); + new ControllerServer(configuration).start(); initWorkdir(); - App.main(args); // this will never return + FXApp.main(args); // this will never return } - static final JavaAutomationPackageReader READER; + private static final JavaAutomationPackageReader READER; static { AutomationPackageSerializationRegistry serializationRegistry = new AutomationPackageSerializationRegistry(); AutomationPackageHookRegistry hookRegistry = new AutomationPackageHookRegistry(); - // required for reading parameters, apparently the manager can be null AutomationPackageParametersRegistration.registerParametersHooks(hookRegistry, serializationRegistry, null); - READER = new JavaAutomationPackageReader(YamlAutomationPackageVersions.ACTUAL_JSON_SCHEMA_PATH, hookRegistry, serializationRegistry, new Configuration()); } private static void initWorkdir() throws Exception { File workDir = new File(workDirName); + File initialDir = new File(initialDirName); if (!workDir.isDirectory() || (workDir.isDirectory() && Objects.requireNonNull(workDir.listFiles()).length == 0)) { logger.info("Work directory is not present or empty, initializing from " + initialDirName); - File initialDir = new File(initialDirName); if (!initialDir.isDirectory()) { - throw new RuntimeException("Not a directory: " + initialDir.getAbsolutePath()); - } - FileUtils.copyDirectory(initialDir, workDir); - if (!workDir.isDirectory()) { - throw new RuntimeException("Something went wrong while initializing directory: " + workDir.getAbsolutePath()); + workDir = Files.createTempDirectory("step-ap-ide-").toFile(); + logger.warn("initialDir is not present, using temporary directory: {}", workDir.getAbsolutePath()); + } else { + FileUtils.copyDirectory(initialDir, workDir); + if (!workDir.isDirectory()) { + throw new RuntimeException("Something went wrong while initializing directory: " + workDir.getAbsolutePath()); + } } } else { logger.info("Using existing work directory at: " + workDir.getAbsolutePath()); @@ -65,21 +74,47 @@ private static void initWorkdir() throws Exception { useAutomationPackageDirectory(workDir); } -// private static void setPropertiesWriteToFragment(Properties properties, String entityName, String fragment) { -// properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, entityName), fragment); -// properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, entityName), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); -// } - static void useAutomationPackageDirectory(File apDir) throws Exception { + verifyOrCreateMainAPFile(apDir); var fragmentManager = StepUp.READER.getAutomationPackageYamlFragmentManager(apDir); Properties properties = new Properties(); - // parameters all go into parameters.yml - properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, Parameter.ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); - properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, Parameter.ENTITY_NAME), "parameters.yml"); - properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, YamlPlan.PLANS_ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.PER_OBJECT.name()); + + // variant 1: + // parameters all go into parameters.yml, plans go into separate files in plans/$PLAN_NAME.yml + // Only works if the target files/directories already exist, so disabled for now + if (1 == 0) { + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, Parameter.ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, Parameter.ENTITY_NAME), "parameters.yml"); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, YamlPlan.PLANS_ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.PER_OBJECT.name()); + // keywords seem to use PER_OBJECT by default? + } + // variant 2: simple, everything goes into main descriptor + if (1 == 1) { + String mainFile = Paths.get(fragmentManager.descriptorYaml.getFragmentUrl().toURI()).toFile().getName(); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, Parameter.ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, Parameter.ENTITY_NAME), mainFile); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, YamlPlan.PLANS_ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, YamlPlan.PLANS_ENTITY_NAME), mainFile); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, "keywords"), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, "keywords"), mainFile); + } fragmentManager.setProperties(properties); var automationPackageCollectionFactory = new AutomationPackageCollectionFactory(new Properties(), fragmentManager); CurrentlyOpenedAutomationPackageCollectionFactory.getInstance().setCurrentFactory(automationPackageCollectionFactory); } + + private static void verifyOrCreateMainAPFile(File apDir) throws Exception { + for (String fileName : JavaAutomationPackageArchive.METADATA_FILES) { + if (new File(apDir, fileName).isFile()) { + return; + } + } + File descriptor = new File(apDir, JavaAutomationPackageArchive.METADATA_FILES.getFirst()); + logger.info("Initializing AP directory with new descriptor: {}", descriptor.getAbsolutePath()); + PrintWriter pw = new PrintWriter(new FileOutputStream(descriptor)); + pw.println("schemaVersion: 1.0.0"); + pw.println("name: \"My package\""); // TODO: make this configurable somehow + pw.close(); + } } diff --git a/step-ap-ide/src/test/resources/step.properties b/step-ap-ide/src/main/resources/step.properties similarity index 100% rename from step-ap-ide/src/test/resources/step.properties rename to step-ap-ide/src/main/resources/step.properties diff --git a/step-ap-ide/src/main/resources/work-initial/automation-package.yml b/step-ap-ide/src/main/resources/work-initial/automation-package.yml index c80fc11302..dc1f2ce474 100644 --- a/step-ap-ide/src/main/resources/work-initial/automation-package.yml +++ b/step-ap-ide/src/main/resources/work-initial/automation-package.yml @@ -13,6 +13,9 @@ alertingRules: BindingValueEqualsPredicate: value: "myValue" plans: [] +parameters: + - key: "parameterInMainDescriptor" + value: "value1" fragments: - "keywords.yml" - "plans/*.yml" diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java index 866f8ca711..a6487f001b 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java @@ -35,8 +35,11 @@ import step.parameter.Parameter; import step.parameter.automation.AutomationPackageParameter; import step.plans.parser.yaml.YamlPlan; +import step.plugins.functions.types.CompositeFunction; +import java.io.ByteArrayInputStream; import java.io.File; +import java.io.InputStream; import java.net.MalformedURLException; import java.net.URL; import java.net.URLEncoder; @@ -44,6 +47,7 @@ import java.nio.file.Path; import java.text.MessageFormat; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -74,7 +78,7 @@ public enum NewObjectFragmentMode { protected final Map pathToYamlFragment; protected Properties properties = new Properties(); - protected final AutomationPackageFragmentYaml descriptorYaml; + public final AutomationPackageFragmentYaml descriptorYaml; public AutomationPackageYamlFragmentManager(AutomationPackageDescriptorYaml descriptorYaml, Map fragmentMap, AutomationPackageDescriptorReader descriptorReader, StagingAutomationPackageContext stagingContext) { @@ -158,9 +162,16 @@ public synchronized Plan savePlan(Plan plan) { public synchronized step.functions.Function saveFunction(step.functions.Function function) { AutomationPackageFragmentYaml fragment = fragmentMap.get(function); if (fragment == null) { - fragment = fragmentForNewObject(function, YamlPlan.PLANS_ENTITY_NAME); + fragment = fragmentForNewObject(function, "keywords"); fragmentMap.put(function, fragment); pathToYamlFragment.put(fragment.getFragmentUrl().toString(), fragment); + YamlAutomationPackageKeyword newKeyword = createNewYamlKeyword(function); + if (newKeyword != null) { + patchableMap.put(function, newKeyword); + addFragmentEntity(fragment, fragment.getKeywords(), newKeyword); + } else { + System.err.println("SAVING OF FUNCTION OF TYPE " + function.getClass().getName() + " IS NOT CURRENTLY SUPPORTED"); + } } else { YamlAutomationPackageKeyword yamlKeyword = (YamlAutomationPackageKeyword) patchableMap.get(function); yamlKeyword.getYamlKeyword().updateFromFunction(function); @@ -169,6 +180,43 @@ public synchronized step.functions.Function saveFunction(step.functions.Function return function; } + private YamlAutomationPackageKeyword createNewYamlKeyword(step.functions.Function function) { + // FIXME: I know, this is is a giant horrible stinking hack for now, there needs to be a better way. + if (function instanceof CompositeFunction compositeFunction) { + try { + // I don't know what the proper way is to serialize this, but we know that deserialization should work... + YamlPlan plan = descriptorReader.getPlanReader().planToYamlPlan(compositeFunction.getPlan()); + plan.setName(null); + // we only want to use the serialization functions here + PatchingContext patchingContext = new PatchingContext(descriptorReader.yamlObjectMapper); + StringBuilder yaml = new StringBuilder(""" + keywords: + - Composite: + plan: + """); + yaml.append(patchingContext.serialize(plan, " ".repeat(8))); + + // There are more attributes, this is just a PoC anyway + Optional.ofNullable(function.getAttribute("name")).ifPresent(value -> { + yaml.append(patchingContext.serialize(Map.of("name", value), " ".repeat(6))); + }); + Optional.ofNullable(function.getDescription()).ifPresent(value -> { + yaml.append(patchingContext.serialize(Map.of("description", value), " ".repeat(6))); + }); + + InputStream is = new ByteArrayInputStream(yaml.toString().getBytes()); + var fragment = descriptorReader.readAutomationPackageFragment(is, "horrible-hack", "horrible-hack"); + return fragment.getKeywords().stream().findFirst().orElse(null); + } catch (Exception e) { + // TODO: better error handling + e.printStackTrace(); + return null; + } + } else { + return null; + } + } + private void addFragmentEntity(AutomationPackageFragmentYaml fragment, PatchableYamlList entityList, T newEntity) { entityList.add(newEntity); fragment.writeToDisk(); @@ -244,7 +292,7 @@ public void removeFunction(step.functions.Function function) { AutomationPackageFragmentYaml fragment = fragmentMap.get(function); YamlAutomationPackageKeyword yamlKeyword = (YamlAutomationPackageKeyword) patchableMap.get(function); - fragment.getPlans().remove(yamlKeyword); + fragment.getKeywords().remove(yamlKeyword); patchableMap.remove(function); fragmentMap.remove(function); @@ -268,20 +316,22 @@ public void removeAdditionalFieldObject(B } public synchronized BO saveAdditionalFieldObject(BO object, java.util.function.Function newYamlObjectCreator, String fieldName) { - AutomationPackageFragmentYaml fragment = fragmentMap.get(object); - if (fragment == null) { + AutomationPackageFragmentYaml fragment; + if (fragmentMap.get(object) == null) { fragment = fragmentForNewObject(object, fieldName); YO newYamlObject = newYamlObjectCreator.apply(fragment.getPatchingContext()); - PatchableYamlList list = (PatchableYamlList) fragment.getAdditionalFields().getOrDefault(fieldName, new PatchableYamlList(fragment.getPatchingContext(), fieldName)); - + PatchableYamlList list = (PatchableYamlList) fragment.getAdditionalFields() + .computeIfAbsent(fieldName, k -> new PatchableYamlList(fragment.getPatchingContext(), fieldName)); fragmentMap.put(object, fragment); pathToYamlFragment.put(fragment.getFragmentUrl().toString(), fragment); addFragmentEntity(fragment, list, newYamlObject); patchableMap.put(object, newYamlObject); } else { + fragment = fragmentMap.get(object); YO newYamlObject = newYamlObjectCreator.apply(fragment.getPatchingContext()); - PatchableYamlList list = (PatchableYamlList) fragment.getAdditionalFields().getOrDefault(fieldName, new PatchableYamlList(fragment.getPatchingContext(), fieldName)); + PatchableYamlList list = (PatchableYamlList) fragment.getAdditionalFields() + .computeIfAbsent(fieldName, k -> new PatchableYamlList(fragment.getPatchingContext(), fieldName)); YO oldYamlObject = (YO) patchableMap.get(object); modifyFragmentEntity(fragment, list, oldYamlObject, newYamlObject); diff --git a/step-core-model/src/main/java/step/core/yaml/PatchingContext.java b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java index 2a0e4f084a..ce0494dbd6 100644 --- a/step-core-model/src/main/java/step/core/yaml/PatchingContext.java +++ b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java @@ -147,10 +147,10 @@ private List getClaimedOuterBounds() { return outerBounds; } - private String serializeUnindented(PatchableYamlModel entity) { + private String serializeUnindented(Object entity) { try { return mapper.writeValueAsString(entity) - .replaceAll("---\n", "") + .replaceFirst("^---\\s*\\n", "") .trim(); } catch (JsonProcessingException e) { throw new AutomationPackageUpdateException("Error Serializing YAML object", e); @@ -163,7 +163,7 @@ be simply a string of spaces (" "), but it also could contain a dash if the item is contained in a list (" - "). Only the first line needs the list marker, all others need to be aligned and consist only of spaces. */ - public String serialize(PatchableYamlModel entity, String contextIndent) { + public String serialize(Object entity, String contextIndent) { return indent(serializeUnindented(entity), contextIndent); } diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/plugins/functions/types/automation/YamlCompositeFunction.java b/step-plans/step-plans-base-artefacts/src/main/java/step/plugins/functions/types/automation/YamlCompositeFunction.java index de1a5f281d..b58d97518d 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/plugins/functions/types/automation/YamlCompositeFunction.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/plugins/functions/types/automation/YamlCompositeFunction.java @@ -34,9 +34,6 @@ import step.plans.parser.yaml.YamlPlan; import step.plugins.functions.types.CompositeFunction; -import java.util.Map; -import java.util.Objects; - @YamlModel(name = "Composite") @JsonInclude(JsonInclude.Include.NON_DEFAULT) public class YamlCompositeFunction extends AbstractYamlFunction { @@ -75,10 +72,14 @@ public void updateFromFunction(Function function) { if (function instanceof CompositeFunction) { Plan plan = ((CompositeFunction) function).getPlan(); // plan name is optional, the composite function name is used by default + // FIXME: discuss what exactly this is supposed to do and how it relates to the comment above :-) if (this.plan.getName() != null && !this.plan.getName().isEmpty()) { - this.plan.setName(plan.getAttribute(AbstractOrganizableObject.NAME));; + this.plan.setName(plan.getAttribute(AbstractOrganizableObject.NAME)); } - ObjectMapper mapper = this.plan.getRoot().getYamlArtefact().getYamlObjectMapper(); + // I have no idea why that mapper would be null sometimes: + // ObjectMapper mapper = this.plan.getRoot().getYamlArtefact().getYamlObjectMapper(); + // That one should work... + ObjectMapper mapper = this.getPlan().getPatchingContext().getMapper(); this.plan.setRoot(new NamedYamlArtefact(AbstractYamlArtefact.toYamlArtefact(plan.getRoot(), mapper))); } } From 87ad6baae74cc50647942de618e60a046c08e48b Mon Sep 17 00:00:00 2001 From: Cyril Misev Date: Fri, 29 May 2026 11:53:21 +0200 Subject: [PATCH 03/10] SED-4672 make fragments list patchable and add glob pattern if not included --- .../src/main/java/step/ap_ide/StepUp.java | 3 +- .../jmeterProject1/jmeterProject1.xml | 4 + .../work-initial/jsProject/jsSample.js | 0 .../resources/work-initial/lib/fakeLib.jar | 0 .../work-initial/nodeProject/nodeSample.ts | 0 .../AutomationPackageParameterCollection.java | 2 +- .../AutomationPackageCollectionTestBase.java | 8 +- ...utomationPackageFragmentReferenceTest.java | 84 +++++++++ ...omationPackageParameterCollectionTest.java | 5 + .../expected/This Plan was renamed.yml | 8 + .../descriptorAfterNewFragmentReference.yml | 26 +++ .../resources/expected/parametersAfterAdd.yml | 1 + .../packages/AutomationPackageReader.java | 38 ++-- .../AutomationPackageYamlFragmentManager.java | 177 +++++++++++------- ...AutomationPackageFragmentDeserializer.java | 8 +- ...AbstractAutomationPackageFragmentYaml.java | 36 ++-- .../model/AutomationPackageFragmentYaml.java | 9 +- ...AutomationPackageDescriptorReaderTest.java | 5 +- .../step/core/yaml/PatchingContextTest.java | 14 +- .../yaml/NamedObjectPatchableYamlModel.java | 24 +++ .../core/yaml/PatchableYamlModelBase.java | 2 +- .../java/step/core/yaml/PatchingContext.java | 2 +- .../PatchableYamlPrimitive.java | 54 ++++++ .../AutomationPackageParameter.java | 4 +- .../java/step/plans/parser/yaml/YamlPlan.java | 4 +- 25 files changed, 377 insertions(+), 141 deletions(-) create mode 100644 step-ap-ide/src/main/resources/work-initial/jmeterProject1/jmeterProject1.xml create mode 100644 step-ap-ide/src/main/resources/work-initial/jsProject/jsSample.js create mode 100644 step-ap-ide/src/main/resources/work-initial/lib/fakeLib.jar create mode 100644 step-ap-ide/src/main/resources/work-initial/nodeProject/nodeSample.ts create mode 100644 step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java create mode 100644 step-automation-packages/step-automation-packages-collections/src/test/resources/expected/This Plan was renamed.yml create mode 100644 step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewFragmentReference.yml create mode 100644 step-core-model/src/main/java/step/core/yaml/NamedObjectPatchableYamlModel.java create mode 100644 step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlPrimitive.java diff --git a/step-ap-ide/src/main/java/step/ap_ide/StepUp.java b/step-ap-ide/src/main/java/step/ap_ide/StepUp.java index 938961b7f2..56778a88b0 100644 --- a/step-ap-ide/src/main/java/step/ap_ide/StepUp.java +++ b/step-ap-ide/src/main/java/step/ap_ide/StepUp.java @@ -21,7 +21,6 @@ import java.io.InputStream; import java.io.PrintWriter; import java.nio.file.Files; -import java.nio.file.Paths; import java.util.Objects; import java.util.Properties; @@ -90,7 +89,7 @@ static void useAutomationPackageDirectory(File apDir) throws Exception { } // variant 2: simple, everything goes into main descriptor if (1 == 1) { - String mainFile = Paths.get(fragmentManager.descriptorYaml.getFragmentUrl().toURI()).toFile().getName(); + String mainFile = fragmentManager.descriptorYaml.getFragmentPath().toFile().getName(); properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, Parameter.ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, Parameter.ENTITY_NAME), mainFile); properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, YamlPlan.PLANS_ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); diff --git a/step-ap-ide/src/main/resources/work-initial/jmeterProject1/jmeterProject1.xml b/step-ap-ide/src/main/resources/work-initial/jmeterProject1/jmeterProject1.xml new file mode 100644 index 0000000000..3ed06774ee --- /dev/null +++ b/step-ap-ide/src/main/resources/work-initial/jmeterProject1/jmeterProject1.xml @@ -0,0 +1,4 @@ + + + + diff --git a/step-ap-ide/src/main/resources/work-initial/jsProject/jsSample.js b/step-ap-ide/src/main/resources/work-initial/jsProject/jsSample.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/step-ap-ide/src/main/resources/work-initial/lib/fakeLib.jar b/step-ap-ide/src/main/resources/work-initial/lib/fakeLib.jar new file mode 100644 index 0000000000..e69de29bb2 diff --git a/step-ap-ide/src/main/resources/work-initial/nodeProject/nodeSample.ts b/step-ap-ide/src/main/resources/work-initial/nodeProject/nodeSample.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/step-automation-packages/step-automation-packages-collections/src/main/java/step/core/collections/AutomationPackageParameterCollection.java b/step-automation-packages/step-automation-packages-collections/src/main/java/step/core/collections/AutomationPackageParameterCollection.java index a7780a210e..22f96f5f9d 100644 --- a/step-automation-packages/step-automation-packages-collections/src/main/java/step/core/collections/AutomationPackageParameterCollection.java +++ b/step-automation-packages/step-automation-packages-collections/src/main/java/step/core/collections/AutomationPackageParameterCollection.java @@ -41,7 +41,7 @@ private void initialzeRecordsFromFragments(AutomationPackageYamlFragmentManager @Override public Parameter save(Parameter parameter){ - return super.save(fragmentManager.saveAdditionalFieldObject(parameter, context -> AutomationPackageParameter.forContext(context, parameter), Parameter.ENTITY_NAME)); + return super.save(fragmentManager.saveAdditionalFieldObject(parameter, AutomationPackageParameter.fromParameter(parameter), Parameter.ENTITY_NAME)); } @Override diff --git a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageCollectionTestBase.java b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageCollectionTestBase.java index 3ed25186ed..9d3a1560e6 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageCollectionTestBase.java +++ b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageCollectionTestBase.java @@ -84,9 +84,13 @@ protected void assertFilesEqual(Path expected, Path actual) throws IOException { } protected void setPropertiesWriteToFragment(String entityName, String fragment) { + setPropertiesWriteMode(entityName, fragment, AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT); + } + + protected void setPropertiesWriteMode(String entityName, String path, AutomationPackageYamlFragmentManager.NewObjectFragmentMode mode) { Properties properties = new Properties(); - properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, entityName), fragment); - properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, entityName), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, entityName), path); + properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, entityName), mode.name()); fragmentManager.setProperties(properties); } diff --git a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java new file mode 100644 index 0000000000..795cb6b387 --- /dev/null +++ b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java @@ -0,0 +1,84 @@ +/******************************************************************************* + * Copyright (C) 2026, exense GmbH + * + * This file is part of STEP + * + * STEP is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * STEP is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with STEP. If not, see . + ******************************************************************************/ +package step.core.collections; + +import org.junit.Before; +import org.junit.Test; +import step.artefacts.Echo; +import step.artefacts.Sequence; +import step.automation.packages.AutomationPackageReadingException; +import step.automation.packages.yaml.AutomationPackageYamlFragmentManager; +import step.core.artefacts.AbstractArtefact; +import step.core.dynamicbeans.DynamicValue; +import step.core.plans.Plan; +import step.core.yaml.deserialization.AutomationPackageConcurrentEditException; +import step.plans.parser.yaml.YamlPlan; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.Assert.*; + +public class AutomationPackageFragmentReferenceTest extends AutomationPackageCollectionTestBase { + + private Collection planCollection; + + public AutomationPackageFragmentReferenceTest() { + super(); + } + + @Before + public void setUp() throws IOException, AutomationPackageReadingException { + super.setUp(); + AutomationPackageCollectionFactory collectionFactory = new AutomationPackageCollectionFactory(new Properties(), fragmentManager); + planCollection = collectionFactory.getCollection(YamlPlan.PLANS_ENTITY_NAME, Plan.class); + } + + + @Test + public void testAddPlanToNewFragmentAndRename() throws IOException { + + Sequence sequence = new Sequence(); + Echo echo = new Echo(); + echo.setText(new DynamicValue<>("Hello World")); + sequence.addChild(echo); + + Plan plan = new Plan(sequence); + Map attributes = new HashMap<>(); + attributes.put(AbstractArtefact.NAME, "Hello World Plan"); + plan.setAttributes(attributes); + + setPropertiesWriteMode(YamlPlan.PLANS_ENTITY_NAME, "newPlansPath", AutomationPackageYamlFragmentManager.NewObjectFragmentMode.PER_OBJECT); + + planCollection.save(plan); + + assertFilesEqual(expectedFilesPath.resolve("Hello World Plan.yml"), destinationDirectory.toPath().resolve("newPlansPath").resolve("Hello World Plan.yml")); + assertFilesEqual(expectedFilesPath.resolve("descriptorAfterNewFragmentReference.yml"), destinationDirectory.toPath().resolve("automation-package.yml")); + + attributes.put(AbstractArtefact.NAME, "This Plan was renamed"); + + planCollection.save(plan); + + assertFilesEqual(expectedFilesPath.resolve("This Plan was renamed.yml"), destinationDirectory.toPath().resolve("newPlansPath").resolve("This Plan was renamed.yml")); + assertFilesEqual(expectedFilesPath.resolve("descriptorAfterNewFragmentReference.yml"), destinationDirectory.toPath().resolve("automation-package.yml")); + } +} diff --git a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageParameterCollectionTest.java b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageParameterCollectionTest.java index 8c33637c55..c08d588e7b 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageParameterCollectionTest.java +++ b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageParameterCollectionTest.java @@ -57,6 +57,9 @@ public void testParameterModify() throws IOException { Parameter parameter = optionalParameter.get(); parameter.getValue().setValue("myModifiedValue"); + + setPropertiesWriteToFragment(Parameter.ENTITY_NAME, "parameters.yml"); + parameterCollection.save(parameter); assertFilesEqual(expectedFilesPath.resolve("parametersAfterModification.yml"), destinationDirectory.toPath().resolve("parameters.yml")); @@ -72,6 +75,7 @@ public void testParameterAddAndModify() throws IOException { setPropertiesWriteToFragment(Parameter.ENTITY_NAME, "parameters.yml"); + parameter.setPriority(1); parameterCollection.save(parameter); assertFilesEqual(expectedFilesPath.resolve("parametersAfterAdd.yml"), destinationDirectory.toPath().resolve("parameters.yml")); @@ -80,6 +84,7 @@ public void testParameterAddAndModify() throws IOException { parameterCollection.save(parameter); parameter.setValue(new DynamicValue<>("foo")); + parameter.setPriority(null); parameterCollection.save(parameter); assertFilesEqual(expectedFilesPath.resolve("parametersAfterAddAndModification.yml"), destinationDirectory.toPath().resolve("parameters.yml")); diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/This Plan was renamed.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/This Plan was renamed.yml new file mode 100644 index 0000000000..55ef8445fc --- /dev/null +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/This Plan was renamed.yml @@ -0,0 +1,8 @@ +--- +plans: + - name: "This Plan was renamed" + root: + sequence: + children: + - echo: + text: "Hello World" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewFragmentReference.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewFragmentReference.yml new file mode 100644 index 0000000000..7724332157 --- /dev/null +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewFragmentReference.yml @@ -0,0 +1,26 @@ +schemaVersion: 1.0.0 +name: "My package" +alertingRules: + - name: "Rule1" + description: "My test alerting rule" + eventClass: ExecutionEndedEvent + conditions: + - BindingCondition: + description: "condition 1" + bindingKey: "myKey" + negate: false + predicate: + BindingValueEqualsPredicate: + value: "myValue" +plans: [] +parameters: + - key: "paramInMainAP" + value: "once" +fragments: + - "keywords.yml" + - "plans/*.yml" + - "schedules.yml" + - "parameters.yml" + - "parameters2.yml" + - "unknown.yml" + - "newPlansPath/*.yml" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/parametersAfterAdd.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/parametersAfterAdd.yml index 0ab61a81ad..3997b99ca0 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/parametersAfterAdd.yml +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/parametersAfterAdd.yml @@ -15,3 +15,4 @@ parameters: - key: "addedParameter" value: "test" description: "This is an added Parameter before modification" + priority: 1 diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java index 4fead6c631..6fcdd64b28 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java @@ -29,6 +29,7 @@ import step.automation.packages.yaml.model.AutomationPackageFragmentYaml; import step.core.plans.Plan; import step.core.yaml.deserialization.PatchableYamlList; +import step.core.yaml.deserialization.PatchableYamlPrimitive; import step.functions.Function; import step.plans.automation.YamlPlainTextPlan; import step.plans.nl.RootArtefactType; @@ -42,13 +43,10 @@ import java.io.InputStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; +import java.util.*; import java.util.stream.Collectors; /** @@ -130,7 +128,7 @@ protected AutomationPackageContent buildAutomationPackage(AutomationPackageDescr // apply imported fragments recursively if (descriptor != null) { - fillAutomationPackageWithImportedFragments(res, descriptor, archive, new HashMap<>()); + fillAutomationPackageWithImportedFragments(res, descriptor, archive, new HashSet<>()); } return res; } @@ -183,32 +181,32 @@ protected AutomationPackageContent newContentInstance() { public AutomationPackageYamlFragmentManager getAutomationPackageYamlFragmentManager(T archive) throws AutomationPackageReadingException { AutomationPackageDescriptorReader reader = getOrCreateDescriptorReader(); - URL descriptorURL = archive.getDescriptorYamlUrl(); - try (InputStream inputStream = descriptorURL.openStream()) { + URL descriptorUrl = archive.getDescriptorYamlUrl(); + try (InputStream inputStream = descriptorUrl.openStream()) { AutomationPackageDescriptorYaml descriptor = reader.readAutomationPackageDescriptor(inputStream, archive.getOriginalFileName()); - descriptor.setFragmentUrl(descriptorURL); + descriptor.setFragmentPath(Path.of(descriptorUrl.getPath())); AutomationPackageContent content = newContentInstance(); - Map fragmentMap = new ConcurrentHashMap<>(); - fillAutomationPackageWithImportedFragments(content, descriptor, archive, fragmentMap); - StagingAutomationPackageContext stagingContext = new StagingAutomationPackageContext(null, AutomationPackageOperationMode.LOCAL, new LocalResourceManagerImpl(Path.of(descriptorURL.getPath()).getParent().toFile()), archive, content, null, null, new HashMap<>()); - return new AutomationPackageYamlFragmentManager(descriptor, fragmentMap, getOrCreateDescriptorReader(), stagingContext); + Set fragments = new HashSet<>(); + fillAutomationPackageWithImportedFragments(content, descriptor, archive, fragments); + StagingAutomationPackageContext stagingContext = new StagingAutomationPackageContext(null, AutomationPackageOperationMode.LOCAL, new LocalResourceManagerImpl(Path.of(descriptorUrl.getPath()).getParent().toFile()), archive, content, null, null, new HashMap<>()); + return new AutomationPackageYamlFragmentManager(descriptor, fragments, getOrCreateDescriptorReader(), stagingContext); } catch (IOException e) { throw new AutomationPackageReadingException("Failed to read automation package for editing", e); } } - private void fillAutomationPackageWithImportedFragments(AutomationPackageContent targetPackage, AutomationPackageFragmentYaml fragment, T archive, Map fragmentYamlMap) throws AutomationPackageReadingException { + private void fillAutomationPackageWithImportedFragments(AutomationPackageContent targetPackage, AutomationPackageFragmentYaml fragment, T archive, Set fragments) throws AutomationPackageReadingException { fillContentSections(targetPackage, fragment, archive); if (!fragment.getFragments().isEmpty()) { - for (String importedFragmentReference : fragment.getFragments()) { - List resources = archive.getResourcesByPattern(importedFragmentReference); + for (PatchableYamlPrimitive importedFragmentReference : fragment.getFragments()) { + List resources = archive.getResourcesByPattern(importedFragmentReference.toString()); for (URL resource : resources) { try (InputStream fragmentYamlStream = resource.openStream()) { - fragment = getOrCreateDescriptorReader().readAutomationPackageFragment(fragmentYamlStream, resource.toString(), archive.getAutomationPackageName()); - fragmentYamlMap.put(resource.toString(), fragment); - fragment.setFragmentUrl(resource); - fillAutomationPackageWithImportedFragments(targetPackage, fragment, archive, fragmentYamlMap); + AutomationPackageFragmentYaml referencedFragment = getOrCreateDescriptorReader().readAutomationPackageFragment(fragmentYamlStream, resource.toString(), archive.getAutomationPackageName()); + fragments.add(referencedFragment); + referencedFragment.setFragmentPath(Path.of(resource.getPath())); + fillAutomationPackageWithImportedFragments(targetPackage, referencedFragment, archive, fragments); } catch (IOException e) { throw new AutomationPackageReadingException("Unable to read fragment in automation package: " + importedFragmentReference, e); } diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java index a6487f001b..ff5472dfa8 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java @@ -18,6 +18,7 @@ ******************************************************************************/ package step.automation.packages.yaml; +import org.apache.commons.io.FileUtils; import step.automation.packages.StagingAutomationPackageContext; import step.automation.packages.model.YamlAutomationPackageKeyword; import step.automation.packages.yaml.model.AutomationPackageDescriptorYaml; @@ -25,12 +26,13 @@ import step.automation.packages.yaml.model.AutomationPackageFragmentYamlImpl; import step.core.accessors.AbstractOrganizableObject; import step.core.plans.Plan; +import step.core.yaml.NamedObjectPatchableYamlModel; import step.core.yaml.PatchableYamlModel; import step.core.yaml.PatchableYamlModelBase; import step.core.yaml.PatchingContext; import step.core.yaml.deserialization.AutomationPackagePerObjectSaveUnsupportedException; -import step.core.yaml.deserialization.AutomationPackageUpdateException; import step.core.yaml.deserialization.PatchableYamlList; +import step.core.yaml.deserialization.PatchableYamlPrimitive; import step.functions.Function; import step.parameter.Parameter; import step.parameter.automation.AutomationPackageParameter; @@ -39,16 +41,14 @@ import java.io.ByteArrayInputStream; import java.io.File; +import java.io.IOException; import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; import java.net.URLEncoder; import java.nio.charset.Charset; +import java.nio.file.FileSystems; import java.nio.file.Path; -import java.text.MessageFormat; -import java.util.Map; -import java.util.Optional; -import java.util.Properties; +import java.nio.file.PathMatcher; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -75,24 +75,24 @@ public enum NewObjectFragmentMode { protected final Map patchableMap = new ConcurrentHashMap<>(); protected final Map fragmentMap = new ConcurrentHashMap<>(); - protected final Map pathToYamlFragment; + protected final Set fragments; protected Properties properties = new Properties(); public final AutomationPackageFragmentYaml descriptorYaml; - public AutomationPackageYamlFragmentManager(AutomationPackageDescriptorYaml descriptorYaml, Map fragmentMap, AutomationPackageDescriptorReader descriptorReader, StagingAutomationPackageContext stagingContext) { + public AutomationPackageYamlFragmentManager(AutomationPackageDescriptorYaml descriptorYaml, Set fragments, AutomationPackageDescriptorReader descriptorReader, StagingAutomationPackageContext stagingContext) { this.descriptorReader = descriptorReader; this.descriptorYaml = descriptorYaml; - pathToYamlFragment = fragmentMap; - apRoot = Path.of(descriptorYaml.getFragmentUrl().getPath()) - .getParent(); + apRoot = descriptorYaml.getFragmentPath().getParent(); this.stagingContext = stagingContext; initializeMaps(descriptorYaml); - pathToYamlFragment.values().stream() + this.fragments = fragments; + + fragments.stream() .filter(f -> f != descriptorYaml) .forEach(this::initializeMaps); } @@ -102,7 +102,6 @@ public void setProperties(Properties properties) { } public void initializeMaps(AutomationPackageFragmentYaml fragment) { - pathToYamlFragment.put(fragment.getFragmentUrl().toString(), fragment); for (YamlPlan yamlPlan : fragment.getPlans()) { Plan plan = descriptorReader.getPlanReader().yamlPlanToPlan(yamlPlan); patchableMap.put(plan, yamlPlan); @@ -145,13 +144,12 @@ public synchronized Plan savePlan(Plan plan) { AutomationPackageFragmentYaml fragment = fragmentMap.get(plan); if (fragment == null) { - fragment = fragmentForNewObject(plan, YamlPlan.PLANS_ENTITY_NAME); + fragment = fragmentForNewObject(newYamlPlan, YamlPlan.PLANS_ENTITY_NAME); fragmentMap.put(plan, fragment); - pathToYamlFragment.put(fragment.getFragmentUrl().toString(), fragment); addFragmentEntity(fragment, fragment.getPlans(), newYamlPlan); } else { YamlPlan oldYamlPlan = (YamlPlan) patchableMap.get(plan); - modifyFragmentEntity(fragment, fragment.getPlans(), oldYamlPlan, newYamlPlan); + modifyFragmentEntity(fragment, fragment.getPlans(), oldYamlPlan, newYamlPlan, YamlPlan.PLANS_ENTITY_NAME); } patchableMap.put(plan, newYamlPlan); @@ -162,10 +160,9 @@ public synchronized Plan savePlan(Plan plan) { public synchronized step.functions.Function saveFunction(step.functions.Function function) { AutomationPackageFragmentYaml fragment = fragmentMap.get(function); if (fragment == null) { - fragment = fragmentForNewObject(function, "keywords"); - fragmentMap.put(function, fragment); - pathToYamlFragment.put(fragment.getFragmentUrl().toString(), fragment); YamlAutomationPackageKeyword newKeyword = createNewYamlKeyword(function); + fragment = fragmentForNewObject(newKeyword, YamlAutomationPackageKeyword.KEYWORDS_ENTITY_NAME); + fragmentMap.put(function, fragment); if (newKeyword != null) { patchableMap.put(function, newKeyword); addFragmentEntity(fragment, fragment.getKeywords(), newKeyword); @@ -175,7 +172,7 @@ public synchronized step.functions.Function saveFunction(step.functions.Function } else { YamlAutomationPackageKeyword yamlKeyword = (YamlAutomationPackageKeyword) patchableMap.get(function); yamlKeyword.getYamlKeyword().updateFromFunction(function); - modifyFragmentEntity(fragment, fragment.getKeywords(), yamlKeyword, yamlKeyword); + modifyFragmentEntity(fragment, fragment.getKeywords(), yamlKeyword, yamlKeyword, YamlAutomationPackageKeyword.KEYWORDS_ENTITY_NAME); } return function; } @@ -222,54 +219,100 @@ private void addFragmentEntity(AutomationPackageF fragment.writeToDisk(); } - private void modifyFragmentEntity(AutomationPackageFragmentYaml fragment, PatchableYamlList entityList, T oldEntity, T newEntity) { + private void modifyFragmentEntity(AutomationPackageFragmentYaml fragment, PatchableYamlList entityList, T oldEntity, T newEntity, String fieldName) { entityList.replaceItem(oldEntity, newEntity); + Path oldRelativePath = determineObjectRelativePath(oldEntity, fieldName, false); + Path newRelativePath = determineObjectRelativePath(newEntity, fieldName, false); + if (!oldRelativePath.equals(newRelativePath)) { + Path absoluteOldPath = apRoot.resolve(oldRelativePath); + + if (absoluteOldPath.equals(fragment.getFragmentPath())) { + Path absoluteNewPath = apRoot.resolve(newRelativePath); + try { + FileUtils.moveFile(absoluteOldPath.toFile(), absoluteNewPath.toFile()); + fragment.setFragmentPath(absoluteNewPath); + + AutomationPackageFragmentYaml referencingFragment = determineReferencingFragment(oldRelativePath) + .orElse(descriptorYaml); + + + Path referencePath = determineObjectRelativePath(newEntity, fieldName, true); + if (referencingFragment.getFragments().removeIf(f -> f.getValue().equals(oldRelativePath.toString()))) { + referencingFragment.getFragments().add(new PatchableYamlPrimitive<>(referencingFragment.getPatchingContext(), referencePath.toString())); + referencingFragment.writeToDisk(); + }; + } catch (IOException ignored) { + } + } + } fragment.writeToDisk(); } - private AutomationPackageFragmentYaml fragmentForNewObject(AbstractOrganizableObject p, String fieldName) { + private AutomationPackageFragmentYaml fragmentForNewObject(PatchableYamlModel p, String fieldName) { - NewObjectFragmentMode mode = NewObjectFragmentMode.valueOf(properties.getProperty(String.format(PROPERTY_NEW_OBJECT_FRAGMENT_MODE, fieldName), NewObjectFragmentMode.PER_OBJECT.name())); - String defaultRelativeFragmentPath = fieldName; - if (mode == NewObjectFragmentMode.FRAGMENT) { - defaultRelativeFragmentPath = defaultRelativeFragmentPath + ".yml"; + Path path = determineObjectRelativePath(p, fieldName, false); + Path absolutePath = apRoot.resolve(path); + + Optional optionalExistingFragment = fragmentMap + .values().stream().filter(f -> f.getFragmentPath().equals(absolutePath)).findAny(); + if (optionalExistingFragment.isPresent()) { + return optionalExistingFragment.get(); } + PatchingContext context = new PatchingContext(absolutePath.toString(), "---", descriptorYaml.getPatchingContext().getMapper()); + AutomationPackageFragmentYaml fragment = new AutomationPackageFragmentYamlImpl(context); + fragments.add(fragment); + fragment.setFragmentPath(absolutePath); - if (mode == NewObjectFragmentMode.PER_OBJECT && !p.hasAttribute(AbstractOrganizableObject.NAME)) { - throw new AutomationPackagePerObjectSaveUnsupportedException(String.format(""" - Saving by object name is unsupported for %1$s, please configure the entity to be stored in a specified single fragment, i.e. - %2$s = %1$s.yml - %3$s = %4$s - """, fieldName, String.format(PROPERTY_NEW_OBJECT_FRAGMENT_PATH, fieldName), String.format(PROPERTY_NEW_OBJECT_FRAGMENT_MODE, fieldName), NewObjectFragmentMode.FRAGMENT.name())); + Optional optionalReferencingFragment = determineReferencingFragment(path); + if (optionalReferencingFragment.isEmpty()) { + Path referencePath = determineObjectRelativePath(p, fieldName, true); + descriptorYaml.getFragments().add(new PatchableYamlPrimitive<>(descriptorYaml.getPatchingContext(), referencePath.toString())); + descriptorYaml.writeToDisk(); } + return fragment; + } - String relativeFragmentPath = properties.getProperty(String.format(PROPERTY_NEW_OBJECT_FRAGMENT_PATH, fieldName), defaultRelativeFragmentPath); - Path path = new File(relativeFragmentPath).toPath(); - if (!path.isAbsolute()) { - path = apRoot.resolve(path); + private Optional determineReferencingFragment(Path path) { + for (AutomationPackageFragmentYaml fragment : fragments) { + for (PatchableYamlPrimitive fragmentPathPattern : fragment.getFragments()) { + PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + fragmentPathPattern.getValue()); + if (matcher.matches(path)) { + return Optional.of(fragment); + } + } } + return Optional.empty(); + } - if (mode == NewObjectFragmentMode.PER_OBJECT) { - path = path.resolve(sanitizeFilename(p.getAttribute(AbstractOrganizableObject.NAME)) + ".yml"); + public Path determineObjectRelativePath(PatchableYamlModel p, String fieldName, boolean globPattern) { + + NewObjectFragmentMode mode = NewObjectFragmentMode.valueOf(properties.getProperty(String.format(PROPERTY_NEW_OBJECT_FRAGMENT_MODE, fieldName), NewObjectFragmentMode.PER_OBJECT.name())); + String defaultRelativeFragmentPath = fieldName; + + if (mode == NewObjectFragmentMode.FRAGMENT) { + defaultRelativeFragmentPath = defaultRelativeFragmentPath + ".yml"; } - try { - URL url = path.toUri().toURL(); + String relativeFragmentPath = properties.getProperty(String.format(PROPERTY_NEW_OBJECT_FRAGMENT_PATH, fieldName), defaultRelativeFragmentPath); + + return switch (mode) { + case NewObjectFragmentMode.FRAGMENT -> new File(relativeFragmentPath).toPath(); + case NewObjectFragmentMode.PER_OBJECT -> { + if (p instanceof NamedObjectPatchableYamlModel namedObjectPatchableYamlModel) { + String name = globPattern ? "*" : namedObjectPatchableYamlModel.getName(); + yield new File(relativeFragmentPath).toPath().resolve(sanitizeFilename(name + ".yml")); + } + throw new AutomationPackagePerObjectSaveUnsupportedException(String.format(""" + Saving by object name is unsupported for %1$s, please configure the entity to be stored in a specified single fragment, i.e. - if (pathToYamlFragment.containsKey(url.toString())) { - return pathToYamlFragment.get(url.toString()); + %2$s = %1$s.yml + %3$s = %4$s + """, fieldName, String.format(PROPERTY_NEW_OBJECT_FRAGMENT_PATH, fieldName), String.format(PROPERTY_NEW_OBJECT_FRAGMENT_MODE, fieldName), NewObjectFragmentMode.FRAGMENT.name())); } - PatchingContext context = new PatchingContext(url.toString(), "---", descriptorYaml.getPatchingContext().getMapper()); - AutomationPackageFragmentYaml fragment = new AutomationPackageFragmentYamlImpl(context); - fragment.setFragmentUrl(url); - return fragment; - } catch (MalformedURLException e) { - throw new AutomationPackageUpdateException(MessageFormat.format("Error creating path for new fragment: {0}", path), e); - } - + }; } public String sanitizeFilename(String inputName) { @@ -315,27 +358,21 @@ public void removeAdditionalFieldObject(B fragment.writeToDisk(); } - public synchronized BO saveAdditionalFieldObject(BO object, java.util.function.Function newYamlObjectCreator, String fieldName) { - AutomationPackageFragmentYaml fragment; - if (fragmentMap.get(object) == null) { - fragment = fragmentForNewObject(object, fieldName); - - YO newYamlObject = newYamlObjectCreator.apply(fragment.getPatchingContext()); - PatchableYamlList list = (PatchableYamlList) fragment.getAdditionalFields() - .computeIfAbsent(fieldName, k -> new PatchableYamlList(fragment.getPatchingContext(), fieldName)); - fragmentMap.put(object, fragment); - pathToYamlFragment.put(fragment.getFragmentUrl().toString(), fragment); - addFragmentEntity(fragment, list, newYamlObject); - patchableMap.put(object, newYamlObject); - } else { - fragment = fragmentMap.get(object); - YO newYamlObject = newYamlObjectCreator.apply(fragment.getPatchingContext()); - PatchableYamlList list = (PatchableYamlList) fragment.getAdditionalFields() - .computeIfAbsent(fieldName, k -> new PatchableYamlList(fragment.getPatchingContext(), fieldName)); + public synchronized BO saveAdditionalFieldObject(BO object, YO yamlObject, String fieldName) { + final AutomationPackageFragmentYaml fragment = fragmentMap.computeIfAbsent(object, o -> fragmentForNewObject(yamlObject, fieldName)); + yamlObject.setPatchingContext(fragment.getPatchingContext()); + PatchableYamlList list = (PatchableYamlList) fragment.getAdditionalFields() + .computeIfAbsent(fieldName, f -> new PatchableYamlList(fragment.getPatchingContext(), fieldName)); + + + if (patchableMap.get(object) == null) { + addFragmentEntity(fragment, list, yamlObject); + patchableMap.put(object, yamlObject); + } else { YO oldYamlObject = (YO) patchableMap.get(object); - modifyFragmentEntity(fragment, list, oldYamlObject, newYamlObject); - patchableMap.put(object, newYamlObject); + modifyFragmentEntity(fragment, list, oldYamlObject, yamlObject, fieldName); + patchableMap.put(object, yamlObject); } return object; } diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/AbstractYamlAutomationPackageFragmentDeserializer.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/AbstractYamlAutomationPackageFragmentDeserializer.java index 815cfaaa2f..4a92941594 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/AbstractYamlAutomationPackageFragmentDeserializer.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/deserialization/AbstractYamlAutomationPackageFragmentDeserializer.java @@ -27,25 +27,19 @@ import com.fasterxml.jackson.databind.deser.ResolvableDeserializer; import step.automation.packages.deserialization.AutomationPackageSerializationRegistry; import step.automation.packages.yaml.model.AbstractAutomationPackageFragmentYaml; -import step.automation.packages.yaml.model.AutomationPackageFragmentYamlImpl; -import step.core.yaml.PatchingContext; import step.core.yaml.deserialization.PatchableYamlList; import java.io.IOException; public abstract class AbstractYamlAutomationPackageFragmentDeserializer extends BeanDeserializer implements ContextualDeserializer, ResolvableDeserializer { - private final BeanDeserializer delegate; public AbstractYamlAutomationPackageFragmentDeserializer(BeanDeserializer deserializer) { super(deserializer); - delegate = deserializer; } @Override - public Object deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return deserialize(p, ctxt, new AutomationPackageFragmentYamlImpl((PatchingContext) ctxt.getAttribute(PatchingContext.class))); - } + abstract public Object deserialize(JsonParser p, DeserializationContext ctxt) throws IOException; @Override protected void handleUnknownVanilla(JsonParser p, DeserializationContext ctxt, Object intoValue, String propName) throws IOException { diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java index c4891c8311..031fe5f7b3 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java @@ -18,34 +18,29 @@ ******************************************************************************/ package step.automation.packages.yaml.model; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonSetter; -import com.fasterxml.jackson.annotation.Nulls; +import com.fasterxml.jackson.annotation.*; import org.apache.commons.io.FileUtils; import step.automation.packages.model.YamlAutomationPackageKeyword; import step.automation.packages.yaml.AutomationPackageWriteToDiskException; import step.core.yaml.PatchingContext; import step.core.yaml.deserialization.AutomationPackageConcurrentEditException; import step.core.yaml.deserialization.PatchableYamlList; +import step.core.yaml.deserialization.PatchableYamlPrimitive; import step.plans.automation.YamlPlainTextPlan; import step.plans.parser.yaml.YamlPlan; import java.io.File; import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.text.MessageFormat; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @JsonInclude(JsonInclude.Include.NON_EMPTY) public abstract class AbstractAutomationPackageFragmentYaml implements AutomationPackageFragmentYaml { - private List fragments = new ArrayList<>(); + private PatchableYamlList> fragments; private PatchableYamlList keywords; private PatchableYamlList plans; private PatchableYamlList plansPlainText; @@ -59,10 +54,11 @@ public AbstractAutomationPackageFragmentYaml(PatchingContext patchingContext) { plans = new PatchableYamlList<>(patchingContext, YamlPlan.PLANS_ENTITY_NAME); keywords = new PatchableYamlList<>(patchingContext, YamlAutomationPackageKeyword.KEYWORDS_ENTITY_NAME); plansPlainText = new PatchableYamlList<>(patchingContext, "plansPlainText"); + fragments = new PatchableYamlList<>(patchingContext, "fragments"); } @JsonIgnore - private URL url; + private Path path; @Override public PatchableYamlList getKeywords() { @@ -85,12 +81,12 @@ public void setPlans(PatchableYamlList plans) { } @Override - public List getFragments() { + public PatchableYamlList> getFragments() { return fragments; } @JsonSetter(nulls = Nulls.AS_EMPTY) - public void setFragments(List fragments) { + public void setFragments(PatchableYamlList> fragments) { this.fragments = fragments; } @@ -116,9 +112,9 @@ public void setPlansPlainText(PatchableYamlList plansPlainTex } @JsonIgnore - public void setFragmentUrl(URL url) { + public void setFragmentPath(Path path) { resetLastModified(); - this.url = url; + this.path = path; } private void resetLastModified() { @@ -126,8 +122,8 @@ private void resetLastModified() { } @JsonIgnore - public URL getFragmentUrl() { - return url; + public Path getFragmentPath() { + return path; } @JsonIgnore @@ -146,14 +142,14 @@ public PatchingContext getPatchingContext() { @Override public void writeToDisk() { try { - File file = new File(url.toURI()); + File file = path.toFile(); if (file.exists() && file.lastModified() > fileLastModified) { - throw new AutomationPackageConcurrentEditException(MessageFormat.format("Automation package fragment {0} was edited outside the editor.", url)); + throw new AutomationPackageConcurrentEditException(MessageFormat.format("Automation package fragment {0} was edited outside the editor.", path)); } FileUtils.writeStringToFile(file, context.getCurrentYaml(), StandardCharsets.UTF_8); resetLastModified(); - } catch (IOException | URISyntaxException e) { - throw new AutomationPackageWriteToDiskException(MessageFormat.format("Error when writing automation package fragment {0} back to disk.", url), e); + } catch (IOException e) { + throw new AutomationPackageWriteToDiskException(MessageFormat.format("Error when writing automation package fragment {0} back to disk.", path), e); } } } diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java index 7cf2be19b3..8f41bb4fc1 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java @@ -21,11 +21,12 @@ import step.automation.packages.model.YamlAutomationPackageKeyword; import step.core.yaml.PatchingContext; import step.core.yaml.deserialization.PatchableYamlList; +import step.core.yaml.deserialization.PatchableYamlPrimitive; import step.plans.automation.YamlPlainTextPlan; import step.plans.parser.yaml.YamlPlan; import java.io.IOException; -import java.net.URL; +import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -37,7 +38,7 @@ public interface AutomationPackageFragmentYaml { List getPlansPlainText(); - List getFragments(); + PatchableYamlList> getFragments(); Map> getAdditionalFields(); @@ -47,9 +48,9 @@ default PatchableYamlList getAdditionalField(String k) { void setAdditionalFields(String key, PatchableYamlList value) throws IOException; - URL getFragmentUrl(); + Path getFragmentPath(); - void setFragmentUrl(URL url); + void setFragmentPath(Path url); PatchingContext getPatchingContext(); diff --git a/step-automation-packages/step-automation-packages-yaml/src/test/java/step/automation/packages/yaml/AutomationPackageDescriptorReaderTest.java b/step-automation-packages/step-automation-packages-yaml/src/test/java/step/automation/packages/yaml/AutomationPackageDescriptorReaderTest.java index 22428f5468..c03d302bd2 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/test/java/step/automation/packages/yaml/AutomationPackageDescriptorReaderTest.java +++ b/step-automation-packages/step-automation-packages-yaml/src/test/java/step/automation/packages/yaml/AutomationPackageDescriptorReaderTest.java @@ -34,6 +34,7 @@ import java.io.InputStream; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import static org.junit.Assert.*; @@ -100,7 +101,7 @@ public void completeDescriptorReadTest() throws AutomationPackageReadingExceptio assertEquals("*/5 * * * *", firstTask.getCron()); assertEquals("TEST", firstTask.getExecutionParameters().get("environment")); - assertEquals(Arrays.asList("importPlans.yml", "importKeywords.yml"), descriptor.getFragments()); + assertEquals(Arrays.asList("importPlans.yml", "importKeywords.yml"), descriptor.getFragments().stream().map(f -> f.getValue()).collect(Collectors.toUnmodifiableList())); } catch (IOException e) { throw new RuntimeException(e); } @@ -119,4 +120,4 @@ public void emptyKeywordReadTest() throws AutomationPackageReadingException { } } -} \ No newline at end of file +} diff --git a/step-automation-packages/step-automation-packages-yaml/src/test/java/step/core/yaml/PatchingContextTest.java b/step-automation-packages/step-automation-packages-yaml/src/test/java/step/core/yaml/PatchingContextTest.java index d4279eddfb..60987b1ba4 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/test/java/step/core/yaml/PatchingContextTest.java +++ b/step-automation-packages/step-automation-packages-yaml/src/test/java/step/core/yaml/PatchingContextTest.java @@ -103,7 +103,6 @@ public class PatchingContextTest { schemaVersion: 1.0.0 name: "complete-package" fragments: - - "importPlans.yml" - "importKeywords.yml" keywords: - Composite: @@ -151,7 +150,6 @@ public class PatchingContextTest { schemaVersion: 1.0.0 name: "complete-package" fragments: - - "importPlans.yml" - "importKeywords.yml" keywords: [] plans: [] @@ -186,35 +184,35 @@ public void testLowLevelDeletionModification() throws Exception { .filter(m -> m instanceof PatchableYamlList) .map(x -> (PatchableYamlList) x) .toList(); - assertEquals(3, lists.size()); + assertEquals(4, lists.size()); // remove first keyword, second plan, second schedule lists.get(0).remove(0); - lists.get(1).remove(1); + lists.get(1).remove(0); lists.get(2).remove(1); + lists.get(3).remove(1); current = ensureValid(patchingContext.getCurrentYaml()); Assert.assertEquals(EXPECTED_REMOVED_1, current); // empty first two lists, modify schedule a little - AutomationPackageSchedule schedule1 = (AutomationPackageSchedule) lists.get(2).getFirst(); + AutomationPackageSchedule schedule1 = (AutomationPackageSchedule) lists.get(3).getFirst(); schedule1.setName("This is now the new schedule name which is rather long, really long in fact"); schedule1.setActive(false); schedule1.getExecutionParameters().clear(); // we have to tell the entity explicitly that it was modified schedule1.setModified(); - lists.subList(0, 2).forEach(ArrayList::removeFirst); + lists.subList(1, 3).forEach(ArrayList::removeFirst); current = ensureValid(patchingContext.getCurrentYaml()); Assert.assertEquals(EXPECTED_REMOVED_2_MODIFIED, current); - lists.get(2).removeFirst(); + lists.get(3).removeFirst(); current = ensureValid(patchingContext.getCurrentYaml()); Assert.assertEquals(""" schemaVersion: 1.0.0 name: "complete-package" fragments: - - "importPlans.yml" - "importKeywords.yml" keywords: [] plans: [] diff --git a/step-core-model/src/main/java/step/core/yaml/NamedObjectPatchableYamlModel.java b/step-core-model/src/main/java/step/core/yaml/NamedObjectPatchableYamlModel.java new file mode 100644 index 0000000000..3c2e741e38 --- /dev/null +++ b/step-core-model/src/main/java/step/core/yaml/NamedObjectPatchableYamlModel.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (C) 2026, exense GmbH + * + * This file is part of STEP + * + * STEP is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * STEP is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with STEP. If not, see . + ******************************************************************************/ +package step.core.yaml; + +public interface NamedObjectPatchableYamlModel { + + String getName(); +} diff --git a/step-core-model/src/main/java/step/core/yaml/PatchableYamlModelBase.java b/step-core-model/src/main/java/step/core/yaml/PatchableYamlModelBase.java index ce6f3bbd53..d11289b8ec 100644 --- a/step-core-model/src/main/java/step/core/yaml/PatchableYamlModelBase.java +++ b/step-core-model/src/main/java/step/core/yaml/PatchableYamlModelBase.java @@ -27,7 +27,7 @@ public class PatchableYamlModelBase extends AbstractYamlModel implements Patchab private static final Logger logger = LoggerFactory.getLogger(PatchableYamlModelBase.class); @JsonIgnore - private PatchingContext context; + protected PatchingContext context; public PatchableYamlModelBase(PatchingContext context) { this.context = context; diff --git a/step-core-model/src/main/java/step/core/yaml/PatchingContext.java b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java index ce0494dbd6..131bb1e2b3 100644 --- a/step-core-model/src/main/java/step/core/yaml/PatchingContext.java +++ b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java @@ -150,7 +150,7 @@ private List getClaimedOuterBounds() { private String serializeUnindented(Object entity) { try { return mapper.writeValueAsString(entity) - .replaceFirst("^---\\s*\\n", "") + .replaceFirst("^---\\s*\\n*", "") .trim(); } catch (JsonProcessingException e) { throw new AutomationPackageUpdateException("Error Serializing YAML object", e); diff --git a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlPrimitive.java b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlPrimitive.java new file mode 100644 index 0000000000..64aef98900 --- /dev/null +++ b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlPrimitive.java @@ -0,0 +1,54 @@ +package step.core.yaml.deserialization; + +import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.core.JsonLocation; +import step.core.yaml.PatchableYamlModelBase; +import step.core.yaml.PatchingContext; + +/******************************************************************************* + * Copyright (C) 2026, exense GmbH + * + * This file is part of STEP + * + * STEP is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * STEP is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with STEP. If not, see . + ******************************************************************************/ +public class PatchableYamlPrimitive extends PatchableYamlModelBase { + @JsonIgnore + private T value; + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public PatchableYamlPrimitive(@JacksonInject(useInput = OptBoolean.FALSE) PatchingContext context, T value) { + super(context); + this.value = value; + } + + @Override + public String toString() { + return value.toString(); + } + + @JsonValue + public T getValue() { + return value; + } + + public void setValue(T value) { + this.value = value; + } + + @Override + public void onParsed(JsonLocation startLocation, JsonLocation endLocation) { + context.claimChunk(startLocation, startLocation, this); + } +} diff --git a/step-core-model/src/main/java/step/parameter/automation/AutomationPackageParameter.java b/step-core-model/src/main/java/step/parameter/automation/AutomationPackageParameter.java index aecc711f00..40616b4dca 100644 --- a/step-core-model/src/main/java/step/parameter/automation/AutomationPackageParameter.java +++ b/step-core-model/src/main/java/step/parameter/automation/AutomationPackageParameter.java @@ -103,8 +103,8 @@ public String getScopeEntity() { return scopeEntity; } - public static AutomationPackageParameter forContext(PatchingContext context, Parameter parameter) { - AutomationPackageParameter yamlParameter = new AutomationPackageParameter(context); + public static AutomationPackageParameter fromParameter(Parameter parameter) { + AutomationPackageParameter yamlParameter = new AutomationPackageParameter(null); yamlParameter.copyFieldsFromObject(parameter, true); Expression expression = parameter.getActivationExpression(); if (expression == null) { diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/YamlPlan.java b/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/YamlPlan.java index 74ba1898bb..28429275ea 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/YamlPlan.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/plans/parser/yaml/YamlPlan.java @@ -27,6 +27,7 @@ import step.core.plans.agents.configuration.AgentProvisioningConfiguration; import step.core.plans.agents.configuration.AgentProvisioningConfigurationDeserializer; import step.core.plans.agents.configuration.AgentProvisioningConfigurationSerializer; +import step.core.yaml.NamedObjectPatchableYamlModel; import step.core.yaml.PatchableYamlModelBase; import step.core.yaml.PatchingContext; import step.core.yaml.model.NamedYamlArtefact; @@ -34,7 +35,7 @@ import java.util.List; @JsonInclude(JsonInclude.Include.NON_NULL) -public class YamlPlan extends PatchableYamlModelBase { +public class YamlPlan extends PatchableYamlModelBase implements NamedObjectPatchableYamlModel { public static final String PLANS_ENTITY_NAME = "plans"; @@ -53,6 +54,7 @@ public YamlPlan(@JacksonInject(useInput = OptBoolean.FALSE, optional = OptBoolea super(context); } + @Override public String getName() { return name; } From c8f9325a62b015352b2a3189f1301f3052006da8 Mon Sep 17 00:00:00 2001 From: Cyril Misev Date: Fri, 5 Jun 2026 14:23:39 +0200 Subject: [PATCH 04/10] SED-4672 address code assist review --- .../packages/AutomationPackageReader.java | 30 +++++++++---- .../AutomationPackageYamlFragmentManager.java | 42 +++++++++++-------- .../java/step/core/yaml/PatchingContext.java | 12 +++--- .../PatchableYamlPrimitive.java | 3 ++ .../ResourcePathMatchingResolver.java | 18 +++++++- 5 files changed, 72 insertions(+), 33 deletions(-) diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java index 6fcdd64b28..a3e2b23a81 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java @@ -45,6 +45,7 @@ import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.FileSystemNotFoundException; import java.nio.file.Path; import java.util.*; import java.util.stream.Collectors; @@ -184,12 +185,19 @@ public AutomationPackageYamlFragmentManager getAutomationPackageYamlFragmentMana URL descriptorUrl = archive.getDescriptorYamlUrl(); try (InputStream inputStream = descriptorUrl.openStream()) { AutomationPackageDescriptorYaml descriptor = reader.readAutomationPackageDescriptor(inputStream, archive.getOriginalFileName()); - descriptor.setFragmentPath(Path.of(descriptorUrl.getPath())); - AutomationPackageContent content = newContentInstance(); - Set fragments = new HashSet<>(); - fillAutomationPackageWithImportedFragments(content, descriptor, archive, fragments); - StagingAutomationPackageContext stagingContext = new StagingAutomationPackageContext(null, AutomationPackageOperationMode.LOCAL, new LocalResourceManagerImpl(Path.of(descriptorUrl.getPath()).getParent().toFile()), archive, content, null, null, new HashMap<>()); - return new AutomationPackageYamlFragmentManager(descriptor, fragments, getOrCreateDescriptorReader(), stagingContext); + + try { + Path descriptorPath = Path.of(descriptorUrl.toURI()); + descriptor.setFragmentPath(descriptorPath); + AutomationPackageContent content = newContentInstance(); + Set fragments = new HashSet<>(); + fillAutomationPackageWithImportedFragments(content, descriptor, archive, fragments); + StagingAutomationPackageContext stagingContext = new StagingAutomationPackageContext(null, AutomationPackageOperationMode.LOCAL, new LocalResourceManagerImpl(descriptorPath.getParent().toFile()), archive, content, null, null, new HashMap<>()); + return new AutomationPackageYamlFragmentManager(archive.getResourcePathMatchingResolver(), descriptor, fragments, getOrCreateDescriptorReader(), stagingContext); + } catch (FileSystemNotFoundException | URISyntaxException e) { + throw new AutomationPackageReadingException("Failed to read automation package for editing. The most likely cause is that you were trying to load " + + "an automation package as a packaged jar. This is not supported and expected behaviour", e); + } } catch (IOException e) { throw new AutomationPackageReadingException("Failed to read automation package for editing", e); } @@ -205,9 +213,15 @@ private void fillAutomationPackageWithImportedFragments(AutomationPackageContent try (InputStream fragmentYamlStream = resource.openStream()) { AutomationPackageFragmentYaml referencedFragment = getOrCreateDescriptorReader().readAutomationPackageFragment(fragmentYamlStream, resource.toString(), archive.getAutomationPackageName()); fragments.add(referencedFragment); - referencedFragment.setFragmentPath(Path.of(resource.getPath())); + try { + referencedFragment.setFragmentPath(Path.of(resource.toURI())); + } catch (FileSystemNotFoundException e) { + logger.warn("Could not set Fragment path for fragment editing while loading fragment. " + + "This is likely due to loading the automation package as a jar and not as a file system folder. This is expected behaviour"); + } + fillAutomationPackageWithImportedFragments(targetPackage, referencedFragment, archive, fragments); - } catch (IOException e) { + } catch (IOException | URISyntaxException e) { throw new AutomationPackageReadingException("Unable to read fragment in automation package: " + importedFragmentReference, e); } } diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java index ff5472dfa8..5c2ca121a0 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java @@ -19,6 +19,7 @@ package step.automation.packages.yaml; import org.apache.commons.io.FileUtils; +import step.automation.packages.ResourcePathMatchingResolver; import step.automation.packages.StagingAutomationPackageContext; import step.automation.packages.model.YamlAutomationPackageKeyword; import step.automation.packages.yaml.model.AutomationPackageDescriptorYaml; @@ -30,6 +31,7 @@ import step.core.yaml.PatchableYamlModel; import step.core.yaml.PatchableYamlModelBase; import step.core.yaml.PatchingContext; +import step.core.yaml.deserialization.AutomationPackageConcurrentEditException; import step.core.yaml.deserialization.AutomationPackagePerObjectSaveUnsupportedException; import step.core.yaml.deserialization.PatchableYamlList; import step.core.yaml.deserialization.PatchableYamlPrimitive; @@ -45,10 +47,11 @@ import java.io.InputStream; import java.net.URLEncoder; import java.nio.charset.Charset; -import java.nio.file.FileSystems; import java.nio.file.Path; -import java.nio.file.PathMatcher; -import java.util.*; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -57,6 +60,7 @@ public class AutomationPackageYamlFragmentManager { protected final Path apRoot; protected final StagingAutomationPackageContext stagingContext; + private final ResourcePathMatchingResolver resourcePatchMatchingResolver; public enum NewObjectFragmentMode { /** @@ -80,8 +84,8 @@ public enum NewObjectFragmentMode { protected Properties properties = new Properties(); public final AutomationPackageFragmentYaml descriptorYaml; - public AutomationPackageYamlFragmentManager(AutomationPackageDescriptorYaml descriptorYaml, Set fragments, AutomationPackageDescriptorReader descriptorReader, StagingAutomationPackageContext stagingContext) { - + public AutomationPackageYamlFragmentManager(ResourcePathMatchingResolver resourcePathMatchingResolver, AutomationPackageDescriptorYaml descriptorYaml, Set fragments, AutomationPackageDescriptorReader descriptorReader, StagingAutomationPackageContext stagingContext) { + this.resourcePatchMatchingResolver = resourcePathMatchingResolver; this.descriptorReader = descriptorReader; this.descriptorYaml = descriptorYaml; @@ -228,21 +232,24 @@ private void modifyFragmentEntity(AutomationPacka if (absoluteOldPath.equals(fragment.getFragmentPath())) { Path absoluteNewPath = apRoot.resolve(newRelativePath); + try { FileUtils.moveFile(absoluteOldPath.toFile(), absoluteNewPath.toFile()); fragment.setFragmentPath(absoluteNewPath); + } catch (IOException e) { + throw new AutomationPackageConcurrentEditException( + String.format("Unable to rename file %s to file %s. Was the file renamed or deleted outside the editor?", absoluteOldPath, absoluteNewPath)); + } - AutomationPackageFragmentYaml referencingFragment = determineReferencingFragment(oldRelativePath) - .orElse(descriptorYaml); + AutomationPackageFragmentYaml referencingFragment = determineReferencingFragment(oldRelativePath) + .orElse(descriptorYaml); - Path referencePath = determineObjectRelativePath(newEntity, fieldName, true); - if (referencingFragment.getFragments().removeIf(f -> f.getValue().equals(oldRelativePath.toString()))) { - referencingFragment.getFragments().add(new PatchableYamlPrimitive<>(referencingFragment.getPatchingContext(), referencePath.toString())); - referencingFragment.writeToDisk(); - }; - } catch (IOException ignored) { - } + Path referencePath = determineObjectRelativePath(newEntity, fieldName, true); + if (referencingFragment.getFragments().removeIf(f -> f.getValue().equals(resourcePatchMatchingResolver.getFragmentReferenceString(oldRelativePath)))) { + referencingFragment.getFragments().add(new PatchableYamlPrimitive<>(referencingFragment.getPatchingContext(), resourcePatchMatchingResolver.getFragmentReferenceString(referencePath))); + referencingFragment.writeToDisk(); + }; } } fragment.writeToDisk(); @@ -267,8 +274,8 @@ private AutomationPackageFragmentYaml fragmentForNewObject(PatchableYamlModel p, Optional optionalReferencingFragment = determineReferencingFragment(path); if (optionalReferencingFragment.isEmpty()) { - Path referencePath = determineObjectRelativePath(p, fieldName, true); - descriptorYaml.getFragments().add(new PatchableYamlPrimitive<>(descriptorYaml.getPatchingContext(), referencePath.toString())); + String referencingPath = resourcePatchMatchingResolver.getFragmentReferenceString(determineObjectRelativePath(p, fieldName, true)); + descriptorYaml.getFragments().add(new PatchableYamlPrimitive<>(descriptorYaml.getPatchingContext(), referencingPath)); descriptorYaml.writeToDisk(); } return fragment; @@ -277,8 +284,7 @@ private AutomationPackageFragmentYaml fragmentForNewObject(PatchableYamlModel p, private Optional determineReferencingFragment(Path path) { for (AutomationPackageFragmentYaml fragment : fragments) { for (PatchableYamlPrimitive fragmentPathPattern : fragment.getFragments()) { - PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + fragmentPathPattern.getValue()); - if (matcher.matches(path)) { + if (resourcePatchMatchingResolver.isMatchingPath(fragmentPathPattern.getValue(), path)) { return Optional.of(fragment); } } diff --git a/step-core-model/src/main/java/step/core/yaml/PatchingContext.java b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java index 131bb1e2b3..e55fab299b 100644 --- a/step-core-model/src/main/java/step/core/yaml/PatchingContext.java +++ b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java @@ -7,11 +7,8 @@ import org.slf4j.LoggerFactory; import step.core.yaml.deserialization.AutomationPackageUpdateException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.nio.file.Path; +import java.util.*; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -32,13 +29,16 @@ public PatchingContext() { } public PatchingContext(ObjectMapper mapper) { - this(null, "", mapper); + this("", "", mapper); } public PatchingContext(String sourceLocation, String yaml, ObjectMapper mapper) { this.sourceLocation = sourceLocation; this.initialLines = new CopyOnWriteArrayList<>(yaml.lines().toList()); this.mapper = mapper; + Objects.requireNonNull(sourceLocation); + Objects.requireNonNull(mapper); + Objects.requireNonNull(initialLines); } public ObjectMapper getMapper() { diff --git a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlPrimitive.java b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlPrimitive.java index 64aef98900..8b2c60719c 100644 --- a/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlPrimitive.java +++ b/step-core-model/src/main/java/step/core/yaml/deserialization/PatchableYamlPrimitive.java @@ -5,6 +5,8 @@ import step.core.yaml.PatchableYamlModelBase; import step.core.yaml.PatchingContext; +import java.util.Objects; + /******************************************************************************* * Copyright (C) 2026, exense GmbH * @@ -31,6 +33,7 @@ public class PatchableYamlPrimitive extends PatchableYamlModelBase { public PatchableYamlPrimitive(@JacksonInject(useInput = OptBoolean.FALSE) PatchingContext context, T value) { super(context); this.value = value; + Objects.requireNonNull(value); } @Override diff --git a/step-core/src/main/java/step/automation/packages/ResourcePathMatchingResolver.java b/step-core/src/main/java/step/automation/packages/ResourcePathMatchingResolver.java index 19553ff896..79bc36a272 100644 --- a/step-core/src/main/java/step/automation/packages/ResourcePathMatchingResolver.java +++ b/step-core/src/main/java/step/automation/packages/ResourcePathMatchingResolver.java @@ -18,10 +18,15 @@ ******************************************************************************/ package step.automation.packages; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; @@ -88,7 +93,7 @@ private void findPathMatchingResourcesRecursive(String[] pathArray, int currentL if (nextLevel < pathArray.length) { String nextPath = pathArray[nextLevel]; List urls = ClassLoaderResourceFilesystem.listDirectory(currentPath); - Pattern pattern = Pattern.compile(nextPath.replaceAll("\\*", ".*")); + Pattern pattern = prepareMatchPattern(nextPath); for (URL url : urls) { String file = url.getFile(); if (file.endsWith(getPathSeparator())) { @@ -109,4 +114,15 @@ private void findPathMatchingResourcesRecursive(String[] pathArray, int currentL } } + private Pattern prepareMatchPattern(String referencingString) { + return Pattern.compile(referencingString.replace("*", ".*")); + } + + public boolean isMatchingPath(String referenceString, Path path) { + return prepareMatchPattern(referenceString).matcher(getFragmentReferenceString(path)).matches(); + } + + public String getFragmentReferenceString(Path path) { + return path.toString().replace(File.pathSeparator, getPathSeparator()); + } } From 88ffe453860a6f12cf146d81cb855014d81abc12 Mon Sep 17 00:00:00 2001 From: Cyril Misev Date: Fri, 5 Jun 2026 15:17:13 +0200 Subject: [PATCH 05/10] SED-4672 remove fragments if completely empty --- ...utomationPackageFragmentReferenceTest.java | 46 +++++++++++-- .../AutomationPackageYamlFragmentManager.java | 67 +++++++++++-------- ...AbstractAutomationPackageFragmentYaml.java | 9 +++ .../model/AutomationPackageFragmentYaml.java | 2 + 4 files changed, 90 insertions(+), 34 deletions(-) diff --git a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java index 795cb6b387..29e49f116f 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java +++ b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java @@ -55,7 +55,7 @@ public void setUp() throws IOException, AutomationPackageReadingException { @Test - public void testAddPlanToNewFragmentAndRename() throws IOException { + public void testAddPlanToNewFragmentAndRenameAndRemove() throws IOException { Sequence sequence = new Sequence(); Echo echo = new Echo(); @@ -71,14 +71,50 @@ public void testAddPlanToNewFragmentAndRename() throws IOException { planCollection.save(plan); - assertFilesEqual(expectedFilesPath.resolve("Hello World Plan.yml"), destinationDirectory.toPath().resolve("newPlansPath").resolve("Hello World Plan.yml")); - assertFilesEqual(expectedFilesPath.resolve("descriptorAfterNewFragmentReference.yml"), destinationDirectory.toPath().resolve("automation-package.yml")); + assertFilesEqual( + expectedFilesPath + .resolve("Hello World Plan.yml"), + destinationDirectory.toPath() + .resolve("newPlansPath") + .resolve("Hello World Plan.yml") + ); + assertFilesEqual( + expectedFilesPath + .resolve("descriptorAfterNewFragmentReference.yml"), + destinationDirectory.toPath() + .resolve("automation-package.yml") + ); attributes.put(AbstractArtefact.NAME, "This Plan was renamed"); planCollection.save(plan); - assertFilesEqual(expectedFilesPath.resolve("This Plan was renamed.yml"), destinationDirectory.toPath().resolve("newPlansPath").resolve("This Plan was renamed.yml")); - assertFilesEqual(expectedFilesPath.resolve("descriptorAfterNewFragmentReference.yml"), destinationDirectory.toPath().resolve("automation-package.yml")); + assertFilesEqual( + expectedFilesPath + .resolve("This Plan was renamed.yml"), + destinationDirectory.toPath() + .resolve("newPlansPath") + .resolve("This Plan was renamed.yml") + ); + + assertFilesEqual( + expectedFilesPath + .resolve("descriptorAfterNewFragmentReference.yml"), + destinationDirectory.toPath() + .resolve("automation-package.yml")); + + planCollection.remove(Filters.equals("attributes.name", "This Plan was renamed")); + + assertFalse(Files.exists( + destinationDirectory.toPath() + .resolve("newPlansPath") + .resolve("This Plan was renamed.yml") + )); + + assertFilesEqual( + expectedFilesPath + .resolve("descriptorAfterNewFragmentReference.yml"), + destinationDirectory.toPath() + .resolve("automation-package.yml")); } } diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java index 5c2ca121a0..fdc78b23fb 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java @@ -244,10 +244,11 @@ private void modifyFragmentEntity(AutomationPacka AutomationPackageFragmentYaml referencingFragment = determineReferencingFragment(oldRelativePath) .orElse(descriptorYaml); + String newReference = resourcePatchMatchingResolver.getFragmentReferenceString(determineObjectRelativePath(newEntity, fieldName, true)); + String oldReference = resourcePatchMatchingResolver.getFragmentReferenceString(oldRelativePath); - Path referencePath = determineObjectRelativePath(newEntity, fieldName, true); - if (referencingFragment.getFragments().removeIf(f -> f.getValue().equals(resourcePatchMatchingResolver.getFragmentReferenceString(oldRelativePath)))) { - referencingFragment.getFragments().add(new PatchableYamlPrimitive<>(referencingFragment.getPatchingContext(), resourcePatchMatchingResolver.getFragmentReferenceString(referencePath))); + if (referencingFragment.getFragments().removeIf(f -> f.getValue().equals(oldReference))) { + referencingFragment.getFragments().add(new PatchableYamlPrimitive<>(referencingFragment.getPatchingContext(), newReference)); referencingFragment.writeToDisk(); }; } @@ -255,6 +256,37 @@ private void modifyFragmentEntity(AutomationPacka fragment.writeToDisk(); } + private void removeFragmentEntity(AutomationPackageFragmentYaml fragment, PatchableYamlList entityList, BO object) { + + PatchableYamlModel yamlObject = patchableMap.get(object); + entityList.remove(yamlObject); + + patchableMap.remove(object); + fragmentMap.remove(object); + + if (fragment.isEmpty()) { + try { + FileUtils.delete(fragment.getFragmentPath().toFile()); + + Optional optionalReferencingFragment = determineReferencingFragment(fragment.getFragmentPath()); + + if (optionalReferencingFragment.isPresent()) { + AutomationPackageFragmentYaml referencingFragment = optionalReferencingFragment.get(); + String relativeFragmentReference = resourcePatchMatchingResolver.getFragmentReferenceString(apRoot.relativize(fragment.getFragmentPath())); + if (referencingFragment.getFragments().removeIf(f -> f.getValue().equals(relativeFragmentReference))) { + referencingFragment.writeToDisk(); + }; + + fragments.remove(fragment); + } + } catch (IOException e) { + throw new AutomationPackageConcurrentEditException(String.format("%s was removed outside the editor", fragment.getFragmentPath())); + } + } else { + fragment.writeToDisk(); + } + } + private AutomationPackageFragmentYaml fragmentForNewObject(PatchableYamlModel p, String fieldName) { Path path = determineObjectRelativePath(p, fieldName, false); @@ -327,41 +359,18 @@ public String sanitizeFilename(String inputName) { public void removePlan(Plan plan) { AutomationPackageFragmentYaml fragment = fragmentMap.get(plan); - YamlPlan yamlPlan = (YamlPlan) patchableMap.get(plan); - - fragment.getPlans().remove(yamlPlan); - - patchableMap.remove(plan); - fragmentMap.remove(plan); - - fragment.writeToDisk(); + removeFragmentEntity(fragment, fragment.getPlans(), plan); } public void removeFunction(step.functions.Function function) { AutomationPackageFragmentYaml fragment = fragmentMap.get(function); - YamlAutomationPackageKeyword yamlKeyword = (YamlAutomationPackageKeyword) patchableMap.get(function); - - fragment.getKeywords().remove(yamlKeyword); - - patchableMap.remove(function); - fragmentMap.remove(function); - - fragment.writeToDisk(); + removeFragmentEntity(fragment, fragment.getKeywords(), function); } public void removeAdditionalFieldObject(BO object, String fieldName) { - AutomationPackageFragmentYaml fragment = fragmentMap.get(object); - PatchableYamlModel yamlObject = patchableMap.get(object); - - fragment.getAdditionalField(fieldName) - .remove(yamlObject); - - patchableMap.remove(object); - fragmentMap.remove(object); - - fragment.writeToDisk(); + removeFragmentEntity(fragment, fragment.getAdditionalField(fieldName), object); } public synchronized BO saveAdditionalFieldObject(BO object, YO yamlObject, String fieldName) { diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java index 031fe5f7b3..f715264730 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AbstractAutomationPackageFragmentYaml.java @@ -152,4 +152,13 @@ public void writeToDisk() { throw new AutomationPackageWriteToDiskException(MessageFormat.format("Error when writing automation package fragment {0} back to disk.", path), e); } } + + @Override + public boolean isEmpty() { + return getFragments().isEmpty() && + getPlans().isEmpty() && + getPlansPlainText().isEmpty() && + getKeywords().isEmpty() && + getAdditionalFields().isEmpty(); + } } diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java index 8f41bb4fc1..8721c6963f 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/model/AutomationPackageFragmentYaml.java @@ -57,4 +57,6 @@ default PatchableYamlList getAdditionalField(String k) { void setPatchingContext(PatchingContext context); void writeToDisk(); + + boolean isEmpty(); } From 20c96731ee577d64a967c54238c13d8498e6634f Mon Sep 17 00:00:00 2001 From: Cyril Misev Date: Fri, 5 Jun 2026 16:13:29 +0200 Subject: [PATCH 06/10] SED-4672 fix existing reference determination --- ...utomationPackageFragmentReferenceTest.java | 67 +++++++++++++++++++ .../AutomationPackageYamlFragmentManager.java | 4 +- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java index 29e49f116f..1d80cd5067 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java +++ b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java @@ -117,4 +117,71 @@ public void testAddPlanToNewFragmentAndRenameAndRemove() throws IOException { destinationDirectory.toPath() .resolve("automation-package.yml")); } + + @Test + public void testAddTwoPlansToNewFragmentAndRemoveOne() throws IOException { + + Sequence sequence = new Sequence(); + Echo echo = new Echo(); + echo.setText(new DynamicValue<>("Hello World")); + sequence.addChild(echo); + + Plan plan = new Plan(sequence); + Map attributes = new HashMap<>(); + attributes.put(AbstractArtefact.NAME, "Hello World Plan"); + plan.setAttributes(attributes); + + setPropertiesWriteMode(YamlPlan.PLANS_ENTITY_NAME, "newPlansPath", AutomationPackageYamlFragmentManager.NewObjectFragmentMode.PER_OBJECT); + + planCollection.save(plan); + + assertFilesEqual( + expectedFilesPath + .resolve("Hello World Plan.yml"), + destinationDirectory.toPath() + .resolve("newPlansPath") + .resolve("Hello World Plan.yml") + ); + assertFilesEqual( + expectedFilesPath + .resolve("descriptorAfterNewFragmentReference.yml"), + destinationDirectory.toPath() + .resolve("automation-package.yml") + ); + + Plan plan2 = new Plan(sequence); + Map attributes2 = new HashMap<>(); + attributes2.put(AbstractArtefact.NAME, "This Plan was renamed"); + plan2.setAttributes(attributes2); + + planCollection.save(plan2); + + assertFilesEqual( + expectedFilesPath + .resolve("This Plan was renamed.yml"), + destinationDirectory.toPath() + .resolve("newPlansPath") + .resolve("This Plan was renamed.yml") + ); + + assertFilesEqual( + expectedFilesPath + .resolve("descriptorAfterNewFragmentReference.yml"), + destinationDirectory.toPath() + .resolve("automation-package.yml")); + + planCollection.remove(Filters.equals("attributes.name", "Hello World Plan")); + + assertFalse(Files.exists( + destinationDirectory.toPath() + .resolve("newPlansPath") + .resolve("Hello World Plan.yml") + )); + + assertFilesEqual( + expectedFilesPath + .resolve("descriptorAfterNewFragmentReference.yml"), + destinationDirectory.toPath() + .resolve("automation-package.yml")); + } } diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java index fdc78b23fb..cf0618c611 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java @@ -54,6 +54,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import java.util.stream.Stream; public class AutomationPackageYamlFragmentManager { @@ -314,7 +315,8 @@ private AutomationPackageFragmentYaml fragmentForNewObject(PatchableYamlModel p, } private Optional determineReferencingFragment(Path path) { - for (AutomationPackageFragmentYaml fragment : fragments) { + + for (AutomationPackageFragmentYaml fragment : Stream.concat(Stream.of(descriptorYaml), fragments.stream()).toList()) { for (PatchableYamlPrimitive fragmentPathPattern : fragment.getFragments()) { if (resourcePatchMatchingResolver.isMatchingPath(fragmentPathPattern.getValue(), path)) { return Optional.of(fragment); From 5a1ff8f10f61abe9667ef697a0e69f7c60b81ac5 Mon Sep 17 00:00:00 2001 From: Cyril Misev Date: Fri, 5 Jun 2026 17:17:08 +0200 Subject: [PATCH 07/10] SED-4672 fix cloning --- .../AutomationPackageFunctionCollection.java | 4 ++-- .../AutomationPackageParameterCollection.java | 4 ++-- .../collections/AutomationPackagePlanCollection.java | 4 ++-- .../java/step/core/dynamicbeans/DynamicValue.java | 11 +++++++++++ .../packages/model/AbstractYamlFunction.java | 2 +- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/step-automation-packages/step-automation-packages-collections/src/main/java/step/core/collections/AutomationPackageFunctionCollection.java b/step-automation-packages/step-automation-packages-collections/src/main/java/step/core/collections/AutomationPackageFunctionCollection.java index 813aad256b..aab8c3d53d 100644 --- a/step-automation-packages/step-automation-packages-collections/src/main/java/step/core/collections/AutomationPackageFunctionCollection.java +++ b/step-automation-packages/step-automation-packages-collections/src/main/java/step/core/collections/AutomationPackageFunctionCollection.java @@ -29,7 +29,7 @@ public class AutomationPackageFunctionCollection extends InMemoryCollection im private final AutomationPackageYamlFragmentManager fragmentManager; public AutomationPackagePlanCollection(AutomationPackageYamlFragmentManager fragmentManager) { - super(true, YamlPlan.PLANS_ENTITY_NAME); + super(false, YamlPlan.PLANS_ENTITY_NAME); this.fragmentManager = fragmentManager; initialzeRecordsFromFragments(fragmentManager); } @@ -41,7 +41,7 @@ private void initialzeRecordsFromFragments(AutomationPackageYamlFragmentManager @Override public Plan save(Plan p) { - return super.save(fragmentManager.savePlan(p)); + return fragmentManager.savePlan(super.save(p)); } @Override diff --git a/step-core-model/src/main/java/step/core/dynamicbeans/DynamicValue.java b/step-core-model/src/main/java/step/core/dynamicbeans/DynamicValue.java index a784ddcb96..9ae2693270 100644 --- a/step-core-model/src/main/java/step/core/dynamicbeans/DynamicValue.java +++ b/step-core-model/src/main/java/step/core/dynamicbeans/DynamicValue.java @@ -141,4 +141,15 @@ protected DynamicValue _cloneValue(DynamicValue clone) { protected boolean hasProtectedAccess() { return false; } + + @Override + public boolean equals(Object obj) { + if (obj == null) return false; + if (obj instanceof DynamicValue other) { + if (dynamic != other.dynamic) return false; + if (dynamic) return expression.equals(other.expression); + return value.equals(other.value); + } + return false; + } } diff --git a/step-core/src/main/java/step/automation/packages/model/AbstractYamlFunction.java b/step-core/src/main/java/step/automation/packages/model/AbstractYamlFunction.java index 451275e59e..43a8452275 100644 --- a/step-core/src/main/java/step/automation/packages/model/AbstractYamlFunction.java +++ b/step-core/src/main/java/step/automation/packages/model/AbstractYamlFunction.java @@ -20,11 +20,11 @@ import jakarta.json.JsonObject; import step.automation.packages.StagingAutomationPackageContext; -import step.core.yaml.YamlModelUtils; import step.core.accessors.AbstractOrganizableObject; import step.core.dynamicbeans.DynamicValue; import step.core.yaml.AbstractYamlModel; import step.core.yaml.YamlFieldCustomCopy; +import step.core.yaml.YamlModelUtils; import step.functions.Function; import step.jsonschema.JsonSchema; import step.jsonschema.JsonSchemaDefaultValueProvider; From f60d37bb97d75be738ee6898f256e0fa8af72a28 Mon Sep 17 00:00:00 2001 From: Cyril Misev Date: Fri, 5 Jun 2026 17:17:47 +0200 Subject: [PATCH 08/10] SED-4672 new default for plan creation --- step-ap-ide/src/main/java/step/ap_ide/StepUp.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/step-ap-ide/src/main/java/step/ap_ide/StepUp.java b/step-ap-ide/src/main/java/step/ap_ide/StepUp.java index 56778a88b0..709d01912d 100644 --- a/step-ap-ide/src/main/java/step/ap_ide/StepUp.java +++ b/step-ap-ide/src/main/java/step/ap_ide/StepUp.java @@ -39,7 +39,7 @@ public static void main(String[] args) throws Exception { configuration.getUnderlyingPropertyObject().load(propsStream); new ControllerServer(configuration).start(); initWorkdir(); - FXApp.main(args); // this will never return + //FXApp.main(args); // this will never return*/ } private static final JavaAutomationPackageReader READER; @@ -78,17 +78,17 @@ static void useAutomationPackageDirectory(File apDir) throws Exception { var fragmentManager = StepUp.READER.getAutomationPackageYamlFragmentManager(apDir); Properties properties = new Properties(); + int variant = 1; // variant 1: // parameters all go into parameters.yml, plans go into separate files in plans/$PLAN_NAME.yml - // Only works if the target files/directories already exist, so disabled for now - if (1 == 0) { + // this is because parameters do not have unique names by design. + if (variant == 1) { properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, Parameter.ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, Parameter.ENTITY_NAME), "parameters.yml"); - properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, YamlPlan.PLANS_ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.PER_OBJECT.name()); - // keywords seem to use PER_OBJECT by default? + // per default, PER_OBJECT is used on all objects? } // variant 2: simple, everything goes into main descriptor - if (1 == 1) { + if (variant == 2) { String mainFile = fragmentManager.descriptorYaml.getFragmentPath().toFile().getName(); properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_MODE, Parameter.ENTITY_NAME), AutomationPackageYamlFragmentManager.NewObjectFragmentMode.FRAGMENT.name()); properties.setProperty(String.format(AutomationPackageYamlFragmentManager.PROPERTY_NEW_OBJECT_FRAGMENT_PATH, Parameter.ENTITY_NAME), mainFile); From e65f2cf5d0c831f03cc3ca5f2cb05e4a57579d57 Mon Sep 17 00:00:00 2001 From: Cyril Misev Date: Fri, 12 Jun 2026 14:47:29 +0200 Subject: [PATCH 09/10] SED-4672 new unit tests for keyword adding and removal --- ...utomationPackageFragmentReferenceTest.java | 94 ++++++++++++++++--- ...utomationPackageKeywordCollectionTest.java | 14 +++ .../Hello World Composite Function.yml | 10 ++ .../expected/This Keyword was renamed.yml | 10 ++ ...iptorAfterNewKeywordsFragmentReference.yml | 26 +++++ ...criptorAfterNewPlansFragmentReference.yml} | 0 .../keywordsAfterCompositeRenamed.yml | 29 ++++++ .../AutomationPackageYamlFragmentManager.java | 8 +- .../packages/model/AbstractYamlFunction.java | 5 - .../model/YamlAutomationPackageKeyword.java | 10 +- .../automation/YamlCompositeFunction.java | 24 ----- 11 files changed, 185 insertions(+), 45 deletions(-) create mode 100644 step-automation-packages/step-automation-packages-collections/src/test/resources/expected/Hello World Composite Function.yml create mode 100644 step-automation-packages/step-automation-packages-collections/src/test/resources/expected/This Keyword was renamed.yml create mode 100644 step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewKeywordsFragmentReference.yml rename step-automation-packages/step-automation-packages-collections/src/test/resources/expected/{descriptorAfterNewFragmentReference.yml => descriptorAfterNewPlansFragmentReference.yml} (100%) create mode 100644 step-automation-packages/step-automation-packages-collections/src/test/resources/expected/keywordsAfterCompositeRenamed.yml diff --git a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java index 1d80cd5067..e70d95fa6a 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java +++ b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageFragmentReferenceTest.java @@ -23,23 +23,26 @@ import step.artefacts.Echo; import step.artefacts.Sequence; import step.automation.packages.AutomationPackageReadingException; +import step.automation.packages.model.YamlAutomationPackageKeyword; import step.automation.packages.yaml.AutomationPackageYamlFragmentManager; import step.core.artefacts.AbstractArtefact; import step.core.dynamicbeans.DynamicValue; import step.core.plans.Plan; -import step.core.yaml.deserialization.AutomationPackageConcurrentEditException; +import step.functions.Function; import step.plans.parser.yaml.YamlPlan; +import step.plugins.functions.types.CompositeFunction; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.StandardCopyOption; -import java.util.*; -import java.util.stream.Collectors; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; -import static org.junit.Assert.*; +import static org.junit.Assert.assertFalse; public class AutomationPackageFragmentReferenceTest extends AutomationPackageCollectionTestBase { + private Collection functionCollection; private Collection planCollection; public AutomationPackageFragmentReferenceTest() { @@ -51,9 +54,78 @@ public void setUp() throws IOException, AutomationPackageReadingException { super.setUp(); AutomationPackageCollectionFactory collectionFactory = new AutomationPackageCollectionFactory(new Properties(), fragmentManager); planCollection = collectionFactory.getCollection(YamlPlan.PLANS_ENTITY_NAME, Plan.class); + functionCollection = collectionFactory.getCollection(YamlAutomationPackageKeyword.KEYWORDS_ENTITY_NAME, Function.class); } + @Test + public void testAddCompositeKeywordToNewFragmentAndRenameAndRemove() throws IOException { + + Sequence sequence = new Sequence(); + Echo echo = new Echo(); + echo.setText(new DynamicValue<>("Hello World")); + sequence.addChild(echo); + + Plan plan = new Plan(sequence); + + CompositeFunction function = new CompositeFunction(); + function.setPlan(plan); + + Map attributes = new HashMap<>(); + attributes.put(AbstractArtefact.NAME, "Hello World Composite Function"); + function.setAttributes(attributes); + + setPropertiesWriteMode(YamlAutomationPackageKeyword.KEYWORDS_ENTITY_NAME, "newKeywordsPath", AutomationPackageYamlFragmentManager.NewObjectFragmentMode.PER_OBJECT); + + functionCollection.save(function); + + assertFilesEqual( + expectedFilesPath + .resolve("Hello World Composite Function.yml"), + destinationDirectory.toPath() + .resolve("newKeywordsPath") + .resolve("Hello World Composite Function.yml") + ); + assertFilesEqual( + expectedFilesPath + .resolve("descriptorAfterNewKeywordsFragmentReference.yml"), + destinationDirectory.toPath() + .resolve("automation-package.yml") + ); + + attributes.put(AbstractArtefact.NAME, "This Keyword was renamed"); + + functionCollection.save(function); + + assertFilesEqual( + expectedFilesPath + .resolve("This Keyword was renamed.yml"), + destinationDirectory.toPath() + .resolve("newKeywordsPath") + .resolve("This Keyword was renamed.yml") + ); + + assertFilesEqual( + expectedFilesPath + .resolve("descriptorAfterNewKeywordsFragmentReference.yml"), + destinationDirectory.toPath() + .resolve("automation-package.yml")); + + functionCollection.remove(Filters.equals("attributes.name", "This Keyword was renamed")); + + assertFalse(Files.exists( + destinationDirectory.toPath() + .resolve("newKeywordsPath") + .resolve("This Keyword was renamed.yml") + )); + + assertFilesEqual( + expectedFilesPath + .resolve("descriptorAfterNewKeywordsFragmentReference.yml"), + destinationDirectory.toPath() + .resolve("automation-package.yml")); + } + @Test public void testAddPlanToNewFragmentAndRenameAndRemove() throws IOException { @@ -80,7 +152,7 @@ public void testAddPlanToNewFragmentAndRenameAndRemove() throws IOException { ); assertFilesEqual( expectedFilesPath - .resolve("descriptorAfterNewFragmentReference.yml"), + .resolve("descriptorAfterNewPlansFragmentReference.yml"), destinationDirectory.toPath() .resolve("automation-package.yml") ); @@ -99,7 +171,7 @@ public void testAddPlanToNewFragmentAndRenameAndRemove() throws IOException { assertFilesEqual( expectedFilesPath - .resolve("descriptorAfterNewFragmentReference.yml"), + .resolve("descriptorAfterNewPlansFragmentReference.yml"), destinationDirectory.toPath() .resolve("automation-package.yml")); @@ -113,7 +185,7 @@ public void testAddPlanToNewFragmentAndRenameAndRemove() throws IOException { assertFilesEqual( expectedFilesPath - .resolve("descriptorAfterNewFragmentReference.yml"), + .resolve("descriptorAfterNewPlansFragmentReference.yml"), destinationDirectory.toPath() .resolve("automation-package.yml")); } @@ -144,7 +216,7 @@ public void testAddTwoPlansToNewFragmentAndRemoveOne() throws IOException { ); assertFilesEqual( expectedFilesPath - .resolve("descriptorAfterNewFragmentReference.yml"), + .resolve("descriptorAfterNewPlansFragmentReference.yml"), destinationDirectory.toPath() .resolve("automation-package.yml") ); @@ -166,7 +238,7 @@ public void testAddTwoPlansToNewFragmentAndRemoveOne() throws IOException { assertFilesEqual( expectedFilesPath - .resolve("descriptorAfterNewFragmentReference.yml"), + .resolve("descriptorAfterNewPlansFragmentReference.yml"), destinationDirectory.toPath() .resolve("automation-package.yml")); @@ -180,7 +252,7 @@ public void testAddTwoPlansToNewFragmentAndRemoveOne() throws IOException { assertFilesEqual( expectedFilesPath - .resolve("descriptorAfterNewFragmentReference.yml"), + .resolve("descriptorAfterNewPlansFragmentReference.yml"), destinationDirectory.toPath() .resolve("automation-package.yml")); } diff --git a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageKeywordCollectionTest.java b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageKeywordCollectionTest.java index 5a52ed56b5..e331f0e15a 100644 --- a/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageKeywordCollectionTest.java +++ b/step-automation-packages/step-automation-packages-collections/src/test/java/step/core/collections/AutomationPackageKeywordCollectionTest.java @@ -82,4 +82,18 @@ public void testModifyCompositeKeyword() throws IOException { assertFilesEqual(expectedFilesPath.resolve("keywordsAfterCompositeModification.yml"), destinationDirectory.toPath().resolve("keywords.yml")); } + @Test + public void testRenameCompositeKeyword() throws IOException { + Optional optionalFunction = functionCollection.find(Filters.equals("attributes.name", "Composite keyword from AP"), null, null, null, 100).findFirst(); + assertTrue(optionalFunction.isPresent()); + + CompositeFunction compositeFunction = (CompositeFunction) optionalFunction.get(); + + compositeFunction.getAttributes().put(AbstractOrganizableObject.NAME, "Renamed Composite Keyword"); + + functionCollection.save(compositeFunction); + + assertFilesEqual(expectedFilesPath.resolve("keywordsAfterCompositeRenamed.yml"), destinationDirectory.toPath().resolve("keywords.yml")); + } + } diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/Hello World Composite Function.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/Hello World Composite Function.yml new file mode 100644 index 0000000000..10a286e1a9 --- /dev/null +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/Hello World Composite Function.yml @@ -0,0 +1,10 @@ +--- +keywords: + - Composite: + name: "Hello World Composite Function" + plan: + root: + sequence: + children: + - echo: + text: "Hello World" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/This Keyword was renamed.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/This Keyword was renamed.yml new file mode 100644 index 0000000000..6216e9e111 --- /dev/null +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/This Keyword was renamed.yml @@ -0,0 +1,10 @@ +--- +keywords: + - Composite: + name: "This Keyword was renamed" + plan: + root: + sequence: + children: + - echo: + text: "Hello World" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewKeywordsFragmentReference.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewKeywordsFragmentReference.yml new file mode 100644 index 0000000000..b213e5a266 --- /dev/null +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewKeywordsFragmentReference.yml @@ -0,0 +1,26 @@ +schemaVersion: 1.0.0 +name: "My package" +alertingRules: + - name: "Rule1" + description: "My test alerting rule" + eventClass: ExecutionEndedEvent + conditions: + - BindingCondition: + description: "condition 1" + bindingKey: "myKey" + negate: false + predicate: + BindingValueEqualsPredicate: + value: "myValue" +plans: [] +parameters: + - key: "paramInMainAP" + value: "once" +fragments: + - "keywords.yml" + - "plans/*.yml" + - "schedules.yml" + - "parameters.yml" + - "parameters2.yml" + - "unknown.yml" + - "newKeywordsPath/*.yml" diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewFragmentReference.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewPlansFragmentReference.yml similarity index 100% rename from step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewFragmentReference.yml rename to step-automation-packages/step-automation-packages-collections/src/test/resources/expected/descriptorAfterNewPlansFragmentReference.yml diff --git a/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/keywordsAfterCompositeRenamed.yml b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/keywordsAfterCompositeRenamed.yml new file mode 100644 index 0000000000..e4ad5103a8 --- /dev/null +++ b/step-automation-packages/step-automation-packages-collections/src/test/resources/expected/keywordsAfterCompositeRenamed.yml @@ -0,0 +1,29 @@ +keywords: + - JMeter: + name: "JMeter keyword from automation package" + description: "JMeter keyword 1" + executeLocally: false + useCustomTemplate: true + callTimeout: 1000 + jmeterTestplan: "jmeterProject1/jmeterProject1.xml" + - Composite: + name: "Renamed Composite Keyword" + plan: + root: + testCase: + children: + - echo: + text: "Just echo" + - return: + output: + - output1: "value" + - output2: + expression: "'some thing dynamic'" + - GeneralScript: + name: "GeneralScript keyword from AP" + scriptLanguage: javascript + scriptFile: "jsProject/jsSample.js" + librariesFile: "lib/fakeLib.jar" + - Node: + name: "NodeAutomation" + jsfile: "nodeProject/nodeSample.ts" diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java index cf0618c611..7bd26d9d6f 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java @@ -164,8 +164,8 @@ public synchronized Plan savePlan(Plan plan) { public synchronized step.functions.Function saveFunction(step.functions.Function function) { AutomationPackageFragmentYaml fragment = fragmentMap.get(function); + YamlAutomationPackageKeyword newKeyword = createNewYamlKeyword(function); if (fragment == null) { - YamlAutomationPackageKeyword newKeyword = createNewYamlKeyword(function); fragment = fragmentForNewObject(newKeyword, YamlAutomationPackageKeyword.KEYWORDS_ENTITY_NAME); fragmentMap.put(function, fragment); if (newKeyword != null) { @@ -175,9 +175,9 @@ public synchronized step.functions.Function saveFunction(step.functions.Function System.err.println("SAVING OF FUNCTION OF TYPE " + function.getClass().getName() + " IS NOT CURRENTLY SUPPORTED"); } } else { - YamlAutomationPackageKeyword yamlKeyword = (YamlAutomationPackageKeyword) patchableMap.get(function); - yamlKeyword.getYamlKeyword().updateFromFunction(function); - modifyFragmentEntity(fragment, fragment.getKeywords(), yamlKeyword, yamlKeyword, YamlAutomationPackageKeyword.KEYWORDS_ENTITY_NAME); + YamlAutomationPackageKeyword oldKeyword = (YamlAutomationPackageKeyword) patchableMap.get(function); + modifyFragmentEntity(fragment, fragment.getKeywords(), oldKeyword, newKeyword, YamlAutomationPackageKeyword.KEYWORDS_ENTITY_NAME); + patchableMap.put(function, newKeyword); } return function; } diff --git a/step-core/src/main/java/step/automation/packages/model/AbstractYamlFunction.java b/step-core/src/main/java/step/automation/packages/model/AbstractYamlFunction.java index 43a8452275..97462ba1a6 100644 --- a/step-core/src/main/java/step/automation/packages/model/AbstractYamlFunction.java +++ b/step-core/src/main/java/step/automation/packages/model/AbstractYamlFunction.java @@ -120,11 +120,6 @@ public T applyAutomationPackageContext(StagingAutomationPackageContext context) return res; } - public void updateFromFunction(Function function) { - copyFieldsFromObject(function, false); - } - - public static class DefaultYamlFunctionNameProvider implements JsonSchemaDefaultValueProvider { public DefaultYamlFunctionNameProvider() { diff --git a/step-core/src/main/java/step/automation/packages/model/YamlAutomationPackageKeyword.java b/step-core/src/main/java/step/automation/packages/model/YamlAutomationPackageKeyword.java index e1e76a63c9..0c1af166d0 100644 --- a/step-core/src/main/java/step/automation/packages/model/YamlAutomationPackageKeyword.java +++ b/step-core/src/main/java/step/automation/packages/model/YamlAutomationPackageKeyword.java @@ -19,11 +19,13 @@ package step.automation.packages.model; import step.automation.packages.StagingAutomationPackageContext; +import step.core.yaml.NamedObjectPatchableYamlModel; import step.core.yaml.PatchableYamlModelBase; import step.core.yaml.PatchingContext; import step.functions.Function; -public class YamlAutomationPackageKeyword extends PatchableYamlModelBase implements AutomationPackageKeyword { +public class YamlAutomationPackageKeyword extends PatchableYamlModelBase + implements AutomationPackageKeyword, NamedObjectPatchableYamlModel { private AbstractYamlFunction yamlKeyword; @@ -34,6 +36,7 @@ public YamlAutomationPackageKeyword(AbstractYamlFunction yamlKeyword, Patchin } + public AbstractYamlFunction getYamlKeyword() { return yamlKeyword; } @@ -46,4 +49,9 @@ public void setYamlKeyword(AbstractYamlFunction yamlKeyword) { public Function prepareKeyword(StagingAutomationPackageContext context) { return yamlKeyword.applyAutomationPackageContext(context); } + + @Override + public String getName() { + return yamlKeyword.getName(); + } } diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/plugins/functions/types/automation/YamlCompositeFunction.java b/step-plans/step-plans-base-artefacts/src/main/java/step/plugins/functions/types/automation/YamlCompositeFunction.java index b58d97518d..f4ef80c2cc 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/plugins/functions/types/automation/YamlCompositeFunction.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/plugins/functions/types/automation/YamlCompositeFunction.java @@ -19,17 +19,13 @@ package step.plugins.functions.types.automation; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.ObjectMapper; import step.automation.packages.StagingAutomationPackageContext; import step.automation.packages.model.AbstractYamlFunction; import step.core.accessors.AbstractOrganizableObject; import step.core.plans.Plan; import step.core.yaml.YamlFieldCustomCopy; import step.core.yaml.YamlModel; -import step.core.yaml.model.AbstractYamlArtefact; -import step.core.yaml.model.NamedYamlArtefact; import step.core.yaml.schema.YamlJsonSchemaHelper; -import step.functions.Function; import step.jsonschema.JsonSchema; import step.plans.parser.yaml.YamlPlan; import step.plugins.functions.types.CompositeFunction; @@ -65,26 +61,6 @@ protected void fillDeclaredFields(CompositeFunction res, StagingAutomationPackag } } - @Override - public void updateFromFunction(Function function) { - copyFieldsFromObject(function, false); - - if (function instanceof CompositeFunction) { - Plan plan = ((CompositeFunction) function).getPlan(); - // plan name is optional, the composite function name is used by default - // FIXME: discuss what exactly this is supposed to do and how it relates to the comment above :-) - if (this.plan.getName() != null && !this.plan.getName().isEmpty()) { - this.plan.setName(plan.getAttribute(AbstractOrganizableObject.NAME)); - } - // I have no idea why that mapper would be null sometimes: - // ObjectMapper mapper = this.plan.getRoot().getYamlArtefact().getYamlObjectMapper(); - // That one should work... - ObjectMapper mapper = this.getPlan().getPatchingContext().getMapper(); - this.plan.setRoot(new NamedYamlArtefact(AbstractYamlArtefact.toYamlArtefact(plan.getRoot(), mapper))); - } - } - - private Plan yamlPlanToPlan(YamlPlan yamlPlan) { Plan plan = new Plan(yamlPlan.getRoot().getYamlArtefact().toArtefact()); From 1c320ae544f6c12338d6ae6b11ae50fdace04a95 Mon Sep 17 00:00:00 2001 From: Cyril Misev Date: Fri, 26 Jun 2026 08:09:17 +0200 Subject: [PATCH 10/10] SED-4672 review comments --- .../packages/AutomationPackageReader.java | 8 +++ .../AutomationPackageDescriptorReader.java | 1 + .../AutomationPackageYamlFragmentManager.java | 51 ++++++++++--------- .../step/core/dynamicbeans/DynamicValue.java | 11 ---- .../java/step/core/yaml/PatchingContext.java | 13 ++--- 5 files changed, 39 insertions(+), 45 deletions(-) diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java index a3e2b23a81..531b779cac 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageReader.java @@ -203,6 +203,14 @@ public AutomationPackageYamlFragmentManager getAutomationPackageYamlFragmentMana } } + /** + * + * @param targetPackage Target Automation package content to be filled by fragment read entities + * @param fragment Fragment to read + * @param archive Automation package archive + * @param fragments Set of all automation package fragments collected during recursive reading of fragments. + * @throws AutomationPackageReadingException Thrown upon errors when reading the fragment + */ private void fillAutomationPackageWithImportedFragments(AutomationPackageContent targetPackage, AutomationPackageFragmentYaml fragment, T archive, Set fragments) throws AutomationPackageReadingException { fillContentSections(targetPackage, fragment, archive); diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageDescriptorReader.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageDescriptorReader.java index 4e139641cc..3a9aa56898 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageDescriptorReader.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageDescriptorReader.java @@ -167,6 +167,7 @@ private ObjectMapper createYamlObjectMapper() { // Disable native type id to enable conversion to generic Documents yamlFactory.disable(YAMLGenerator.Feature.USE_NATIVE_TYPE_ID); yamlFactory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR); + yamlFactory.disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER); ObjectMapper yamlMapper = DefaultJacksonMapperProvider.getObjectMapper(yamlFactory); diff --git a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java index 7bd26d9d6f..eeea9e7f57 100644 --- a/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java +++ b/step-automation-packages/step-automation-packages-yaml/src/main/java/step/automation/packages/yaml/AutomationPackageYamlFragmentManager.java @@ -228,18 +228,29 @@ private void modifyFragmentEntity(AutomationPacka entityList.replaceItem(oldEntity, newEntity); Path oldRelativePath = determineObjectRelativePath(oldEntity, fieldName, false); Path newRelativePath = determineObjectRelativePath(newEntity, fieldName, false); + + // Path did not change - skip entire move logic if (!oldRelativePath.equals(newRelativePath)) { - Path absoluteOldPath = apRoot.resolve(oldRelativePath); + Path oldAbsolutePath = apRoot.resolve(oldRelativePath); + + /* oldRelativePath is the path which would have been given to old version of the entity + by the fragment manager. If it matches the fragment path, this means that + the fragment path was intended to follow the naming convention based on the configuration + (i.e. PER_OBJECT naming) - if (absoluteOldPath.equals(fragment.getFragmentPath())) { - Path absoluteNewPath = apRoot.resolve(newRelativePath); + if the paths don't match, then simply skip the renaming. This silently allows for: + - legacy fragments which don't follow the naming convention + - FRAGMENT type naming (fixed fragment for object types such as Parameters) + */ + if (oldAbsolutePath.equals(fragment.getFragmentPath())) { + Path newAbsolutePath = apRoot.resolve(newRelativePath); try { - FileUtils.moveFile(absoluteOldPath.toFile(), absoluteNewPath.toFile()); - fragment.setFragmentPath(absoluteNewPath); + FileUtils.moveFile(oldAbsolutePath.toFile(), newAbsolutePath.toFile()); + fragment.setFragmentPath(newAbsolutePath); } catch (IOException e) { throw new AutomationPackageConcurrentEditException( - String.format("Unable to rename file %s to file %s. Was the file renamed or deleted outside the editor?", absoluteOldPath, absoluteNewPath)); + String.format("Unable to rename file %s to file %s. Was the file renamed or deleted outside the editor?", oldAbsolutePath, newAbsolutePath)); } AutomationPackageFragmentYaml referencingFragment = determineReferencingFragment(oldRelativePath) @@ -269,17 +280,14 @@ private voi try { FileUtils.delete(fragment.getFragmentPath().toFile()); - Optional optionalReferencingFragment = determineReferencingFragment(fragment.getFragmentPath()); - - if (optionalReferencingFragment.isPresent()) { - AutomationPackageFragmentYaml referencingFragment = optionalReferencingFragment.get(); - String relativeFragmentReference = resourcePatchMatchingResolver.getFragmentReferenceString(apRoot.relativize(fragment.getFragmentPath())); + determineReferencingFragment(fragment.getFragmentPath()).ifPresent(referencingFragment -> { + String relativeFragmentReference = resourcePatchMatchingResolver + .getFragmentReferenceString(apRoot.relativize(fragment.getFragmentPath())); if (referencingFragment.getFragments().removeIf(f -> f.getValue().equals(relativeFragmentReference))) { referencingFragment.writeToDisk(); }; - fragments.remove(fragment); - } + }); } catch (IOException e) { throw new AutomationPackageConcurrentEditException(String.format("%s was removed outside the editor", fragment.getFragmentPath())); } @@ -304,9 +312,7 @@ private AutomationPackageFragmentYaml fragmentForNewObject(PatchableYamlModel p, fragments.add(fragment); fragment.setFragmentPath(absolutePath); - - Optional optionalReferencingFragment = determineReferencingFragment(path); - if (optionalReferencingFragment.isEmpty()) { + if (determineReferencingFragment(path).isEmpty()) { String referencingPath = resourcePatchMatchingResolver.getFragmentReferenceString(determineObjectRelativePath(p, fieldName, true)); descriptorYaml.getFragments().add(new PatchableYamlPrimitive<>(descriptorYaml.getPatchingContext(), referencingPath)); descriptorYaml.writeToDisk(); @@ -315,15 +321,10 @@ private AutomationPackageFragmentYaml fragmentForNewObject(PatchableYamlModel p, } private Optional determineReferencingFragment(Path path) { - - for (AutomationPackageFragmentYaml fragment : Stream.concat(Stream.of(descriptorYaml), fragments.stream()).toList()) { - for (PatchableYamlPrimitive fragmentPathPattern : fragment.getFragments()) { - if (resourcePatchMatchingResolver.isMatchingPath(fragmentPathPattern.getValue(), path)) { - return Optional.of(fragment); - } - } - } - return Optional.empty(); + return Stream.concat(Stream.of(descriptorYaml), fragments.stream()) + .filter(fragment -> fragment.getFragments().stream() + .anyMatch(pattern -> resourcePatchMatchingResolver.isMatchingPath(pattern.getValue(), path))) + .findFirst(); } public Path determineObjectRelativePath(PatchableYamlModel p, String fieldName, boolean globPattern) { diff --git a/step-core-model/src/main/java/step/core/dynamicbeans/DynamicValue.java b/step-core-model/src/main/java/step/core/dynamicbeans/DynamicValue.java index 9ae2693270..a784ddcb96 100644 --- a/step-core-model/src/main/java/step/core/dynamicbeans/DynamicValue.java +++ b/step-core-model/src/main/java/step/core/dynamicbeans/DynamicValue.java @@ -141,15 +141,4 @@ protected DynamicValue _cloneValue(DynamicValue clone) { protected boolean hasProtectedAccess() { return false; } - - @Override - public boolean equals(Object obj) { - if (obj == null) return false; - if (obj instanceof DynamicValue other) { - if (dynamic != other.dynamic) return false; - if (dynamic) return expression.equals(other.expression); - return value.equals(other.value); - } - return false; - } } diff --git a/step-core-model/src/main/java/step/core/yaml/PatchingContext.java b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java index e55fab299b..02c8f5af3f 100644 --- a/step-core-model/src/main/java/step/core/yaml/PatchingContext.java +++ b/step-core-model/src/main/java/step/core/yaml/PatchingContext.java @@ -33,12 +33,9 @@ public PatchingContext(ObjectMapper mapper) { } public PatchingContext(String sourceLocation, String yaml, ObjectMapper mapper) { - this.sourceLocation = sourceLocation; - this.initialLines = new CopyOnWriteArrayList<>(yaml.lines().toList()); - this.mapper = mapper; - Objects.requireNonNull(sourceLocation); - Objects.requireNonNull(mapper); - Objects.requireNonNull(initialLines); + this.sourceLocation = Objects.requireNonNull(sourceLocation); + this.initialLines = new CopyOnWriteArrayList<>(Objects.requireNonNull(yaml).lines().toList()); + this.mapper = Objects.requireNonNull(mapper); } public ObjectMapper getMapper() { @@ -149,9 +146,7 @@ private List getClaimedOuterBounds() { private String serializeUnindented(Object entity) { try { - return mapper.writeValueAsString(entity) - .replaceFirst("^---\\s*\\n*", "") - .trim(); + return mapper.writeValueAsString(entity); } catch (JsonProcessingException e) { throw new AutomationPackageUpdateException("Error Serializing YAML object", e); }