From 0c5064674734fe473adfb71295089c53384ea83d Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Thu, 13 Apr 2023 11:31:29 +0200 Subject: [PATCH 01/15] Support for setting execution order in SpecInfo Can be used by extensions. Relates to #1443. --- docs/release_notes.adoc | 2 ++ .../runtime/SpockEngineDiscoveryPostProcessor.java | 3 +++ .../org/spockframework/runtime/model/SpecInfo.java | 10 ++++++++++ 3 files changed, 15 insertions(+) diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index 45652ce7d0..c2708d81ff 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -5,6 +5,8 @@ include::include.adoc[] == 2.4 (tbd) +* Support for setting execution order (a.k.a. run order) in `SpecInfo`, can be used by extensions + == 2.4-M1 (2022-11-30) * Fix issues with Spring 6/Spring Boot 3 spockPull:1541[] diff --git a/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java b/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java index 6c900e1704..d01d4e29cc 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java +++ b/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java @@ -5,6 +5,8 @@ import org.junit.platform.engine.*; +import static java.util.Comparator.comparingInt; + class SpockEngineDiscoveryPostProcessor { private static final Object[] EMPTY_ARGS = new Object[0]; @@ -14,6 +16,7 @@ SpockEngineDescriptor postProcessEngineDescriptor(UniqueId uniqueId, RunContext SpockEngineDescriptor processedEngineDescriptor = new SpockEngineDescriptor(uniqueId, runContext); engineDescriptor.getChildren().stream() .map(child -> processSpecNode(child, runContext)) + .sorted(comparingInt(child -> child instanceof SpecNode ? ((SpecNode) child).getNodeInfo().getExecutionOrder() : 0)) .forEach(processedEngineDescriptor::addChild); return processedEngineDescriptor; } diff --git a/spock-core/src/main/java/org/spockframework/runtime/model/SpecInfo.java b/spock-core/src/main/java/org/spockframework/runtime/model/SpecInfo.java index 287528214b..22da3adee0 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/model/SpecInfo.java +++ b/spock-core/src/main/java/org/spockframework/runtime/model/SpecInfo.java @@ -48,6 +48,8 @@ public class SpecInfo extends SpecElementInfo> implements IMe private ExecutionMode executionMode = null; private ExecutionMode childExecutionMode = null; + private int executionOrder; + private String pkg; private String filename; private String narrative; @@ -66,6 +68,14 @@ public class SpecInfo extends SpecElementInfo> implements IMe private final List features = new ArrayList<>(); + public int getExecutionOrder() { + return executionOrder; + } + + public void setExecutionOrder(int executionOrder) { + this.executionOrder = executionOrder; + } + public String getPackage() { return pkg; } From f623b476b9e3bdb9be96f9b4e7dbbc3e06786212 Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Thu, 13 Apr 2023 11:34:28 +0200 Subject: [PATCH 02/15] New built-in, global RandomRunOrderExtension Supports run order randomization for specifications, features or a combination of both. Relates to #1443. --- docs/extensions.adoc | 30 +++++++++++++ docs/release_notes.adoc | 2 + .../builtin/RandomRunOrderExtension.java | 42 +++++++++++++++++++ .../spock/config/RunnerConfiguration.java | 6 ++- ...amework.runtime.extension.IGlobalExtension | 1 + .../runtime/ExtensionClassesLoaderSpec.groovy | 2 +- 6 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java diff --git a/docs/extensions.adoc b/docs/extensions.adoc index 004a3c72b5..2366ab1baa 100644 --- a/docs/extensions.adoc +++ b/docs/extensions.adoc @@ -44,6 +44,18 @@ runner { See the <> section for a detailed description. +=== Random Test Order Configuration + +[source,groovy] +---- +runner { + randomizeSpecRunOrder true // randomize specification run order + randomizeFeatureRunOrder true // randomize feature run order within each specification +} +---- + +See the <<#_randomize_run_order>> section for more details. + == Built-In Extensions Most of Spock's built-in extensions are _annotation-driven_. In other words, they are triggered by annotating a @@ -709,6 +721,24 @@ runner { } ---- +=== Randomize Run Order + +Ideally, automated tests in general and Spock specifications in particular should be independent of each other. The same +applies to feature methods within a specification. One heuristic way to gain confidence that this is indeed the case, is +to randomize the order in which specifications and/or features within each specification are executed. + +By default, Spock executes both specifications and features in deterministic order, even though users should not rely on +any particular run order. You can explicitly activate run order randomization separately for specifications and features +or combine both randomization modes, using the <>: + +[source,groovy] +---- +runner { + randomizeSpecRunOrder true // randomize specification run order + randomizeFeatureRunOrder true // randomize feature run order within each specification +} +---- + == Third-Party Extensions You can find a list of third-party extensions in the https://github.com/spockframework/spock/wiki/Third-Party-Extensions[Spock Wiki]. diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index c2708d81ff..ba2e45ec33 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -6,6 +6,8 @@ include::include.adoc[] == 2.4 (tbd) * Support for setting execution order (a.k.a. run order) in `SpecInfo`, can be used by extensions +* New built-in `RandomRunOrderExtension` supports run order randomization for specifications, features or a combination of both. + See manual section <>. == 2.4-M1 (2022-11-30) diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java new file mode 100644 index 0000000000..2646eb9d5a --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java @@ -0,0 +1,42 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.runtime.extension.builtin; + +import org.spockframework.runtime.extension.IGlobalExtension; +import org.spockframework.runtime.model.FeatureInfo; +import org.spockframework.runtime.model.SpecInfo; +import spock.config.RunnerConfiguration; + +import java.util.Random; + +public class RandomRunOrderExtension implements IGlobalExtension { + private static final Random RANDOM = new Random(); + + private final RunnerConfiguration config; + + public RandomRunOrderExtension(RunnerConfiguration config) { + this.config = config; + } + + @Override + public void visitSpec(SpecInfo spec) { + if (config.randomizeSpecRunOrder) + spec.setExecutionOrder(RANDOM.nextInt()); + if (config.randomizeFeatureRunOrder) { + for (FeatureInfo featureInfo : spec.getAllFeatures()) + featureInfo.setExecutionOrder(RANDOM.nextInt()); + } + } +} diff --git a/spock-core/src/main/java/spock/config/RunnerConfiguration.java b/spock-core/src/main/java/spock/config/RunnerConfiguration.java index 5a01be9887..73ffd08bfb 100644 --- a/spock-core/src/main/java/spock/config/RunnerConfiguration.java +++ b/spock-core/src/main/java/spock/config/RunnerConfiguration.java @@ -28,7 +28,9 @@ * annotation some.pkg.Slow * baseClass IntegrationSpec * } - * filterStackTrace true // this is the default + * filterStackTrace true // this is the default + * randomizeSpecRunOrder false // this is the default + * randomizeFeatureRunOrder false // this is the default * } * */ @@ -39,4 +41,6 @@ public class RunnerConfiguration { public ParallelConfiguration parallel = new ParallelConfiguration(); public boolean filterStackTrace = true; public boolean optimizeRunOrder = false; + public boolean randomizeSpecRunOrder = false; + public boolean randomizeFeatureRunOrder = false; } diff --git a/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension b/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension index 778826130b..f78e62228f 100644 --- a/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension +++ b/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension @@ -1,3 +1,4 @@ org.spockframework.runtime.extension.builtin.IncludeExcludeExtension org.spockframework.runtime.extension.builtin.OptimizeRunOrderExtension +org.spockframework.runtime.extension.builtin.RandomRunOrderExtension org.spockframework.runtime.extension.builtin.UnrollExtension diff --git a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy index a0d72daf58..6fdfd5ed9f 100644 --- a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy +++ b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy @@ -15,7 +15,7 @@ class ExtensionClassesLoaderSpec extends Specification { def result = new ExtensionClassesLoader().loadExtensionClassesFromDefaultLocation() then: - result == [IncludeExcludeExtension, OptimizeRunOrderExtension, UnrollExtension] + result == [IncludeExcludeExtension, OptimizeRunOrderExtension, RandomRunOrderExtension, UnrollExtension] } def "loads global ConfigObjects"() { From 6400eb7acec7511e7f38354d53dede931fd44fe0 Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Thu, 13 Apr 2023 14:35:35 +0200 Subject: [PATCH 03/15] Add reference to PR #1631 to release notes --- docs/release_notes.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index ba2e45ec33..55bfef038b 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -5,9 +5,9 @@ include::include.adoc[] == 2.4 (tbd) -* Support for setting execution order (a.k.a. run order) in `SpecInfo`, can be used by extensions +* Support for setting execution order (a.k.a. run order) in `SpecInfo`, can be used by extensions spockPull:1631[] * New built-in `RandomRunOrderExtension` supports run order randomization for specifications, features or a combination of both. - See manual section <>. + See manual section <>. spockPull:1631[] == 2.4-M1 (2022-11-30) From 0b7727122cbd053037a5cc3c9d8ab8b556b23144 Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Thu, 13 Apr 2023 14:37:54 +0200 Subject: [PATCH 04/15] Add test for RandomRunOrderExtension Relates to #1443. --- .../RandomRunOrderExtensionSpec.groovy | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 spock-specs/src/test/groovy/org/spockframework/smoke/extension/RandomRunOrderExtensionSpec.groovy diff --git a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/RandomRunOrderExtensionSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/RandomRunOrderExtensionSpec.groovy new file mode 100644 index 0000000000..b50452fd18 --- /dev/null +++ b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/RandomRunOrderExtensionSpec.groovy @@ -0,0 +1,142 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.smoke.extension + +import groovy.transform.Canonical +import groovy.transform.ToString +import org.spockframework.EmbeddedSpecification +import org.spockframework.runtime.extension.builtin.RandomRunOrderExtension +import spock.lang.Retry + +import static spock.lang.Retry.Mode.SETUP_FEATURE_CLEANUP + +class RandomRunOrderExtensionSpec extends EmbeddedSpecification { + static ThreadLocal> executionsTL = new ThreadLocal<>() + + List specs + + def setup() { + executionsTL.set([]) + compiler.addClassMemberImport(RandomRunOrderExtensionSpec) + specs = compiler.compileWithImports(""" +class FirstSpec extends Specification { + @Shared execution = new SpecExecution(spec: this.class.simpleName) + def setup() { execution.features << specificationContext.currentFeature.name } + def cleanupSpec() { executionsTL.get() << execution } + def one() { expect: true } + def two() { expect: true } + def three() { expect: true } +} + +class SecondSpec extends Specification { + @Shared execution = new SpecExecution(spec: this.class.simpleName) + def setup() { execution.features << specificationContext.currentFeature.name } + def cleanupSpec() { executionsTL.get() << execution } + def one() { expect: true } + def two() { expect: true } + def three() { expect: true } +} + +class ThirdSpec extends Specification { + @Shared execution = new SpecExecution(spec: this.class.simpleName) + def setup() { execution.features << specificationContext.currentFeature.name } + def cleanupSpec() { executionsTL.get() << execution } + def one() { expect: true } + def two() { expect: true } + def three() { expect: true } +} + """) + } + + def "deterministic default order"() { + runner.configurationScript = { + runner {} + } + runner.extensionClasses << RandomRunOrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec'] + executions*.features == [['one', 'two', 'three'], ['one', 'two', 'three'], ['one', 'two', 'three']] + } + + @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) // if random order == deterministic order + def "random spec order"() { + runner.configurationScript = { + runner { + randomizeSpecRunOrder true + randomizeFeatureRunOrder false + } + } + runner.extensionClasses << RandomRunOrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec != ['FirstSpec', 'SecondSpec', 'ThirdSpec'] + executions*.features == [['one', 'two', 'three'], ['one', 'two', 'three'], ['one', 'two', 'three']] + } + + @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) // if random order == deterministic order + def "random feature order"() { + runner.configurationScript = { + runner { + randomizeSpecRunOrder false + randomizeFeatureRunOrder true + } + } + runner.extensionClasses << RandomRunOrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec'] + executions*.features != [['one', 'two', 'three'], ['one', 'two', 'three'], ['one', 'two', 'three']] + } + + @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) // if random order == deterministic order + def "random feature and spec order"() { + runner.configurationScript = { + runner { + randomizeSpecRunOrder true + randomizeFeatureRunOrder true + } + } + runner.extensionClasses << RandomRunOrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec != ['FirstSpec', 'SecondSpec', 'ThirdSpec'] + executions*.features != [['one', 'two', 'three'], ['one', 'two', 'three'], ['one', 'two', 'three']] + } + + @Canonical + @ToString(includePackage = false) + static class SpecExecution { + String spec + List features = [] + } + +} From 539da7c5e995a4b4bac6567f044320ca2f6c6ec8 Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Sun, 16 Apr 2023 11:33:31 +0200 Subject: [PATCH 05/15] Extend IGlobalExtension lifecylce by 'initSpecs' New lifecycle method initSpecs(Collection specs) is called once, before visiting single specifications later on in `visitSpec`. It enables global extensions to view all specifications as an ensemble, e.g. for iterating over them and rearranging their execution order. Relates to #1443. --- docs/extensions.adoc | 8 ++++++-- .../spockframework/runtime/ExtensionRunner.java | 17 +++++++++++++++++ .../org/spockframework/runtime/RunContext.java | 4 ++++ .../SpockEngineDiscoveryPostProcessor.java | 10 +++++++++- .../runtime/extension/IGlobalExtension.java | 9 ++++++--- 5 files changed, 42 insertions(+), 6 deletions(-) diff --git a/docs/extensions.adoc b/docs/extensions.adoc index 2366ab1baa..508bfc8f71 100644 --- a/docs/extensions.adoc +++ b/docs/extensions.adoc @@ -763,13 +763,17 @@ fully-qualified class name in a file `META-INF/services/org.spockframework.runti class path. As soon as these two conditions are satisfied, the extension is automatically loaded and used when Spock is running. -`IGlobalExtension` has the following three methods: +`IGlobalExtension` has the following four methods: `start()`:: This is called once at the very start of the Spock execution. +`initSpecs(Collection specs)`:: + This is called once, before visiting single specifications later on in `visitSpec`. It enables global extensions to + view all specifications as an ensemble, e.g. for iterating over them and rearranging their execution order. + `visitSpec(SpecInfo spec)`:: - This is called once for each specification. In this method you can prepare a specification with your extension magic, + This is called once for each specification. In this method, you can prepare a specification with your extension magic, like attaching interceptors to various interception points as described in the chapter <>. `stop()`:: diff --git a/spock-core/src/main/java/org/spockframework/runtime/ExtensionRunner.java b/spock-core/src/main/java/org/spockframework/runtime/ExtensionRunner.java index 687bb37619..ecb81804e5 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/ExtensionRunner.java +++ b/spock-core/src/main/java/org/spockframework/runtime/ExtensionRunner.java @@ -14,6 +14,7 @@ package org.spockframework.runtime; +import org.junit.platform.engine.TestDescriptor; import org.spockframework.runtime.extension.*; import org.spockframework.runtime.model.*; import org.spockframework.util.Nullable; @@ -30,6 +31,8 @@ */ @SuppressWarnings("rawtypes") public class ExtensionRunner { + private static final SpecInfo EMPTY_SPEC = new SpecInfo(); + private final SpecInfo spec; private final IExtensionRegistry extensionRegistry; private final IConfigurationRegistry configurationRegistry; @@ -42,6 +45,20 @@ public ExtensionRunner(SpecInfo spec, IExtensionRegistry extensionRegistry, ICon this.configurationRegistry = configurationRegistry; } + public ExtensionRunner(IExtensionRegistry extensionRegistry, IConfigurationRegistry configurationRegistry) { + this(EMPTY_SPEC, extensionRegistry, configurationRegistry); + } + + public void initGlobalExtensions(Set testDescriptors) { + List specs = testDescriptors.stream() + .filter(testDescriptor -> testDescriptor instanceof SpecNode) + .map(testDescriptor -> ((SpecNode) testDescriptor).getNodeInfo()) + .collect(toList()); + for (IGlobalExtension extension : extensionRegistry.getGlobalExtensions()) { + extension.initSpecs(specs); + } + } + public void run() { runGlobalExtensions(); runAnnotationDrivenExtensions(); diff --git a/spock-core/src/main/java/org/spockframework/runtime/RunContext.java b/spock-core/src/main/java/org/spockframework/runtime/RunContext.java index eefe913732..c1301e09ae 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/RunContext.java +++ b/spock-core/src/main/java/org/spockframework/runtime/RunContext.java @@ -81,6 +81,10 @@ public ExtensionRunner createExtensionRunner(SpecInfo spec) { return new ExtensionRunner(spec, globalExtensionRegistry, globalExtensionRegistry); } + public ExtensionRunner createExtensionRunner() { + return new ExtensionRunner(globalExtensionRegistry, globalExtensionRegistry); + } + public PlatformParameterizedSpecRunner createSpecRunner(SpecInfo spec) { return new PlatformParameterizedSpecRunner( new MasterRunSupervisor(spec, createStackTraceFilter(spec), diffedObjectRenderer)); diff --git a/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java b/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java index d01d4e29cc..961a95b7bc 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java +++ b/spock-core/src/main/java/org/spockframework/runtime/SpockEngineDiscoveryPostProcessor.java @@ -5,6 +5,8 @@ import org.junit.platform.engine.*; +import java.util.Set; + import static java.util.Comparator.comparingInt; class SpockEngineDiscoveryPostProcessor { @@ -14,7 +16,9 @@ class SpockEngineDiscoveryPostProcessor { SpockEngineDescriptor postProcessEngineDescriptor(UniqueId uniqueId, RunContext runContext, SpockEngineDescriptor engineDescriptor) { SpockEngineDescriptor processedEngineDescriptor = new SpockEngineDescriptor(uniqueId, runContext); - engineDescriptor.getChildren().stream() + Set testDescriptors = engineDescriptor.getChildren(); + initSpecNodes(testDescriptors, runContext); + testDescriptors.stream() .map(child -> processSpecNode(child, runContext)) .sorted(comparingInt(child -> child instanceof SpecNode ? ((SpecNode) child).getNodeInfo().getExecutionOrder() : 0)) .forEach(processedEngineDescriptor::addChild); @@ -46,6 +50,10 @@ private UniqueId toUniqueId(UniqueId parentId, FeatureInfo feature) { return parentId.append("feature", feature.getFeatureMethod().getReflection().getName()); } + private void initSpecNodes(Set testDescriptors, RunContext runContext) { + runContext.createExtensionRunner().initGlobalExtensions(testDescriptors); + } + private TestDescriptor processSpecNode(TestDescriptor child, RunContext runContext) { if (child instanceof SpecNode) { SpecNode specNode = (SpecNode) child; diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/IGlobalExtension.java b/spock-core/src/main/java/org/spockframework/runtime/extension/IGlobalExtension.java index f9315864c7..059438e5a7 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/IGlobalExtension.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/IGlobalExtension.java @@ -18,10 +18,13 @@ import org.spockframework.runtime.model.SpecInfo; +import java.util.Collection; + // TODO: start/stop lifecycle // TODO: design threading model public interface IGlobalExtension { - default void start() {}; - default void visitSpec(SpecInfo spec){}; - default void stop(){}; + default void start() {} + default void initSpecs(Collection specs) {} + default void visitSpec(SpecInfo spec) {} + default void stop() {} } From 268f7b3c7f3a8a12bcbcb9f8885645d0f3b3e80e Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Sun, 16 Apr 2023 11:41:44 +0200 Subject: [PATCH 06/15] Introduce SpecProcessor interface and SpecOrderer, implementing it SpecProcessor is designed to generically process a Collection, which e.g. is available to global extensions using the recently introduced lifecycle method initSpecs(Collection). New abstract class SpecOrderer is meant to be extended by other orderers wishing to assign run orders to specs/features via - SpecInfo.setExecutionOrder, - FeatureInfo.setExecutionOrder. Relates to #1443. --- .../spockframework/runtime/SpecProcessor.java | 9 +++++ .../builtin/orderer/SpecOrderer.java | 35 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java create mode 100644 spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java diff --git a/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java b/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java new file mode 100644 index 0000000000..b575c857ed --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java @@ -0,0 +1,9 @@ +package org.spockframework.runtime; + +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; + +public interface SpecProcessor { + void process(Collection specs); +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java new file mode 100644 index 0000000000..70bff7262a --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java @@ -0,0 +1,35 @@ +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.SpecProcessor; +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; + +public abstract class SpecOrderer implements SpecProcessor { + protected final boolean orderSpecs; + protected final boolean orderFeatures; + + public SpecOrderer(boolean orderSpecs, boolean orderFeatures) { + this.orderSpecs = orderSpecs; + this.orderFeatures = orderFeatures; + } + + @Override + public void process(Collection specs) { + if (orderSpecs) + orderSpecs(specs); + if (orderFeatures) + orderFeatures(specs); + } + + protected abstract void orderSpecs(Collection specs); + protected abstract void orderFeatures(Collection specs); + + public boolean isOrderSpecs() { + return orderSpecs; + } + + public boolean isOrderFeatures() { + return orderFeatures; + } +} From 1ae3bd7faa198ad25949aa45e277f9d4c30c9c77 Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Sun, 16 Apr 2023 11:45:23 +0200 Subject: [PATCH 07/15] Introduce run ordering as a generic feature Extending the initial implementation for run order randomisation, the former is not merely a special case of run ordering. We now have - DefaultSpecOrderer (basically a no-op) - RandomSpecOrderer, - AlphabeticalSpecOrderer, - AnnotatationBasedSpecOrderer with @Order(int). Relates to #1443. --- docs/extensions.adoc | 114 ++++++- ...rderExtension.java => OrderExtension.java} | 18 +- .../orderer/AlphabeticalSpecOrderer.java | 47 +++ .../orderer/AnnotatationBasedSpecOrderer.java | 32 ++ .../builtin/orderer/DefaultSpecOrderer.java | 17 + .../builtin/orderer/RandomSpecOrderer.java | 39 +++ .../spock/config/RunnerConfiguration.java | 14 +- .../src/main/java/spock/lang/Order.java | 38 +++ ...amework.runtime.extension.IGlobalExtension | 2 +- .../runtime/ExtensionClassesLoaderSpec.groovy | 2 +- .../smoke/extension/OrderExtensionSpec.groovy | 310 ++++++++++++++++++ .../RandomRunOrderExtensionSpec.groovy | 142 -------- 12 files changed, 600 insertions(+), 175 deletions(-) rename spock-core/src/main/java/org/spockframework/runtime/extension/builtin/{RandomRunOrderExtension.java => OrderExtension.java} (62%) create mode 100644 spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java create mode 100644 spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java create mode 100644 spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java create mode 100644 spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java create mode 100644 spock-core/src/main/java/spock/lang/Order.java create mode 100644 spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy delete mode 100644 spock-specs/src/test/groovy/org/spockframework/smoke/extension/RandomRunOrderExtensionSpec.groovy diff --git a/docs/extensions.adoc b/docs/extensions.adoc index 508bfc8f71..7240b099b3 100644 --- a/docs/extensions.adoc +++ b/docs/extensions.adoc @@ -44,17 +44,17 @@ runner { See the <> section for a detailed description. -=== Random Test Order Configuration +=== Test Order Configuration [source,groovy] ---- runner { - randomizeSpecRunOrder true // randomize specification run order - randomizeFeatureRunOrder true // randomize feature run order within each specification + orderer new RandomSpecOrderer() // randomize specification run order } ---- -See the <<#_randomize_run_order>> section for more details. +Instead of the default run order, you can configure Spock to execute specifications and/or features e.g. in random, +alphabetical or manually assigned, annotation-based order. See the <<#_run_order>> section for more details. == Built-In Extensions @@ -721,21 +721,111 @@ runner { } ---- -=== Randomize Run Order +=== Run Order Ideally, automated tests in general and Spock specifications in particular should be independent of each other. The same -applies to feature methods within a specification. One heuristic way to gain confidence that this is indeed the case, is -to randomize the order in which specifications and/or features within each specification are executed. +applies to feature methods within a specification. Therefore, you should not rely on any specific order of execution +(run order). -By default, Spock executes both specifications and features in deterministic order, even though users should not rely on -any particular run order. You can explicitly activate run order randomization separately for specifications and features -or combine both randomization modes, using the <>: +Nevertheless, you have options to influence the run order, using the <> and a set of built-in +orderers derived from super class `SpecOrderer`. Please check the Javadocs for package +`org.spockframework.runtime.extension.builtin.orderer` for more details. You can also write your own `SpecOrderer`, if +none of the built-in ones satisfies your needs. + +Please note that `@Stepwise` always trumps any run order you might have configured, i.e. `@Stepwise` "wins" against +`SpecOrderer`. + +==== Random Run Order + +One helpful way to heuristically increase your confidence that your tests are indeed independent of each other, is to +explicitly say goodbye to deterministic run order by randomizing it. + +[source,groovy] +---- +import org.spockframework.runtime.extension.builtin.orderer.RandomSpecOrderer + +runner { + orderer new RandomSpecOrderer( + true, // Randomize overall specification run order + true, // Randomize the run order of feature methods within specifications + System.currentTimeMillis() // Set a fixed value, if you want repeatable pseudo-random numbers. + // This might be helpful for reproducing issues when debugging your tests. + ) +} +---- + +==== Alphabetical Run Order + +Less useful than random run order, but available anyway, is a way to execute specifications and/or features +alphabetically, based on their display names and a simple `String.compareTo(String)` (no fancy locale-based collation). +The default sorting direction is ascending, optionally you can also sort elements in descending order. [source,groovy] ---- +import org.spockframework.runtime.extension.builtin.orderer.AlphabeticalSpecOrderer + runner { - randomizeSpecRunOrder true // randomize specification run order - randomizeFeatureRunOrder true // randomize feature run order within each specification + orderer new AlphabeticalSpecOrderer( + true, // Run specifications in alphabetical order by display name + true, // Run feature methods within specifications in alphabetical order by display name + false // Sort in ascending order (use 'true' for descending order) + ) +} +---- + +==== Annotation-Based Run Order + +If you want to basically retain Spock's default run order for most or at least some of your specifications and/or +feature methods, but modify it for particular specs/features, or take it to the extreme and manually assign run orders +everywhere, use the `@Order(int)` annotation in combination with the annotation-based orderer: + +[source,groovy] +---- +import org.spockframework.runtime.extension.builtin.orderer.AnnotatationBasedSpecOrderer + +runner { + orderer new AnnotatationBasedSpecOrderer() +} +---- + +Please note, that `@Order` annotations have no effect whatsoever, if `AnnotatationBasedSpecOrderer` is not configured +as the active orderer. E.g., you cannot expect to be able to use random ordering in combination with manually assigning +run orders via annotations for some exceptions. Annotation-based ordering must be explicitly activated and is only +available as a modification of Spock's default run order. + +Using `@Order`, the basic idea is to assume unannotated specifications and features to all carry an implicit `@Order(0)` +annotation. If you wish to run some specs/features before others, assign them a lower (negative) run order. If you want +to run them after the default-ordered elements, assign them a higher (positive) order number: + +[source,groovy] +---- +@Order(1) // Execute after default-ordered specs +class FirstSpec extends Specification { + // Execute features in order 'three', 'one', 'two' in ascending order of assigned @Order values + @Order(2) def one() { expect: true } + @Order(3) def two() { expect: true } + @Order(1) def three() { expect: true } +} + +@Order(-1) // Execute before default-ordered specs +class SecondSpec extends Specification { + def foo() { expect: true } // Default order + @Order(99) def bar() { expect: true } // Execute after default-ordered features + @Order(-5) def zot() { expect: true } // Execute before default-ordered features +} + +// Default order +class ThirdSpec extends Specification { + def "some feature"() { expect: true } // Default order + @Order(1) def "another feature"() { expect: true } // Execute after default-ordered features + def "one more feature"() { expect: true } // Default order +} + +// Default order +class FourthSpec extends Specification { + def 'feature X'() { expect: true } // Default order + def 'feature M'() { expect: true } // Default order + @Order(-1) def 'feature D'() { expect: true } // Execute before default-ordered features } ---- diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/OrderExtension.java similarity index 62% rename from spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java rename to spock-core/src/main/java/org/spockframework/runtime/extension/builtin/OrderExtension.java index 2646eb9d5a..2b77d70477 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/RandomRunOrderExtension.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/OrderExtension.java @@ -15,28 +15,20 @@ package org.spockframework.runtime.extension.builtin; import org.spockframework.runtime.extension.IGlobalExtension; -import org.spockframework.runtime.model.FeatureInfo; import org.spockframework.runtime.model.SpecInfo; import spock.config.RunnerConfiguration; -import java.util.Random; - -public class RandomRunOrderExtension implements IGlobalExtension { - private static final Random RANDOM = new Random(); +import java.util.Collection; +public class OrderExtension implements IGlobalExtension { private final RunnerConfiguration config; - public RandomRunOrderExtension(RunnerConfiguration config) { + public OrderExtension(RunnerConfiguration config) { this.config = config; } @Override - public void visitSpec(SpecInfo spec) { - if (config.randomizeSpecRunOrder) - spec.setExecutionOrder(RANDOM.nextInt()); - if (config.randomizeFeatureRunOrder) { - for (FeatureInfo featureInfo : spec.getAllFeatures()) - featureInfo.setExecutionOrder(RANDOM.nextInt()); - } + public void initSpecs(Collection specs) { + config.orderer.process(specs); } } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java new file mode 100644 index 0000000000..2037ea82bf --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java @@ -0,0 +1,47 @@ +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; +import java.util.concurrent.atomic.AtomicInteger; + +public class AlphabeticalSpecOrderer extends SpecOrderer { + private final boolean descending; + + public AlphabeticalSpecOrderer(boolean orderSpecs, boolean orderFeatures, boolean descending) { + super(orderSpecs, orderFeatures); + this.descending = descending; + } + + public AlphabeticalSpecOrderer(boolean orderSpecs, boolean orderFeatures) { + this(orderSpecs, orderFeatures, false); + } + + @Override + protected void orderSpecs(Collection specs) { + AtomicInteger i = new AtomicInteger(); + specs.stream() + .sorted((o1, o2) -> descending + ? o2.getDisplayName().compareTo(o1.getDisplayName()) + : o1.getDisplayName().compareTo(o2.getDisplayName()) + ) + .forEach(specInfo -> specInfo.setExecutionOrder(i.getAndIncrement())); + } + + @Override + protected void orderFeatures(Collection specs) { + for (SpecInfo spec : specs) { + AtomicInteger i = new AtomicInteger(); + spec.getAllFeatures().stream() + .sorted((o1, o2) -> descending + ? o2.getDisplayName().compareTo(o1.getDisplayName()) + : o1.getDisplayName().compareTo(o2.getDisplayName()) + ) + .forEach(featureInfo -> featureInfo.setExecutionOrder(i.getAndIncrement())); + } + } + + public boolean isDescending() { + return descending; + } +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java new file mode 100644 index 0000000000..ac234e3677 --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java @@ -0,0 +1,32 @@ +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.model.FeatureInfo; +import org.spockframework.runtime.model.SpecInfo; +import spock.lang.Order; + +import java.util.Collection; +import java.util.concurrent.atomic.AtomicInteger; + +public class AnnotatationBasedSpecOrderer extends SpecOrderer { + public AnnotatationBasedSpecOrderer() { + super(true, true); + } + + @Override + protected void orderSpecs(Collection specs) { + for (SpecInfo spec : specs) { + Order orderAnnotation = spec.getAnnotation(Order.class); + spec.setExecutionOrder(orderAnnotation == null ? 0 : orderAnnotation.value()); + } + } + + @Override + protected void orderFeatures(Collection specs) { + for (SpecInfo spec : specs) { + for (FeatureInfo feature : spec.getAllFeatures()) { + Order orderAnnotation = feature.getFeatureMethod().getAnnotation(Order.class); + feature.setExecutionOrder(orderAnnotation == null ? 0 : orderAnnotation.value()); + } + } + } +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java new file mode 100644 index 0000000000..7d115791ac --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java @@ -0,0 +1,17 @@ +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; + +public class DefaultSpecOrderer extends SpecOrderer { + public DefaultSpecOrderer() { + super(false, false); + } + + @Override + protected void orderSpecs(Collection specs) { } + + @Override + protected void orderFeatures(Collection specs) { } +} diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java new file mode 100644 index 0000000000..186a6a9482 --- /dev/null +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java @@ -0,0 +1,39 @@ +package org.spockframework.runtime.extension.builtin.orderer; + +import org.spockframework.runtime.model.FeatureInfo; +import org.spockframework.runtime.model.SpecInfo; + +import java.util.Collection; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +public class RandomSpecOrderer extends SpecOrderer { + private final Random random; + + public RandomSpecOrderer(boolean orderSpecs, boolean orderFeatures, long seed) { + super(orderSpecs, orderFeatures); + random = new Random(seed); + } + + public RandomSpecOrderer(boolean orderSpecs, boolean orderFeatures) { + this(orderSpecs, orderFeatures, System.currentTimeMillis()); + } + + public RandomSpecOrderer() { + this(true, true); + } + + @Override + protected void orderSpecs(Collection specs) { + for (SpecInfo spec : specs) + spec.setExecutionOrder(random.nextInt()); + } + + @Override + protected void orderFeatures(Collection specs) { + for (SpecInfo spec : specs) { + for (FeatureInfo feature : spec.getAllFeatures()) + feature.setExecutionOrder(random.nextInt()); + } + } +} diff --git a/spock-core/src/main/java/spock/config/RunnerConfiguration.java b/spock-core/src/main/java/spock/config/RunnerConfiguration.java index 73ffd08bfb..9afe511c0d 100644 --- a/spock-core/src/main/java/spock/config/RunnerConfiguration.java +++ b/spock-core/src/main/java/spock/config/RunnerConfiguration.java @@ -14,23 +14,26 @@ package spock.config; +import org.spockframework.runtime.extension.builtin.orderer.DefaultSpecOrderer; +import org.spockframework.runtime.extension.builtin.orderer.SpecOrderer; + /** * Configuration settings for the spec runner. * *

Example: * 

+ * import org.spockframework.runtime.extension.builtin.orderer.RandomSpecOrderer
  * import some.pkg.Fast
  * import some.pkg.IntegrationSpec
  *
  * runner {
- *   include Fast // could be either an annotation or a (base) class
+ *   include Fast  // could be either an annotation or a (base) class
  *   exclude {
  *     annotation some.pkg.Slow
  *     baseClass IntegrationSpec
  *   }
- *   filterStackTrace true          // this is the default
- *   randomizeSpecRunOrder false    // this is the default
- *   randomizeFeatureRunOrder false // this is the default
+ *   filterStackTrace true            // this is the default
+ *   orderer new RandomSpecOrderer()  // DefaultSpecOrderer (no-op) is the default
  * }
  *
*/ @@ -39,8 +42,7 @@ public class RunnerConfiguration { public IncludeExcludeCriteria include = new IncludeExcludeCriteria(); public IncludeExcludeCriteria exclude = new IncludeExcludeCriteria(); public ParallelConfiguration parallel = new ParallelConfiguration(); + public SpecOrderer orderer = new DefaultSpecOrderer(); public boolean filterStackTrace = true; public boolean optimizeRunOrder = false; - public boolean randomizeSpecRunOrder = false; - public boolean randomizeFeatureRunOrder = false; } diff --git a/spock-core/src/main/java/spock/lang/Order.java b/spock-core/src/main/java/spock/lang/Order.java new file mode 100644 index 0000000000..46af7265c5 --- /dev/null +++ b/spock-core/src/main/java/spock/lang/Order.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package spock.lang; + +import org.spockframework.runtime.extension.builtin.OrderExtension; +import org.spockframework.runtime.extension.builtin.orderer.AnnotatationBasedSpecOrderer; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Assigns an execution order to a specification or feature. + *

+ * Annotations of this type are picked up by the {@link OrderExtension}, if and only if the + * {@link AnnotatationBasedSpecOrderer} is activated in the Spock configuration file: 

+ * runner {
+ *   orderer new AnnotatationBasedSpecOrderer()
+ * }
+ */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.TYPE, ElementType.METHOD }) +public @interface Order { + int value(); +} diff --git a/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension b/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension index f78e62228f..77c6e009f6 100644 --- a/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension +++ b/spock-core/src/main/resources/META-INF/services/org.spockframework.runtime.extension.IGlobalExtension @@ -1,4 +1,4 @@ org.spockframework.runtime.extension.builtin.IncludeExcludeExtension org.spockframework.runtime.extension.builtin.OptimizeRunOrderExtension -org.spockframework.runtime.extension.builtin.RandomRunOrderExtension +org.spockframework.runtime.extension.builtin.OrderExtension org.spockframework.runtime.extension.builtin.UnrollExtension diff --git a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy index 6fdfd5ed9f..9cbaa9762f 100644 --- a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy +++ b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy @@ -15,7 +15,7 @@ class ExtensionClassesLoaderSpec extends Specification { def result = new ExtensionClassesLoader().loadExtensionClassesFromDefaultLocation() then: - result == [IncludeExcludeExtension, OptimizeRunOrderExtension, RandomRunOrderExtension, UnrollExtension] + result == [IncludeExcludeExtension, OptimizeRunOrderExtension, OrderExtension, UnrollExtension] } def "loads global ConfigObjects"() { diff --git a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy new file mode 100644 index 0000000000..ea645135ab --- /dev/null +++ b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy @@ -0,0 +1,310 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.spockframework.smoke.extension + +import groovy.transform.Canonical +import groovy.transform.ToString +import org.spockframework.EmbeddedSpecification +import org.spockframework.runtime.extension.builtin.OrderExtension +import org.spockframework.runtime.extension.builtin.orderer.AlphabeticalSpecOrderer +import org.spockframework.runtime.extension.builtin.orderer.AnnotatationBasedSpecOrderer +import org.spockframework.runtime.extension.builtin.orderer.RandomSpecOrderer +import spock.lang.Order +import spock.lang.Retry + +import static spock.lang.Retry.Mode.SETUP_FEATURE_CLEANUP + +class OrderExtensionSpec extends EmbeddedSpecification { + + @Canonical + @ToString(includePackage = false) + static class SpecExecution { + String spec + List features = [] + } + + static ThreadLocal> executionsTL = new ThreadLocal<>() + + List specs + + def setup() { + executionsTL.set([]) + compiler.addClassMemberImport(OrderExtensionSpec) + compiler.addClassImport(Order) + specs = compiler.compileWithImports(""" +class FirstSpec extends Specification { + @Shared execution = new SpecExecution(spec: this.class.simpleName) + def setup() { execution.features << specificationContext.currentFeature.name } + def cleanupSpec() { executionsTL.get() << execution } + def one() { expect: true } + def two() { expect: true } + def three() { expect: true } +} + +@Order(-1) +class SecondSpec extends Specification { + @Shared execution = new SpecExecution(spec: this.class.simpleName) + def setup() { execution.features << specificationContext.currentFeature.name } + def cleanupSpec() { executionsTL.get() << execution } + def foo() { expect: true } + @Order(99) def bar() { expect: true } + @Order(-5) def zot() { expect: true } +} + +class ThirdSpec extends Specification { + @Shared execution = new SpecExecution(spec: this.class.simpleName) + def setup() { execution.features << specificationContext.currentFeature.name } + def cleanupSpec() { executionsTL.get() << execution } + def "some feature"() { expect: true } + @Order(1) def "another feature"() { expect: true } + def "one more feature"() { expect: true } +} + +class FourthSpec extends Specification { + @Shared execution = new SpecExecution(spec: this.class.simpleName) + def setup() { execution.features << specificationContext.currentFeature.name } + def cleanupSpec() { executionsTL.get() << execution } + def 'feature X'() { expect: true } + def 'feature M'() { expect: true } + @Order(-1) def 'feature D'() { expect: true } +} + """) + } + + def 'default order'() { + runner.configurationScript = { + runner {} + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + executions*.features == [ + ['one', 'two', 'three'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'], + ['feature X', 'feature M', 'feature D'] + ] + } + + @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) + // if random order == deterministic order + def 'random spec order'() { + runner.configurationScript = { + runner { + orderer new RandomSpecOrderer(true, false) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec != ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + executions*.features.containsAll([ + ['one', 'two', 'three'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'], + ['feature X', 'feature M', 'feature D'] + ]) + } + + @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) + // if random order == deterministic order + def 'random feature order'() { + runner.configurationScript = { + runner { + orderer new RandomSpecOrderer(false, true) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + !executions*.features.containsAll([ + ['one', 'two', 'three'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'], + ['feature X', 'feature M', 'feature D'] + ]) + } + + @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) + // if random order == deterministic order + def 'random spec and feature order'() { + runner.configurationScript = { + runner { + orderer new RandomSpecOrderer(true, true) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec != ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + !executions*.features.containsAll([ + ['one', 'two', 'three'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'], + ['feature X', 'feature M', 'feature D'] + ]) + } + + def 'repeatable, random-seeded spec and feature order'() { + runner.configurationScript = { + runner { + orderer new RandomSpecOrderer(true, true, 42L) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['SecondSpec', 'ThirdSpec', 'FourthSpec', 'FirstSpec'] + executions*.features == [ + ['bar', 'zot', 'foo'], + ['another feature', 'one more feature', 'some feature'], + ['feature M', 'feature X', 'feature D'], + ['one', 'two', 'three'] + ] + } + + def 'alphabetical spec order'() { + runner.configurationScript = { + runner { + orderer new AlphabeticalSpecOrderer(true, false) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'FourthSpec', 'SecondSpec', 'ThirdSpec'] + executions*.features == [ + ['one', 'two', 'three'], + ['feature X', 'feature M', 'feature D'], + ['foo', 'bar', 'zot'], + ['some feature', 'another feature', 'one more feature'] + ] + } + + def 'alphabetical feature order'() { + runner.configurationScript = { + runner { + orderer new AlphabeticalSpecOrderer(false, true) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec', 'FourthSpec'] + executions*.features == [ + ['one', 'three', 'two'], + ['bar', 'foo', 'zot'], + ['another feature', 'one more feature', 'some feature'], + ['feature D', 'feature M', 'feature X'] + ] + } + + def 'alphabetical spec and feature order'() { + runner.configurationScript = { + runner { + orderer new AlphabeticalSpecOrderer(true, true) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'FourthSpec', 'SecondSpec', 'ThirdSpec'] + executions*.features == [ + ['one', 'three', 'two'], + ['feature D', 'feature M', 'feature X'], + ['bar', 'foo', 'zot'], + ['another feature', 'one more feature', 'some feature'] + ] + } + + def 'descending, alphabetical spec and feature order'() { + runner.configurationScript = { + runner { + orderer new AlphabeticalSpecOrderer(true, true, true) + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['FirstSpec', 'FourthSpec', 'SecondSpec', 'ThirdSpec'].reverse() + executions*.features == [ + ['one', 'three', 'two'].reverse(), + ['feature D', 'feature M', 'feature X'].reverse(), + ['bar', 'foo', 'zot'].reverse(), + ['another feature', 'one more feature', 'some feature'].reverse() + ].reverse() + } + + def 'annotation-based spec and feature order'() { + runner.configurationScript = { + runner { + orderer new AnnotatationBasedSpecOrderer() + } + } + runner.extensionClasses << OrderExtension + + when: + runner.runClasses(specs) + def executions = executionsTL.get() + + then: + executions*.spec == ['SecondSpec', 'FirstSpec', 'ThirdSpec', 'FourthSpec'] + executions*.features == [ + ['zot', 'foo', 'bar'], + ['one', 'two', 'three'], + ['some feature', 'one more feature', 'another feature'], + ['feature D', 'feature X', 'feature M'] + ] + } + +} diff --git a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/RandomRunOrderExtensionSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/RandomRunOrderExtensionSpec.groovy deleted file mode 100644 index b50452fd18..0000000000 --- a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/RandomRunOrderExtensionSpec.groovy +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * https://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.spockframework.smoke.extension - -import groovy.transform.Canonical -import groovy.transform.ToString -import org.spockframework.EmbeddedSpecification -import org.spockframework.runtime.extension.builtin.RandomRunOrderExtension -import spock.lang.Retry - -import static spock.lang.Retry.Mode.SETUP_FEATURE_CLEANUP - -class RandomRunOrderExtensionSpec extends EmbeddedSpecification { - static ThreadLocal> executionsTL = new ThreadLocal<>() - - List specs - - def setup() { - executionsTL.set([]) - compiler.addClassMemberImport(RandomRunOrderExtensionSpec) - specs = compiler.compileWithImports(""" -class FirstSpec extends Specification { - @Shared execution = new SpecExecution(spec: this.class.simpleName) - def setup() { execution.features << specificationContext.currentFeature.name } - def cleanupSpec() { executionsTL.get() << execution } - def one() { expect: true } - def two() { expect: true } - def three() { expect: true } -} - -class SecondSpec extends Specification { - @Shared execution = new SpecExecution(spec: this.class.simpleName) - def setup() { execution.features << specificationContext.currentFeature.name } - def cleanupSpec() { executionsTL.get() << execution } - def one() { expect: true } - def two() { expect: true } - def three() { expect: true } -} - -class ThirdSpec extends Specification { - @Shared execution = new SpecExecution(spec: this.class.simpleName) - def setup() { execution.features << specificationContext.currentFeature.name } - def cleanupSpec() { executionsTL.get() << execution } - def one() { expect: true } - def two() { expect: true } - def three() { expect: true } -} - """) - } - - def "deterministic default order"() { - runner.configurationScript = { - runner {} - } - runner.extensionClasses << RandomRunOrderExtension - - when: - runner.runClasses(specs) - def executions = executionsTL.get() - - then: - executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec'] - executions*.features == [['one', 'two', 'three'], ['one', 'two', 'three'], ['one', 'two', 'three']] - } - - @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) // if random order == deterministic order - def "random spec order"() { - runner.configurationScript = { - runner { - randomizeSpecRunOrder true - randomizeFeatureRunOrder false - } - } - runner.extensionClasses << RandomRunOrderExtension - - when: - runner.runClasses(specs) - def executions = executionsTL.get() - - then: - executions*.spec != ['FirstSpec', 'SecondSpec', 'ThirdSpec'] - executions*.features == [['one', 'two', 'three'], ['one', 'two', 'three'], ['one', 'two', 'three']] - } - - @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) // if random order == deterministic order - def "random feature order"() { - runner.configurationScript = { - runner { - randomizeSpecRunOrder false - randomizeFeatureRunOrder true - } - } - runner.extensionClasses << RandomRunOrderExtension - - when: - runner.runClasses(specs) - def executions = executionsTL.get() - - then: - executions*.spec == ['FirstSpec', 'SecondSpec', 'ThirdSpec'] - executions*.features != [['one', 'two', 'three'], ['one', 'two', 'three'], ['one', 'two', 'three']] - } - - @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) // if random order == deterministic order - def "random feature and spec order"() { - runner.configurationScript = { - runner { - randomizeSpecRunOrder true - randomizeFeatureRunOrder true - } - } - runner.extensionClasses << RandomRunOrderExtension - - when: - runner.runClasses(specs) - def executions = executionsTL.get() - - then: - executions*.spec != ['FirstSpec', 'SecondSpec', 'ThirdSpec'] - executions*.features != [['one', 'two', 'three'], ['one', 'two', 'three'], ['one', 'two', 'three']] - } - - @Canonical - @ToString(includePackage = false) - static class SpecExecution { - String spec - List features = [] - } - -} From 113d4ce5e9c9dee6e551041c6252deb044b808d0 Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Sun, 16 Apr 2023 12:28:55 +0200 Subject: [PATCH 08/15] Update release notes to run ordering status quo Relates to #1443. --- docs/release_notes.adoc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/release_notes.adoc b/docs/release_notes.adoc index 55bfef038b..614be023b9 100644 --- a/docs/release_notes.adoc +++ b/docs/release_notes.adoc @@ -5,9 +5,11 @@ include::include.adoc[] == 2.4 (tbd) +* New lifecycle method `IGlobalExtension.initSpecs` spockPull:1631[] * Support for setting execution order (a.k.a. run order) in `SpecInfo`, can be used by extensions spockPull:1631[] -* New built-in `RandomRunOrderExtension` supports run order randomization for specifications, features or a combination of both. - See manual section <>. spockPull:1631[] +* New built-in `OrderExtension` supports run order modification for specifications, features or a combination of both. + Built-in orderers can create random, alphabetical or user-defined ordering, the latter using the new `@Order(int)` + annotation. See manual section <>. spockPull:1631[] == 2.4-M1 (2022-11-30) From 015bfce05c0c11ff3063f0d3448656cc6d614b93 Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Sun, 16 Apr 2023 15:24:23 +0200 Subject: [PATCH 09/15] Rename AlphabeticalSpecOrderer.descending to ascending Ascending sort order is the default, so instead of making 'descending' default to false, we make 'ascending' default to true, getting rid of the logical double negation of calling the default "not descending". Relates to #1443. --- .../orderer/AlphabeticalSpecOrderer.java | 40 ++++++++++--------- .../smoke/extension/OrderExtensionSpec.groovy | 2 +- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java index 2037ea82bf..877efed593 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java @@ -6,42 +6,44 @@ import java.util.concurrent.atomic.AtomicInteger; public class AlphabeticalSpecOrderer extends SpecOrderer { - private final boolean descending; + private final boolean ascending; - public AlphabeticalSpecOrderer(boolean orderSpecs, boolean orderFeatures, boolean descending) { + public AlphabeticalSpecOrderer(boolean orderSpecs, boolean orderFeatures, boolean ascending) { super(orderSpecs, orderFeatures); - this.descending = descending; + this.ascending = ascending; } public AlphabeticalSpecOrderer(boolean orderSpecs, boolean orderFeatures) { - this(orderSpecs, orderFeatures, false); + this(orderSpecs, orderFeatures, true); + } + + public AlphabeticalSpecOrderer() { + this(true, true); } @Override protected void orderSpecs(Collection specs) { AtomicInteger i = new AtomicInteger(); specs.stream() - .sorted((o1, o2) -> descending - ? o2.getDisplayName().compareTo(o1.getDisplayName()) - : o1.getDisplayName().compareTo(o2.getDisplayName()) + .sorted((o1, o2) -> ascending + ? o1.getDisplayName().compareTo(o2.getDisplayName()) + : o2.getDisplayName().compareTo(o1.getDisplayName()) ) .forEach(specInfo -> specInfo.setExecutionOrder(i.getAndIncrement())); } @Override - protected void orderFeatures(Collection specs) { - for (SpecInfo spec : specs) { - AtomicInteger i = new AtomicInteger(); - spec.getAllFeatures().stream() - .sorted((o1, o2) -> descending - ? o2.getDisplayName().compareTo(o1.getDisplayName()) - : o1.getDisplayName().compareTo(o2.getDisplayName()) - ) - .forEach(featureInfo -> featureInfo.setExecutionOrder(i.getAndIncrement())); - } + protected void orderFeatures(Collection features) { + AtomicInteger i = new AtomicInteger(); + features.stream() + .sorted((o1, o2) -> ascending + ? o1.getDisplayName().compareTo(o2.getDisplayName()) + : o2.getDisplayName().compareTo(o1.getDisplayName()) + ) + .forEach(featureInfo -> featureInfo.setExecutionOrder(i.getAndIncrement())); } - public boolean isDescending() { - return descending; + public boolean isAscending() { + return ascending; } } diff --git a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy index ea645135ab..f2b01a61bf 100644 --- a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy +++ b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy @@ -266,7 +266,7 @@ class FourthSpec extends Specification { def 'descending, alphabetical spec and feature order'() { runner.configurationScript = { runner { - orderer new AlphabeticalSpecOrderer(true, true, true) + orderer new AlphabeticalSpecOrderer(true, true, false) } } runner.extensionClasses << OrderExtension From ed3937aeaf160cb94edbb7eca940674409759d0f Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Sun, 16 Apr 2023 15:31:16 +0200 Subject: [PATCH 10/15] Improve API of SpecOrderer.orderFeatures Use Collection method parameter instead of Collection, streamlining method implementations by factoring out looping over SpecInfos into SpecOrderer.process. Relates to #1443. --- .../builtin/orderer/AnnotatationBasedSpecOrderer.java | 11 ++++------- .../extension/builtin/orderer/DefaultSpecOrderer.java | 3 ++- .../extension/builtin/orderer/RandomSpecOrderer.java | 9 +++------ .../extension/builtin/orderer/SpecOrderer.java | 9 ++++++--- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java index ac234e3677..953ee1973b 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java @@ -5,7 +5,6 @@ import spock.lang.Order; import java.util.Collection; -import java.util.concurrent.atomic.AtomicInteger; public class AnnotatationBasedSpecOrderer extends SpecOrderer { public AnnotatationBasedSpecOrderer() { @@ -21,12 +20,10 @@ protected void orderSpecs(Collection specs) { } @Override - protected void orderFeatures(Collection specs) { - for (SpecInfo spec : specs) { - for (FeatureInfo feature : spec.getAllFeatures()) { - Order orderAnnotation = feature.getFeatureMethod().getAnnotation(Order.class); - feature.setExecutionOrder(orderAnnotation == null ? 0 : orderAnnotation.value()); - } + protected void orderFeatures(Collection features) { + for (FeatureInfo feature : features) { + Order orderAnnotation = feature.getFeatureMethod().getAnnotation(Order.class); + feature.setExecutionOrder(orderAnnotation == null ? 0 : orderAnnotation.value()); } } } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java index 7d115791ac..b1efc71d51 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java @@ -1,5 +1,6 @@ package org.spockframework.runtime.extension.builtin.orderer; +import org.spockframework.runtime.model.FeatureInfo; import org.spockframework.runtime.model.SpecInfo; import java.util.Collection; @@ -13,5 +14,5 @@ public DefaultSpecOrderer() { protected void orderSpecs(Collection specs) { } @Override - protected void orderFeatures(Collection specs) { } + protected void orderFeatures(Collection features) { } } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java index 186a6a9482..5774c06107 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java @@ -5,7 +5,6 @@ import java.util.Collection; import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; public class RandomSpecOrderer extends SpecOrderer { private final Random random; @@ -30,10 +29,8 @@ protected void orderSpecs(Collection specs) { } @Override - protected void orderFeatures(Collection specs) { - for (SpecInfo spec : specs) { - for (FeatureInfo feature : spec.getAllFeatures()) - feature.setExecutionOrder(random.nextInt()); - } + protected void orderFeatures(Collection features) { + for (FeatureInfo feature : features) + feature.setExecutionOrder(random.nextInt()); } } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java index 70bff7262a..13e33c5540 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java @@ -18,12 +18,15 @@ public SpecOrderer(boolean orderSpecs, boolean orderFeatures) { public void process(Collection specs) { if (orderSpecs) orderSpecs(specs); - if (orderFeatures) - orderFeatures(specs); + if (!orderFeatures) + return; + for (SpecInfo spec : specs) + orderFeatures(spec.getAllFeatures()); } protected abstract void orderSpecs(Collection specs); - protected abstract void orderFeatures(Collection specs); + + protected abstract void orderFeatures(Collection features); public boolean isOrderSpecs() { return orderSpecs; From a38e407773523cca58adcc0f25cf3ef2e72c823d Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Sun, 16 Apr 2023 15:32:37 +0200 Subject: [PATCH 11/15] Add Javadoc to spec processor and spec orderers Relates to #1443. --- .../spockframework/runtime/SpecProcessor.java | 12 ++++ .../orderer/AlphabeticalSpecOrderer.java | 24 ++++++++ .../orderer/AnnotatationBasedSpecOrderer.java | 8 +++ .../builtin/orderer/DefaultSpecOrderer.java | 6 ++ .../builtin/orderer/RandomSpecOrderer.java | 24 ++++++++ .../builtin/orderer/SpecOrderer.java | 56 +++++++++++++++++++ .../src/main/java/spock/lang/Order.java | 2 + 7 files changed, 132 insertions(+) diff --git a/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java b/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java index b575c857ed..fb65b25180 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java +++ b/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java @@ -1,9 +1,21 @@ package org.spockframework.runtime; +import org.spockframework.runtime.extension.builtin.orderer.SpecOrderer; import org.spockframework.runtime.model.SpecInfo; import java.util.Collection; +/** + * Generic bulk processor for a collection of {@link SpecInfo} elements + * + * @see SpecOrderer + */ public interface SpecProcessor { + /** + * Bulk-process a collection of {@link SpecInfo} elements in-place, i.e. do not return anything but operate on the + * elements given, changing their state if necessary. + * + * @param specs spec-info instances to be processed + */ void process(Collection specs); } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java index 877efed593..ce1dc88c4d 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java @@ -1,22 +1,46 @@ package org.spockframework.runtime.extension.builtin.orderer; +import org.spockframework.runtime.model.FeatureInfo; import org.spockframework.runtime.model.SpecInfo; import java.util.Collection; import java.util.concurrent.atomic.AtomicInteger; +/** + * Orderer capable of assigning specification and/or feature method run orders according to spec/feature display names, + * comparing them alphabetically. There is no locale-specific collation, only simple string comparison based on the + * default JVM locale, using by {@link String#compareTo(String)}. + */ public class AlphabeticalSpecOrderer extends SpecOrderer { private final boolean ascending; + /** + * Create an alphabetical spec orderer + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + * @param ascending sort in ascending order (yes/no)? + */ public AlphabeticalSpecOrderer(boolean orderSpecs, boolean orderFeatures, boolean ascending) { super(orderSpecs, orderFeatures); this.ascending = ascending; } + /** + * Create an alphabetical spec orderer with a default ascending sort order + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + * @see #AlphabeticalSpecOrderer(boolean, boolean, boolean) + */ public AlphabeticalSpecOrderer(boolean orderSpecs, boolean orderFeatures) { this(orderSpecs, orderFeatures, true); } + /** + * Create an alphabetical spec orderer with default values. This is a shorthand for calling + * {@link #AlphabeticalSpecOrderer(boolean, boolean, boolean)} with parameters {@code true, true, true}. + */ public AlphabeticalSpecOrderer() { this(true, true); } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java index 953ee1973b..3dcf58baed 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java @@ -6,7 +6,15 @@ import java.util.Collection; +/** + * Spec orderer for usef-defined, manual specification and/or feature method ordering, to be used in connection with + * {@link Order @Order} annotations. See the Spock user manual for more details. + */ public class AnnotatationBasedSpecOrderer extends SpecOrderer { + /** + * Create an annotation-based spec orderer + * @see Order + */ public AnnotatationBasedSpecOrderer() { super(true, true); } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java index b1efc71d51..8b08a40a1a 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java @@ -5,7 +5,13 @@ import java.util.Collection; +/** + * No-op spec orderer, used as a default if no other orderer is configured + */ public class DefaultSpecOrderer extends SpecOrderer { + /** + * Create a no-op spec orderer + */ public DefaultSpecOrderer() { super(false, false); } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java index 5774c06107..08e32df1f7 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java @@ -6,18 +6,42 @@ import java.util.Collection; import java.util.Random; +/** + * Orderer capable of randomizing specification and/or feature method run order + */ public class RandomSpecOrderer extends SpecOrderer { private final Random random; + /** + * Create a random spec orderer + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + * @param seed random seed to be used in {@link Random#Random(long)}; setting this to a fixed value enables + * you to create test runs with repeatable pseudo-random run ordering, which can be helpful when + * e.g. debugging tests failing due to a particular run order, making them more independent of + * each other in the process + */ public RandomSpecOrderer(boolean orderSpecs, boolean orderFeatures, long seed) { super(orderSpecs, orderFeatures); random = new Random(seed); } + /** + * Create a random spec orderer with a default random seed of {@code System.currentTimeMillis()} + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + * @see #RandomSpecOrderer(boolean, boolean, long) + */ public RandomSpecOrderer(boolean orderSpecs, boolean orderFeatures) { this(orderSpecs, orderFeatures, System.currentTimeMillis()); } + /** + * Create a random spec orderer with default values. This is a shorthand for calling + * {@link #RandomSpecOrderer(boolean, boolean, long)} with parameters {@code true, true, System.currentTimeMillis()}. + */ public RandomSpecOrderer() { this(true, true); } diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java index 13e33c5540..e26b42f124 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java @@ -1,19 +1,40 @@ package org.spockframework.runtime.extension.builtin.orderer; import org.spockframework.runtime.SpecProcessor; +import org.spockframework.runtime.model.FeatureInfo; import org.spockframework.runtime.model.SpecInfo; import java.util.Collection; +/** + * Abstract base class for specification and feature method orderers, i.e. workers modifying the corresponding execution + * order properties of spec-info and feature-info instances. + * + * @see DefaultSpecOrderer + * @see RandomSpecOrderer + * @see AnnotatationBasedSpecOrderer + * @see AlphabeticalSpecOrderer + */ public abstract class SpecOrderer implements SpecProcessor { protected final boolean orderSpecs; protected final boolean orderFeatures; + /** + * Create a spec-orderer with a user-defined operational scope + * + * @param orderSpecs modify specification run order (yes/no)? + * @param orderFeatures modify feature run order within a specification (yes/no)? + */ public SpecOrderer(boolean orderSpecs, boolean orderFeatures) { this.orderSpecs = orderSpecs; this.orderFeatures = orderFeatures; } + /** + * Dispatch to {@link #orderSpecs(Collection)} and then to {@link #orderFeatures(Collection)} for each spec-info + * + * @param specs spec-info instances to be processed + */ @Override public void process(Collection specs) { if (orderSpecs) @@ -24,8 +45,43 @@ public void process(Collection specs) { orderFeatures(spec.getAllFeatures()); } + /** + * Assign values to specification run orders. Implementors are expected to modify the corresponding object state + * in place, e.g. like this: + * 
+   * for (SpecInfo spec : specs)
+   *   spec.setExecutionOrder(random.nextInt());
+   *
+ * Or maybe: + * 
+   * AtomicInteger i = new AtomicInteger();
+   * specs.stream()
+   *   .sorted(myComparator)
+   *   .forEach(specInfo -> specInfo.setExecutionOrder(i.getAndIncrement()));
+   *
+ * + * @param specs spec-info instances to be ordered + */ + protected abstract void orderSpecs(Collection specs); + /** + * Assign values to feature run orders. Implementors are expected to modify the corresponding object state + * in place, e.g. like this: + * 
+   * for (FeatureInfo feature : features)
+   *   feature.setExecutionOrder(random.nextInt());
+   *
+ * Or maybe: + * 
+   * AtomicInteger i = new AtomicInteger();
+   * features.stream()
+   *   .sorted(myComparator)
+   *   .forEach(featureInfo -> featureInfo.setExecutionOrder(i.getAndIncrement()));
+   *
+ * + * @param features feature-info instances to be ordered + */ protected abstract void orderFeatures(Collection features); public boolean isOrderSpecs() { diff --git a/spock-core/src/main/java/spock/lang/Order.java b/spock-core/src/main/java/spock/lang/Order.java index 46af7265c5..39341e8467 100644 --- a/spock-core/src/main/java/spock/lang/Order.java +++ b/spock-core/src/main/java/spock/lang/Order.java @@ -30,6 +30,8 @@ * runner { * orderer new AnnotatationBasedSpecOrderer() * } + *

+ * See the Spock user manual for more details. */ @Retention(RetentionPolicy.RUNTIME) @Target({ ElementType.TYPE, ElementType.METHOD }) From f0117e0ff066dc880bcc1d34f45c2f3941d7068c Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Sun, 16 Apr 2023 17:50:28 +0200 Subject: [PATCH 12/15] Pull up executionOrder property into SpecElementInfo Both SpecInfo and FeatureInfo had identical executionOrder bean properties, declared redundantly. Therefore, I pulled them up into the base class. Relates to #1443. --- .../org/spockframework/runtime/model/FeatureInfo.java | 9 --------- .../spockframework/runtime/model/SpecElementInfo.java | 10 ++++++++++ .../org/spockframework/runtime/model/SpecInfo.java | 10 ---------- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/spock-core/src/main/java/org/spockframework/runtime/model/FeatureInfo.java b/spock-core/src/main/java/org/spockframework/runtime/model/FeatureInfo.java index 7c0f341d34..d3b633b672 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/model/FeatureInfo.java +++ b/spock-core/src/main/java/org/spockframework/runtime/model/FeatureInfo.java @@ -14,7 +14,6 @@ */ public class FeatureInfo extends SpecElementInfo implements ITestTaggable { private int declarationOrder; // per spec class - private int executionOrder; // per spec inheritance chain private final List parameterNames = new ArrayList<>(); private final List dataVariables = new ArrayList<>(); @@ -59,14 +58,6 @@ public void setDeclarationOrder(int declarationOrder) { this.declarationOrder = declarationOrder; } - public int getExecutionOrder() { - return executionOrder; - } - - public void setExecutionOrder(int executionOrder) { - this.executionOrder = executionOrder; - } - public List getParameterNames() { return parameterNames; } diff --git a/spock-core/src/main/java/org/spockframework/runtime/model/SpecElementInfo.java b/spock-core/src/main/java/org/spockframework/runtime/model/SpecElementInfo.java index dd58692020..188c3137b3 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/model/SpecElementInfo.java +++ b/spock-core/src/main/java/org/spockframework/runtime/model/SpecElementInfo.java @@ -31,9 +31,11 @@ public abstract class SpecElementInfo

tags = new ArrayList<>(); private final List attachments = new ArrayList<>(); private final List interceptors = new ArrayList<>(); + @Override public boolean isSkipped() { return skipped; @@ -75,6 +77,14 @@ public void setExcluded(boolean excluded) { this.excluded = excluded; } + public int getExecutionOrder() { + return executionOrder; + } + + public void setExecutionOrder(int executionOrder) { + this.executionOrder = executionOrder; + } + @Override public List getTags() { return tags; diff --git a/spock-core/src/main/java/org/spockframework/runtime/model/SpecInfo.java b/spock-core/src/main/java/org/spockframework/runtime/model/SpecInfo.java index 22da3adee0..287528214b 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/model/SpecInfo.java +++ b/spock-core/src/main/java/org/spockframework/runtime/model/SpecInfo.java @@ -48,8 +48,6 @@ public class SpecInfo extends SpecElementInfo> implements IMe private ExecutionMode executionMode = null; private ExecutionMode childExecutionMode = null; - private int executionOrder; - private String pkg; private String filename; private String narrative; @@ -68,14 +66,6 @@ public class SpecInfo extends SpecElementInfo> implements IMe private final List features = new ArrayList<>(); - public int getExecutionOrder() { - return executionOrder; - } - - public void setExecutionOrder(int executionOrder) { - this.executionOrder = executionOrder; - } - public String getPackage() { return pkg; } From ae342b2030c261e9a0bd28732644699748afff4e Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Sun, 16 Apr 2023 18:09:11 +0200 Subject: [PATCH 13/15] Add missing copyright headers --- .../org/spockframework/runtime/SpecProcessor.java | 14 ++++++++++++++ .../builtin/orderer/AlphabeticalSpecOrderer.java | 14 ++++++++++++++ .../orderer/AnnotatationBasedSpecOrderer.java | 14 ++++++++++++++ .../builtin/orderer/DefaultSpecOrderer.java | 14 ++++++++++++++ .../builtin/orderer/RandomSpecOrderer.java | 14 ++++++++++++++ .../extension/builtin/orderer/SpecOrderer.java | 14 ++++++++++++++ .../runtime/ExtensionClassesLoaderSpec.groovy | 14 ++++++++++++++ 7 files changed, 98 insertions(+) diff --git a/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java b/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java index fb65b25180..479efb3c39 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java +++ b/spock-core/src/main/java/org/spockframework/runtime/SpecProcessor.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.spockframework.runtime; import org.spockframework.runtime.extension.builtin.orderer.SpecOrderer; diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java index ce1dc88c4d..dfab34cd06 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AlphabeticalSpecOrderer.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.spockframework.runtime.extension.builtin.orderer; import org.spockframework.runtime.model.FeatureInfo; diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java index 3dcf58baed..ddbe80b85a 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/AnnotatationBasedSpecOrderer.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.spockframework.runtime.extension.builtin.orderer; import org.spockframework.runtime.model.FeatureInfo; diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java index 8b08a40a1a..216e3c0b50 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/DefaultSpecOrderer.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.spockframework.runtime.extension.builtin.orderer; import org.spockframework.runtime.model.FeatureInfo; diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java index 08e32df1f7..7e6eb67584 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/RandomSpecOrderer.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.spockframework.runtime.extension.builtin.orderer; import org.spockframework.runtime.model.FeatureInfo; diff --git a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java index e26b42f124..255530c5f5 100644 --- a/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java +++ b/spock-core/src/main/java/org/spockframework/runtime/extension/builtin/orderer/SpecOrderer.java @@ -1,3 +1,17 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.spockframework.runtime.extension.builtin.orderer; import org.spockframework.runtime.SpecProcessor; diff --git a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy index 9cbaa9762f..3c0075d810 100644 --- a/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy +++ b/spock-specs/src/test/groovy/org/spockframework/runtime/ExtensionClassesLoaderSpec.groovy @@ -1,3 +1,17 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.spockframework.runtime import org.spockframework.report.log.ReportLogConfiguration From ddea5824cf4dfb281446f1e401a94971c19be9bc Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Tue, 18 Apr 2023 08:10:51 +0200 Subject: [PATCH 14/15] OrderExtensionSpec: explain @Retry usage better --- .../smoke/extension/OrderExtensionSpec.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy index f2b01a61bf..4e1be80761 100644 --- a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy +++ b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy @@ -103,8 +103,8 @@ class FourthSpec extends Specification { ] } + // Retry, if random order accidentally equals default order @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) - // if random order == deterministic order def 'random spec order'() { runner.configurationScript = { runner { @@ -127,8 +127,8 @@ class FourthSpec extends Specification { ]) } + // Retry, if random order accidentally equals default order @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) - // if random order == deterministic order def 'random feature order'() { runner.configurationScript = { runner { @@ -151,8 +151,8 @@ class FourthSpec extends Specification { ]) } + // Retry, if random order accidentally equals default order @Retry(count = 50, mode = SETUP_FEATURE_CLEANUP) - // if random order == deterministic order def 'random spec and feature order'() { runner.configurationScript = { runner { From da3817f2e13afbd4ca3d1d7227e50ffb6a599559 Mon Sep 17 00:00:00 2001 From: Alexander Kriegisch Date: Tue, 18 Apr 2023 08:12:10 +0200 Subject: [PATCH 15/15] OrderExtensionSpec: condense redundant code in embedded spec by extracting logging code into base spec. --- .../smoke/extension/OrderExtensionSpec.groovy | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy index 4e1be80761..d42454581c 100644 --- a/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy +++ b/spock-specs/src/test/groovy/org/spockframework/smoke/extension/OrderExtensionSpec.groovy @@ -44,38 +44,32 @@ class OrderExtensionSpec extends EmbeddedSpecification { compiler.addClassMemberImport(OrderExtensionSpec) compiler.addClassImport(Order) specs = compiler.compileWithImports(""" -class FirstSpec extends Specification { +abstract class BaseSpec extends Specification { @Shared execution = new SpecExecution(spec: this.class.simpleName) def setup() { execution.features << specificationContext.currentFeature.name } def cleanupSpec() { executionsTL.get() << execution } +} + +class FirstSpec extends BaseSpec { def one() { expect: true } def two() { expect: true } def three() { expect: true } } @Order(-1) -class SecondSpec extends Specification { - @Shared execution = new SpecExecution(spec: this.class.simpleName) - def setup() { execution.features << specificationContext.currentFeature.name } - def cleanupSpec() { executionsTL.get() << execution } +class SecondSpec extends BaseSpec { def foo() { expect: true } @Order(99) def bar() { expect: true } @Order(-5) def zot() { expect: true } } -class ThirdSpec extends Specification { - @Shared execution = new SpecExecution(spec: this.class.simpleName) - def setup() { execution.features << specificationContext.currentFeature.name } - def cleanupSpec() { executionsTL.get() << execution } +class ThirdSpec extends BaseSpec { def "some feature"() { expect: true } @Order(1) def "another feature"() { expect: true } def "one more feature"() { expect: true } } -class FourthSpec extends Specification { - @Shared execution = new SpecExecution(spec: this.class.simpleName) - def setup() { execution.features << specificationContext.currentFeature.name } - def cleanupSpec() { executionsTL.get() << execution } +class FourthSpec extends BaseSpec { def 'feature X'() { expect: true } def 'feature M'() { expect: true } @Order(-1) def 'feature D'() { expect: true }