Skip to content

Commit d762696

Browse files
bentshermanbentsherman
committed
Typed workflows
Signed-off-by: Ben Sherman <bentshermann@gmail.com>
1 parent 432fce8 commit d762696

File tree

4 files changed

+237
-129
lines changed

4 files changed

+237
-129
lines changed

‎build.gradle‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,25 +34,25 @@ configurations {
3434
}
3535

3636
dependencies {
37-
implementation 'io.nextflow:nf-lang:26.03.0-edge'
38-
implementation 'org.apache.groovy:groovy:4.0.30'
39-
implementation 'org.apache.groovy:groovy-json:4.0.30'
40-
implementation 'org.apache.groovy:groovy-yaml:4.0.30'
37+
implementation 'io.nextflow:nf-lang:26.03.1-edge'
38+
implementation 'org.apache.groovy:groovy:4.0.31'
39+
implementation 'org.apache.groovy:groovy-json:4.0.31'
40+
implementation 'org.apache.groovy:groovy-yaml:4.0.31'
4141
implementation 'org.eclipse.lsp4j:org.eclipse.lsp4j:0.23.0'
4242
implementation 'org.eclipse.lsp4j:org.eclipse.lsp4j.jsonrpc:0.23.0'
4343

4444
// runtime dependencies for Nextflow scripts
45-
runtimeOnly 'org.apache.groovy:groovy-templates:4.0.30'
45+
runtimeOnly 'org.apache.groovy:groovy-templates:4.0.31'
4646
runtimeOnly 'org.yaml:snakeyaml:2.2'
4747

4848
// include Nextflow runtime at build-time to extract language definitions
49-
nextflowRuntime 'io.nextflow:nextflow:26.03.0-edge'
50-
nextflowRuntime 'io.nextflow:nf-amazon:3.8.0'
51-
nextflowRuntime 'io.nextflow:nf-azure:1.22.1'
52-
nextflowRuntime 'io.nextflow:nf-google:1.27.0'
49+
nextflowRuntime 'io.nextflow:nextflow:26.03.1-edge'
50+
nextflowRuntime 'io.nextflow:nf-amazon:3.8.1'
51+
nextflowRuntime 'io.nextflow:nf-azure:1.22.2'
52+
nextflowRuntime 'io.nextflow:nf-google:1.27.1'
5353
nextflowRuntime 'io.nextflow:nf-k8s:1.5.1'
5454
nextflowRuntime 'io.nextflow:nf-seqera:0.16.0'
55-
nextflowRuntime 'io.nextflow:nf-tower:1.22.0'
55+
nextflowRuntime 'io.nextflow:nf-tower:1.23.0'
5656
nextflowRuntime 'io.nextflow:nf-wave:1.19.0'
5757

5858
testImplementation ('org.objenesis:objenesis:3.4')

src/main/java/nextflow/script/control/TupleOpResolver.java renamed to src/main/java/nextflow/script/control/DataflowOpResolver.java

Lines changed: 75 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,19 @@
1515
*/
1616
package nextflow.script.control;
1717

18-
import java.util.Collections;
18+
import java.lang.reflect.Modifier;
19+
import java.util.ArrayList;
1920
import java.util.List;
20-
import java.util.Map;
2121

2222
import nextflow.script.types.Bag;
2323
import nextflow.script.types.Channel;
24+
import nextflow.script.types.Record;
2425
import nextflow.script.types.Tuple;
26+
import nextflow.script.types.TypesEx;
2527
import nextflow.script.types.Value;
2628
import org.codehaus.groovy.ast.ClassHelper;
2729
import org.codehaus.groovy.ast.ClassNode;
30+
import org.codehaus.groovy.ast.FieldNode;
2831
import org.codehaus.groovy.ast.GenericsType;
2932
import org.codehaus.groovy.ast.MethodNode;
3033
import org.codehaus.groovy.ast.expr.Expression;
@@ -37,16 +40,17 @@
3740
*
3841
* @author Ben Sherman <bentshermann@gmail.com>
3942
*/
40-
class TupleOpResolver {
43+
class DataflowOpResolver {
4144

4245
private static final ClassNode BAG_TYPE = ClassHelper.makeCached(Bag.class);
4346
private static final ClassNode CHANNEL_TYPE = ClassHelper.makeCached(Channel.class);
47+
private static final ClassNode RECORD_TYPE = ClassHelper.makeCached(Record.class);
4448
private static final ClassNode TUPLE_TYPE = ClassHelper.makeCached(Tuple.class);
4549
private static final ClassNode VALUE_TYPE = ClassHelper.makeCached(Value.class);
4650

4751
/**
48-
* Resolve the return type of dataflow operators that tranform
49-
* tuples, such as `combine`, `groupTuple`, and `join`.
52+
* Resolve the return type of dataflow operators where applicable,
53+
* such as `combine`, `groupBy`, and `join`.
5054
*
5155
* @param lhsType
5256
* @param method
@@ -58,7 +62,7 @@ public ClassNode apply(ClassNode lhsType, MethodNode method, List<Expression> ar
5862
if( "combine".equals(name) )
5963
return applyCombine(lhsType, arguments);
6064

61-
if( "groupTuple".equals(name) )
65+
if( "groupBy".equals(name) )
6266
return applyGroupBy(lhsType, arguments);
6367

6468
if( "join".equals(name) )
@@ -71,122 +75,109 @@ public ClassNode apply(ClassNode lhsType, MethodNode method, List<Expression> ar
7175
* Resolve the result type of a `combine` operation in terms of the left
7276
* and right operands.
7377
*
74-
* Given arguments of type `(L1, L2, ..., Lm)` and `R`, `combine`
75-
* produces a tuple of type `(L1, L2, ..., Lm, R).
76-
*
77-
* When the `by` option is specified, `combine` produces the same result
78-
* type as `join`.
78+
* Given arguments of type `L` and `R`, `combine` produces a tuple of type
79+
* `(L, R)`. If `L` and/or `R` are tuples, they are flattened into the resulting
80+
* tuple.
7981
*
8082
* @param lhsType
8183
* @param arguments
8284
*/
8385
private ClassNode applyCombine(ClassNode lhsType, List<Expression> arguments) {
84-
if( !TUPLE_TYPE.equals(lhsType) )
85-
return ClassHelper.dynamicType();
86-
87-
var namedArgs = namedArgs(arguments);
88-
if( namedArgs.containsKey("by") )
89-
return applyJoin(lhsType, arguments);
90-
91-
var argType = getType(arguments.get(arguments.size() - 1));
92-
var rhsType = dataflowElementType(argType);
86+
if( arguments.size() == 1 && arguments.get(0) instanceof NamedArgumentListExpression nale )
87+
return applyCombineNamedArgs(lhsType, nale);
88+
var rhsType = dataflowElementType(getType(arguments.get(0)));
89+
var componentTypes = new ArrayList<ClassNode>();
90+
if( !combineTupleOrValue(componentTypes, lhsType) )
91+
return channelTupleType(null);
92+
if( !combineTupleOrValue(componentTypes, rhsType) )
93+
return channelTupleType(null);
94+
var gts = componentTypes.stream()
95+
.map(cn -> new GenericsType(cn))
96+
.toArray(GenericsType[]::new);
97+
return channelTupleType(gts);
98+
}
9399

94-
var lgts = lhsType.getGenericsTypes();
95-
if( lgts == null || lgts.length == 0 )
100+
private ClassNode applyCombineNamedArgs(ClassNode lhsType, NamedArgumentListExpression nale) {
101+
if( !RECORD_TYPE.equals(lhsType) )
96102
return ClassHelper.dynamicType();
103+
var rhsType = new ClassNode(Record.class);
104+
for( var entry : nale.getMapEntryExpressions() ) {
105+
var name = entry.getKeyExpression().getText();
106+
var value = entry.getValueExpression();
107+
var valueType = dataflowValueType(getType(value));
108+
var fn = new FieldNode(name, Modifier.PUBLIC, valueType, rhsType, null);
109+
fn.setDeclaringClass(rhsType);
110+
rhsType.addField(fn);
111+
}
112+
var elementType = recordSumType(lhsType, rhsType);
113+
return makeType(CHANNEL_TYPE, elementType);
114+
}
97115

98-
var gts = new GenericsType[lgts.length + 1];
99-
for( int i = 0; i < lgts.length; i++ )
100-
gts[i] = lgts[i];
101-
gts[lgts.length] = new GenericsType(rhsType);
116+
private static ClassNode dataflowValueType(ClassNode type) {
117+
if( CHANNEL_TYPE.equals(type) )
118+
return ClassHelper.dynamicType();
119+
if( VALUE_TYPE.equals(type) )
120+
return elementType(type);
121+
return type;
122+
}
102123

103-
return channelTupleType(gts);
124+
private boolean combineTupleOrValue(List<ClassNode> componentTypes, ClassNode type) {
125+
if( TUPLE_TYPE.equals(type) ) {
126+
var gts = type.getGenericsTypes();
127+
if( gts == null && gts.length == 0 )
128+
return false;
129+
for( int i = 0; i < gts.length; i++ )
130+
componentTypes.add(gts[i].getType());
131+
}
132+
else {
133+
componentTypes.add(type);
134+
}
135+
return true;
104136
}
105137

106138
/**
107-
* Resolve the result type of a `groupTuple` operation.
139+
* Resolve the result type of a `groupBy` operation.
108140
*
109-
* Given source tuples of type `(K, V1, V2, ..., Vn)`,
110-
* `groupTuple` produces a tuple of type `(K, Bag<V1>, Bag<V2>, ..., Bag<Vn>)`.
141+
* Given source tuples of type `(K, N, V)` or `(K, V)`,
142+
* `groupBy` produces a tuple of type `(K, Bag<V>)`.
111143
*
112144
* @param lhsType
113145
* @param arguments
114146
*/
115147
private ClassNode applyGroupBy(ClassNode lhsType, List<Expression> arguments) {
116148
if( !TUPLE_TYPE.equals(lhsType) )
117149
return ClassHelper.dynamicType();
118-
119-
var namedArgs = namedArgs(arguments);
120-
if( namedArgs.containsKey("by") )
121-
return ClassHelper.dynamicType();
122-
123150
var lgts = lhsType.getGenericsTypes();
124-
if( lgts == null || lgts.length == 0 )
151+
if( lgts == null || !(lgts.length == 2 || lgts.length == 3) )
125152
return ClassHelper.dynamicType();
126-
127-
// TODO: group on index specified by `by` option
128-
// TODO: skip if `by` option isn't a single integer
129-
var gts = new GenericsType[lgts.length];
130-
gts[0] = lgts[0];
131-
for( int i = 1; i < lgts.length; i++ ) {
132-
var groupType = makeType(BAG_TYPE, lgts[i].getType());
133-
gts[i] = new GenericsType(groupType);
134-
}
135-
153+
var keyType = lgts[0].getType();
154+
var valueType = lgts[lgts.length - 1].getType();
155+
var gts = new GenericsType[] {
156+
new GenericsType(keyType),
157+
new GenericsType(makeType(BAG_TYPE, valueType))
158+
};
136159
return channelTupleType(gts);
137160
}
138161

139162
/**
140163
* Resolve the result type of a `join` operation in terms of the left
141164
* and right operands.
142165
*
143-
* Given tuples of type `(K, L1, L2, ..., Lm)` and `(K, R1, R2, ..., Rn)`,
144-
* `join` produces a tuple of type `(K, L1, L2, ..., Lm, R1, R2, ..., Rn).
166+
* Given two metching records R1 and R2, `join` produces R1 + R2.
145167
*
146168
* @param lhsType
147169
* @param arguments
148170
*/
149171
private ClassNode applyJoin(ClassNode lhsType, List<Expression> arguments) {
150-
if( !TUPLE_TYPE.equals(lhsType) )
151-
return ClassHelper.dynamicType();
152-
153-
var namedArgs = namedArgs(arguments);
154-
if( namedArgs.containsKey("by") )
172+
if( !RECORD_TYPE.equals(lhsType) )
155173
return ClassHelper.dynamicType();
156-
157174
var argType = getType(arguments.get(arguments.size() - 1));
158175
var rhsType = dataflowElementType(argType);
159-
if( !TUPLE_TYPE.equals(rhsType) )
176+
if( !RECORD_TYPE.equals(rhsType) )
160177
return ClassHelper.dynamicType();
161-
162-
var lgts = lhsType.getGenericsTypes();
163-
var rgts = rhsType.getGenericsTypes();
164-
if( lgts == null || lgts.length == 0 || rgts == null || rgts.length == 0 )
165-
return ClassHelper.dynamicType();
166-
167-
// TODO: join on index specified by `by` option
168-
// TODO: skip if `by` option isn't a single integer
169-
var gts = new GenericsType[lgts.length + rgts.length - 1];
170-
for( int i = 0; i < lgts.length; i++ )
171-
gts[i] = lgts[i];
172-
for( int i = 1; i < rgts.length; i++ )
173-
gts[lgts.length + i - 1] = rgts[i];
174-
175-
return channelTupleType(gts);
176-
}
177-
178-
private static Map<String,Expression> namedArgs(List<Expression> args) {
179-
return args.size() > 0 && args.get(0) instanceof NamedArgumentListExpression nale
180-
? Map.ofEntries(
181-
nale.getMapEntryExpressions().stream()
182-
.map((entry) -> {
183-
var name = entry.getKeyExpression().getText();
184-
var value = entry.getValueExpression();
185-
return Map.entry(name, value);
186-
})
187-
.toArray(Map.Entry[]::new)
188-
)
189-
: Collections.emptyMap();
178+
// TODO: report error if `by` field is not in both records
179+
var elementType = recordSumType(lhsType, rhsType);
180+
return makeType(CHANNEL_TYPE, elementType);
190181
}
191182

192183
private static ClassNode dataflowElementType(ClassNode type) {

0 commit comments

Comments
 (0)