From f32fcf5ea20e8bd2fb3333800c45cc8ec372e335 Mon Sep 17 00:00:00 2001 From: Magnus Eide-Fredriksen Date: Fri, 14 Mar 2025 11:22:46 +0100 Subject: [PATCH 1/7] feat(LSP): add comment to groupingparser to allow comments in YQL --- .../language-server/src/main/ccc/grouping/GroupingParser.ccc | 3 +++ .../language-server/src/main/ccc/yqlplus/YQLPlus.ccc | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc index a84f2392585b..aec8588725bd 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc @@ -179,6 +179,9 @@ TOKEN : ; +UNPARSED : + +; // -------------------------------------------------------------------------------- // diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc index 36c989a8e989..47b3d3e26c06 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -125,7 +125,7 @@ TOKEN : ; UNPARSED : - + ; // -------------------------------------------------------------------------------- From 778f54f526f6b19878e1918fc4ac836d320bb69e Mon Sep 17 00:00:00 2001 From: Magnus Eide-Fredriksen Date: Fri, 14 Mar 2025 12:28:55 +0100 Subject: [PATCH 2/7] feat(LSP): general interface for finding comments in code --- .../src/main/ccc/SchemaParser.ccc | 11 ++++ .../src/main/ccc/grouping/GroupingParser.ccc | 12 +++++ .../ccc/indexinglanguage/IndexingParser.ccc | 12 +++++ .../RankingExpressionParser.ccc | 12 +++++ .../src/main/ccc/yqlplus/YQLPlus.ccc | 12 +++++ .../ai/vespa/schemals/common/StringUtils.java | 41 +++++++++++++- .../semantictokens/SemanticTokenUtils.java | 16 ++++++ .../semantictokens/SchemaSemanticTokens.java | 53 +++---------------- .../schemals/parser/GeneralTokenSource.java | 18 +++++++ .../schema/IdentifyDirtySchemaNodes.java | 4 +- .../schema/IdentifySymbolReferences.java | 5 +- .../java/ai/vespa/schemals/tree/CSTUtils.java | 8 +-- .../java/ai/vespa/schemals/tree/Node.java | 3 ++ .../ai/vespa/schemals/tree/SchemaNode.java | 7 ++- .../java/ai/vespa/schemals/tree/YQLNode.java | 11 ++++ 15 files changed, 170 insertions(+), 55 deletions(-) create mode 100644 integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/semantictokens/SemanticTokenUtils.java create mode 100644 integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/parser/GeneralTokenSource.java diff --git a/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc index 4528301ecba5..df5f80059561 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc @@ -166,6 +166,17 @@ INJECT SchemaParser: } +INJECT TokenSource : +{ + import ai.vespa.schemals.parser.GeneralTokenSource; +} + +INJECT : +{ + abstract public class TokenSource implements GeneralTokenSource { + } +} + INJECT SchemaParserLexer: { public static EnumSet getRegularTokens() { diff --git a/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc index aec8588725bd..44c47b8f9783 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc @@ -47,6 +47,18 @@ INJECT GroupingParserLexer: } } +INJECT TokenSource : +{ + import ai.vespa.schemals.parser.GeneralTokenSource; +} + +INJECT : +{ + abstract public class TokenSource implements GeneralTokenSource { + } +} + + TOKEN : (["l","L"])? | (["l","L"])? | (["l","L"])?> | diff --git a/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc index 85b6715ffdf9..ff4bc5db7fca 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc @@ -136,6 +136,18 @@ INJECT IndexingParserLexer: } } +INJECT TokenSource : +{ + import ai.vespa.schemals.parser.GeneralTokenSource; +} + +INJECT : +{ + abstract public class TokenSource implements GeneralTokenSource { + } +} + + SKIP : " " | "\t" | "\r" | "\f" ; diff --git a/integration/schema-language-server/language-server/src/main/ccc/rankingexpression/RankingExpressionParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/rankingexpression/RankingExpressionParser.ccc index 29126bb4e2ab..a1b3dd47f4fe 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/rankingexpression/RankingExpressionParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/rankingexpression/RankingExpressionParser.ccc @@ -56,6 +56,18 @@ INJECT RankingExpressionParserLexer: } } +INJECT TokenSource : +{ + import ai.vespa.schemals.parser.GeneralTokenSource; +} + +INJECT : +{ + abstract public class TokenSource implements GeneralTokenSource { + } +} + + INJECT BaseNode : import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; { diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc index 47b3d3e26c06..c8be3338db34 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -19,6 +19,18 @@ INJECT YQLPlusLexer: } } +INJECT TokenSource : +{ + import ai.vespa.schemals.parser.GeneralTokenSource; +} + +INJECT : +{ + abstract public class TokenSource implements GeneralTokenSource { + } +} + + // -------------------------------------------------------------------------------- // // Token declarations. diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java index 5f4ec74c5ccd..df51ef693944 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java @@ -1,12 +1,15 @@ package ai.vespa.schemals.common; +import java.util.ArrayList; import java.util.List; import org.eclipse.lsp4j.Position; import org.eclipse.lsp4j.Range; +import ai.vespa.schemals.tree.CSTUtils; import ai.vespa.schemals.tree.Node; -import ai.vespa.schemals.tree.SchemaNode; + +import ai.vespa.schemals.parser.GeneralTokenSource; /** * StringUtils @@ -127,6 +130,42 @@ public static boolean isInsideComment(String content, Position pos) { return false; } + /** + * @param rootNode Root AST node to search for single line comments + * @param commentMarker Substring marking the start of a single line comment + * @return A sorted list of {@link Range} where start position is location of commentMarker + * and end position is location of newline character at the end of the comment. + */ + public static List findSingleLineComments(Node rootNode, String commentMarker) { + ArrayList ret = new ArrayList<>(); + GeneralTokenSource tokenSource = rootNode.getTokenSource(); + + String content = tokenSource.toString(); + + int index = content.indexOf(commentMarker); + while (index >= 0) { + Position start = CSTUtils.getPositionFromOffset(tokenSource, index); + if (CSTUtils.getLeafNodeAtPosition(rootNode, start) != null) { + index = content.indexOf(commentMarker, index + 1); + continue; + } + + index = content.indexOf("\n", index + 1); + + if (index < 0) { + index = content.length() - 1; + } + + Position end = CSTUtils.getPositionFromOffset(tokenSource, index); + + ret.add(new Range(start, end)); + + index = content.indexOf(commentMarker, index + 1); + } + + return ret; + } + public static Position getStringPosition(String str) { int lines = 0; int column = str.length(); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/semantictokens/SemanticTokenUtils.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/semantictokens/SemanticTokenUtils.java new file mode 100644 index 000000000000..a9b616a919ff --- /dev/null +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/semantictokens/SemanticTokenUtils.java @@ -0,0 +1,16 @@ +package ai.vespa.schemals.lsp.common.semantictokens; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.eclipse.lsp4j.Range; + +public class SemanticTokenUtils { + public static List convertCommentRanges(List commentRanges) { + int commentTokenType = CommonSemanticTokens.getType("comment"); + return commentRanges.stream() + .map(range -> new SemanticTokenMarker(commentTokenType, range)) + .collect(Collectors.toList()); + } +} diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java index b86e352af04a..a872ece0faa6 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java @@ -18,11 +18,13 @@ import ai.vespa.schemals.tree.SchemaNode; import ai.vespa.schemals.tree.Node.LanguageType; import ai.vespa.schemals.common.ClientLogger; +import ai.vespa.schemals.common.StringUtils; import ai.vespa.schemals.context.EventDocumentContext; import ai.vespa.schemals.index.Symbol.SymbolStatus; import ai.vespa.schemals.index.Symbol.SymbolType; import ai.vespa.schemals.lsp.common.semantictokens.CommonSemanticTokens; import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenMarker; +import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenUtils; import ai.vespa.schemals.parser.Token.TokenType; import ai.vespa.schemals.parser.TokenSource; import ai.vespa.schemals.parser.ast.FILTER; @@ -250,50 +252,9 @@ private static List findSemanticMarkersForSymbol(SchemaNode return ret; } - private static ArrayList convertCommentRanges(ArrayList comments) { - ArrayList ret = new ArrayList<>(); - - int tokenType = CommonSemanticTokens.getType("comment"); - - for (Range range : comments) { - ret.add(new SemanticTokenMarker(tokenType, range)); - } - - return ret; - } - - private static ArrayList findComments(SchemaNode rootNode) { - TokenSource tokenSource = rootNode.getTokenSource(); - String source = tokenSource.toString(); - ArrayList ret = new ArrayList<>(); - - int index = source.indexOf("#"); - while (index >= 0) { - Position start = CSTUtils.getPositionFromOffset(tokenSource, index); - if (CSTUtils.getLeafNodeAtPosition(rootNode, start) != null) { - index = source.indexOf("#", index + 1); - continue; - } - - index = source.indexOf("\n", index + 1); - - if (index < 0) { - index = source.length() - 1; - } - - Position end = CSTUtils.getPositionFromOffset(tokenSource, index); - - ret.add(new Range(start, end)); - - index = source.indexOf("#", index + 1); - } - - return convertCommentRanges(ret); - } - // This function assumes that both of the lists are sorted, and that no elements are overlapping - private static ArrayList mergeSemanticTokenMarkers(ArrayList lhs, ArrayList rhs) { - ArrayList ret = new ArrayList<>(lhs.size() + rhs.size()); + private static List mergeSemanticTokenMarkers(List lhs, List rhs) { + List ret = new ArrayList<>(lhs.size() + rhs.size()); int lhsIndex = 0; int rhsIndex = 0; @@ -335,9 +296,11 @@ public static SemanticTokens getSemanticTokens(EventDocumentContext context) { return new SemanticTokens(new ArrayList<>()); } - ArrayList comments = findComments(context.document.getRootNode()); + List comments = SemanticTokenUtils.convertCommentRanges( + StringUtils.findSingleLineComments(context.document.getRootNode(), "#") + ); - ArrayList markers = traverseCST(node, context.logger); + List markers = traverseCST(node, context.logger); List compactMarkers = SemanticTokenMarker.concatCompactForm( mergeSemanticTokenMarkers(markers, comments) ); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/parser/GeneralTokenSource.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/parser/GeneralTokenSource.java new file mode 100644 index 000000000000..43680403f699 --- /dev/null +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/parser/GeneralTokenSource.java @@ -0,0 +1,18 @@ +package ai.vespa.schemals.parser; + +/** + * Interface to capture logic needed from the different parser's generated TokenSource classes. + */ +public interface GeneralTokenSource extends CharSequence { + public String getInputSource(); + public String getText(int startOffset, int endOffset); + public String toString(); + public CharSequence subSequence(int start, int end); + public char charAt(int pos); + public int length(); + public int getCodePointColumnFromOffset(int pos); + public int getLineEndOffset(int lineNumber); + public int getLineFromOffset(int pos); + public int getLineStartOffset(int lineNumber); + public void setInputSource(String inputSource); +} diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifyDirtySchemaNodes.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifyDirtySchemaNodes.java index a25f448dabf4..e56f961d715b 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifyDirtySchemaNodes.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifyDirtySchemaNodes.java @@ -8,7 +8,7 @@ import org.eclipse.lsp4j.Range; import ai.vespa.schemals.parser.ParseException; -import ai.vespa.schemals.parser.TokenSource; +import ai.vespa.schemals.parser.GeneralTokenSource; import ai.vespa.schemals.parser.Token.ParseExceptionSource; import ai.vespa.schemals.schemadocument.parser.Identifier; import ai.vespa.schemals.schemadocument.parser.IdentifyDirtyNodes; @@ -46,7 +46,7 @@ public List identify(SchemaNode node) { ParseExceptionSource parseException = node.getParseExceptionSource(); if (parseException != null) { - TokenSource tokenSource = node.getTokenSource(); + GeneralTokenSource tokenSource = node.getTokenSource(); Range range = CSTUtils.getRangeFromOffsets(tokenSource, parseException.beginOffset, parseException.endOffset); String message = getParseExceptionMessage(parseException.parseException); ret.add(new SchemaDiagnostic.Builder() diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolReferences.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolReferences.java index 40caf74aed86..db6f236f95af 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolReferences.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolReferences.java @@ -20,6 +20,7 @@ import ai.vespa.schemals.index.Symbol; import ai.vespa.schemals.index.Symbol.SymbolStatus; import ai.vespa.schemals.index.Symbol.SymbolType; +import ai.vespa.schemals.parser.TokenSource; import ai.vespa.schemals.parser.ast.COLON; import ai.vespa.schemals.parser.ast.FIELD; import ai.vespa.schemals.parser.ast.annotationRefDataType; @@ -300,7 +301,7 @@ private ArrayList handleFieldReference(SchemaNode identifierNode) { newEnd += subfields[i].length() + 1; identifierStr newASTNode = new identifierStr(); - newASTNode.setTokenSource(identifierNode.getTokenSource()); + newASTNode.setTokenSource((TokenSource)identifierNode.getTokenSource()); newASTNode.setBeginOffset(identifierNode.getOriginalSchemaNode().getBeginOffset()); newASTNode.setEndOffset(identifierNode.getOriginalSchemaNode().getEndOffset()); @@ -377,7 +378,7 @@ private ArrayList handleImportField(SchemaNode identifierNode) { newEnd += subfields[1].length() + 1; identifierStr newASTNode = new identifierStr(); - newASTNode.setTokenSource(identifierNode.getTokenSource()); + newASTNode.setTokenSource((TokenSource)identifierNode.getTokenSource()); newASTNode.setBeginOffset(identifierNode.getOriginalSchemaNode().getBeginOffset()); newASTNode.setEndOffset(identifierNode.getOriginalSchemaNode().getEndOffset()); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/CSTUtils.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/CSTUtils.java index dd092ca36595..92e044fa6306 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/CSTUtils.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/CSTUtils.java @@ -13,6 +13,7 @@ import ai.vespa.schemals.index.Symbol; import ai.vespa.schemals.index.Symbol.SymbolStatus; import ai.vespa.schemals.index.Symbol.SymbolType; +import ai.vespa.schemals.parser.GeneralTokenSource; import ai.vespa.schemals.parser.TokenSource; import ai.vespa.schemals.parser.ast.documentElm; import ai.vespa.schemals.parser.ast.functionElm; @@ -25,14 +26,14 @@ public class CSTUtils { - public static Position getPositionFromOffset(TokenSource tokenSource, int offset) { + public static Position getPositionFromOffset(GeneralTokenSource tokenSource, int offset) { int line = tokenSource.getLineFromOffset(offset) - 1; int startOfLineOffset = tokenSource.getLineStartOffset(line + 1); int column = offset - startOfLineOffset; return new Position(line, column); } - public static Range getRangeFromOffsets(TokenSource tokenSource, int beginOffset, int endOffset) { + public static Range getRangeFromOffsets(GeneralTokenSource tokenSource, int beginOffset, int endOffset) { Position begin = getPositionFromOffset(tokenSource, beginOffset); Position end = getPositionFromOffset(tokenSource, endOffset); return new Range(begin, end); @@ -43,8 +44,7 @@ public static Range getNodeRange(ai.vespa.schemals.parser.Node node) { try { return getRangeFromOffsets(tokenSource, node.getBeginOffset(), node.getEndOffset()); } catch(Exception e) { - // TODO: something happens when offsets are bad - } + } return new Range(new Position(0, 0), new Position(0, 0)); } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java index 9f5b166286c2..5a18a688cb4d 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java @@ -9,6 +9,7 @@ import ai.vespa.schemals.index.Symbol; import ai.vespa.schemals.index.Symbol.SymbolType; +import ai.vespa.schemals.parser.GeneralTokenSource; public abstract class Node implements Iterable { @@ -122,6 +123,8 @@ public Node getPrevious() { return parent.get(parentIndex - 1); } + public abstract GeneralTokenSource getTokenSource(); + /** * Returns the next SchemaNode of the sibilings of the node. * If there is no next node, it returns the next node of the parent node. diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java index 9fd520c5d3c0..03ca222d9a61 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java @@ -18,6 +18,7 @@ import ai.vespa.schemals.tree.indexinglanguage.ILUtils; import ai.vespa.schemals.tree.rankingexpression.RankNode; import ai.vespa.schemals.tree.rankingexpression.RankingExpressionUtils; +import ai.vespa.schemals.parser.GeneralTokenSource; import ai.vespa.schemals.parser.ast.expression; import ai.vespa.schemals.parser.ast.featureListElm; @@ -387,9 +388,13 @@ public ParseExceptionSource getParseExceptionSource() { return null; } - public TokenSource getTokenSource() { + public GeneralTokenSource getTokenSource() { if (language == LanguageType.SCHEMA) { return originalSchemaNode.getTokenSource(); + } else if (language == LanguageType.RANK_EXPRESSION) { + return originalRankExpressionNode.getTokenSource(); + } else if (language == LanguageType.INDEXING) { + return originalIndexingNode.getTokenSource(); } return null; diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java index dd21c9435856..f054a9f645d3 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java @@ -3,6 +3,7 @@ import org.eclipse.lsp4j.Position; import org.eclipse.lsp4j.Range; +import ai.vespa.schemals.parser.GeneralTokenSource; import ai.vespa.schemals.parser.grouping.ast.request; import ai.vespa.schemals.parser.yqlplus.Node; import ai.vespa.schemals.tree.YQL.YQLUtils; @@ -142,4 +143,14 @@ public String toString() { Position end = range.getEnd(); return "YQLNode(" + getASTClass() + ", " + start.getLine() + "," + start.getCharacter() + "->" + end.getLine() + "," + end.getCharacter() + ")"; } + + @Override + public GeneralTokenSource getTokenSource() { + if (language == LanguageType.YQLPlus) { + return originalYQLNode.getTokenSource(); + } else if (language == LanguageType.GROUPING) { + return originalGroupingNode.getTokenSource(); + } + return null; + } } From 0e8afe6961e21b4803982891235077c4763b2e0f Mon Sep 17 00:00:00 2001 From: Magnus Eide-Fredriksen Date: Fri, 28 Mar 2025 14:11:28 +0100 Subject: [PATCH 3/7] fix(lsp): correctly mark single line comments in yql files --- .../src/main/ccc/SchemaParser.ccc | 11 ----- .../src/main/ccc/grouping/GroupingParser.ccc | 13 ----- .../ccc/indexinglanguage/IndexingParser.ccc | 12 ----- .../RankingExpressionParser.ccc | 12 ----- .../src/main/ccc/yqlplus/YQLPlus.ccc | 12 ----- .../ai/vespa/schemals/common/StringUtils.java | 49 +++++++++---------- .../semantictokens/SemanticTokenUtils.java | 37 ++++++++++++++ .../semantictokens/SchemaSemanticTokens.java | 40 +-------------- .../semantictokens/YQLPlusSemanticTokens.java | 12 ++++- .../schemals/parser/GeneralTokenSource.java | 18 ------- .../schema/IdentifyDirtySchemaNodes.java | 8 +-- .../schema/IdentifySymbolReferences.java | 5 +- .../java/ai/vespa/schemals/tree/CSTUtils.java | 5 +- .../java/ai/vespa/schemals/tree/Node.java | 3 -- .../ai/vespa/schemals/tree/SchemaNode.java | 13 ----- .../java/ai/vespa/schemals/tree/YQLNode.java | 14 ++---- 16 files changed, 85 insertions(+), 179 deletions(-) delete mode 100644 integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/parser/GeneralTokenSource.java diff --git a/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc index df5f80059561..4528301ecba5 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc @@ -166,17 +166,6 @@ INJECT SchemaParser: } -INJECT TokenSource : -{ - import ai.vespa.schemals.parser.GeneralTokenSource; -} - -INJECT : -{ - abstract public class TokenSource implements GeneralTokenSource { - } -} - INJECT SchemaParserLexer: { public static EnumSet getRegularTokens() { diff --git a/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc index 44c47b8f9783..8beb4c8c3ded 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc @@ -47,19 +47,6 @@ INJECT GroupingParserLexer: } } -INJECT TokenSource : -{ - import ai.vespa.schemals.parser.GeneralTokenSource; -} - -INJECT : -{ - abstract public class TokenSource implements GeneralTokenSource { - } -} - - - TOKEN : (["l","L"])? | (["l","L"])? | (["l","L"])?> | <#DECIMAL: ["1"-"9"] (["0"-"9"])*> | diff --git a/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc index ff4bc5db7fca..85b6715ffdf9 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc @@ -136,18 +136,6 @@ INJECT IndexingParserLexer: } } -INJECT TokenSource : -{ - import ai.vespa.schemals.parser.GeneralTokenSource; -} - -INJECT : -{ - abstract public class TokenSource implements GeneralTokenSource { - } -} - - SKIP : " " | "\t" | "\r" | "\f" ; diff --git a/integration/schema-language-server/language-server/src/main/ccc/rankingexpression/RankingExpressionParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/rankingexpression/RankingExpressionParser.ccc index a1b3dd47f4fe..29126bb4e2ab 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/rankingexpression/RankingExpressionParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/rankingexpression/RankingExpressionParser.ccc @@ -56,18 +56,6 @@ INJECT RankingExpressionParserLexer: } } -INJECT TokenSource : -{ - import ai.vespa.schemals.parser.GeneralTokenSource; -} - -INJECT : -{ - abstract public class TokenSource implements GeneralTokenSource { - } -} - - INJECT BaseNode : import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; { diff --git a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc index c8be3338db34..47b3d3e26c06 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc @@ -19,18 +19,6 @@ INJECT YQLPlusLexer: } } -INJECT TokenSource : -{ - import ai.vespa.schemals.parser.GeneralTokenSource; -} - -INJECT : -{ - abstract public class TokenSource implements GeneralTokenSource { - } -} - - // -------------------------------------------------------------------------------- // // Token declarations. diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java index df51ef693944..015dc30dbab7 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java @@ -9,8 +9,6 @@ import ai.vespa.schemals.tree.CSTUtils; import ai.vespa.schemals.tree.Node; -import ai.vespa.schemals.parser.GeneralTokenSource; - /** * StringUtils * For dealing with specific problems related to Strings that represent text documents. @@ -130,40 +128,41 @@ public static boolean isInsideComment(String content, Position pos) { return false; } - /** - * @param rootNode Root AST node to search for single line comments - * @param commentMarker Substring marking the start of a single line comment - * @return A sorted list of {@link Range} where start position is location of commentMarker - * and end position is location of newline character at the end of the comment. - */ - public static List findSingleLineComments(Node rootNode, String commentMarker) { - ArrayList ret = new ArrayList<>(); - GeneralTokenSource tokenSource = rootNode.getTokenSource(); + public static List findSingleLineComments(String content, String commentMarker, ClientLogger debug) { + List commentRanges = new ArrayList<>(); + if (content == null) return commentRanges; - String content = tokenSource.toString(); - int index = content.indexOf(commentMarker); - while (index >= 0) { - Position start = CSTUtils.getPositionFromOffset(tokenSource, index); - if (CSTUtils.getLeafNodeAtPosition(rootNode, start) != null) { - index = content.indexOf(commentMarker, index + 1); + int currentLineNum = 0; + int currentLineStart = 0; + int nextComment = content.indexOf(commentMarker); + int nextNewLine = content.indexOf('\n'); + + // Iterate the text while keeping track of line offset information + while (nextComment != -1) { + // Consume lines before the comment + if (nextNewLine != -1 && nextNewLine < nextComment) { + currentLineStart = nextNewLine + 1; + nextNewLine = content.indexOf('\n', nextNewLine + 1); + currentLineNum++; continue; } - index = content.indexOf("\n", index + 1); + int commentEndOffset = nextNewLine; - if (index < 0) { - index = content.length() - 1; + if (nextNewLine == -1) { + commentEndOffset = content.length(); } - Position end = CSTUtils.getPositionFromOffset(tokenSource, index); - - ret.add(new Range(start, end)); + debug.info("Comment found at offset: " + nextComment); + Position commentStart = new Position(currentLineNum, nextComment - currentLineStart); + Position commentEnd = new Position(currentLineNum, commentEndOffset - currentLineStart); + commentRanges.add(new Range(commentStart, commentEnd)); - index = content.indexOf(commentMarker, index + 1); + nextComment = content.indexOf(commentMarker, commentEndOffset + 1); } - return ret; + return commentRanges; } public static Position getStringPosition(String str) { diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/semantictokens/SemanticTokenUtils.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/semantictokens/SemanticTokenUtils.java index a9b616a919ff..5ed1eee6ea49 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/semantictokens/SemanticTokenUtils.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/semantictokens/SemanticTokenUtils.java @@ -4,8 +4,11 @@ import java.util.List; import java.util.stream.Collectors; +import org.eclipse.lsp4j.Position; import org.eclipse.lsp4j.Range; +import ai.vespa.schemals.tree.CSTUtils; + public class SemanticTokenUtils { public static List convertCommentRanges(List commentRanges) { int commentTokenType = CommonSemanticTokens.getType("comment"); @@ -13,4 +16,38 @@ public static List convertCommentRanges(List comment .map(range -> new SemanticTokenMarker(commentTokenType, range)) .collect(Collectors.toList()); } + + // This function assumes that both of the lists are sorted, and that no elements are overlapping + public static List mergeSemanticTokenMarkers(List lhs, List rhs) { + List ret = new ArrayList<>(lhs.size() + rhs.size()); + + int lhsIndex = 0; + int rhsIndex = 0; + while ( + lhsIndex < lhs.size() && + rhsIndex < rhs.size() + ) { + Position rhsPos = rhs.get(rhsIndex).getRange().getStart(); + Position lhsPos = lhs.get(lhsIndex).getRange().getStart(); + + if (CSTUtils.positionLT(lhsPos, rhsPos)) { + ret.add(lhs.get(lhsIndex)); + lhsIndex++; + } else { + ret.add(rhs.get(rhsIndex)); + rhsIndex++; + } + } + + for (int i = lhsIndex; i < lhs.size(); i++) { + ret.add(lhs.get(i)); + } + + for (int i = rhsIndex; i < rhs.size(); i++) { + ret.add(rhs.get(i)); + } + + return ret; + } + } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java index a872ece0faa6..a643884f1048 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java @@ -7,13 +7,11 @@ import java.util.Map; import java.util.Set; -import org.eclipse.lsp4j.Position; import org.eclipse.lsp4j.Range; import org.eclipse.lsp4j.SemanticTokenModifiers; import org.eclipse.lsp4j.SemanticTokenTypes; import org.eclipse.lsp4j.SemanticTokens; -import ai.vespa.schemals.tree.CSTUtils; import ai.vespa.schemals.tree.Node; import ai.vespa.schemals.tree.SchemaNode; import ai.vespa.schemals.tree.Node.LanguageType; @@ -26,7 +24,6 @@ import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenMarker; import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenUtils; import ai.vespa.schemals.parser.Token.TokenType; -import ai.vespa.schemals.parser.TokenSource; import ai.vespa.schemals.parser.ast.FILTER; import ai.vespa.schemals.parser.ast.RANK_TYPE; import ai.vespa.schemals.parser.ast.bool; @@ -252,39 +249,6 @@ private static List findSemanticMarkersForSymbol(SchemaNode return ret; } - // This function assumes that both of the lists are sorted, and that no elements are overlapping - private static List mergeSemanticTokenMarkers(List lhs, List rhs) { - List ret = new ArrayList<>(lhs.size() + rhs.size()); - - int lhsIndex = 0; - int rhsIndex = 0; - while ( - lhsIndex < lhs.size() && - rhsIndex < rhs.size() - ) { - Position rhsPos = rhs.get(rhsIndex).getRange().getStart(); - Position lhsPos = lhs.get(lhsIndex).getRange().getStart(); - - if (CSTUtils.positionLT(lhsPos, rhsPos)) { - ret.add(lhs.get(lhsIndex)); - lhsIndex++; - } else { - ret.add(rhs.get(rhsIndex)); - rhsIndex++; - } - } - - for (int i = lhsIndex; i < lhs.size(); i++) { - ret.add(lhs.get(i)); - } - - for (int i = rhsIndex; i < rhs.size(); i++) { - ret.add(rhs.get(i)); - } - - return ret; - } - public static SemanticTokens getSemanticTokens(EventDocumentContext context) { if (context.document == null) { @@ -297,12 +261,12 @@ public static SemanticTokens getSemanticTokens(EventDocumentContext context) { } List comments = SemanticTokenUtils.convertCommentRanges( - StringUtils.findSingleLineComments(context.document.getRootNode(), "#") + StringUtils.findSingleLineComments(context.document.getCurrentContent(), "#", context.logger) ); List markers = traverseCST(node, context.logger); List compactMarkers = SemanticTokenMarker.concatCompactForm( - mergeSemanticTokenMarkers(markers, comments) + SemanticTokenUtils.mergeSemanticTokenMarkers(markers, comments) ); return new SemanticTokens(compactMarkers); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java index 9fa33d26f545..8b8a2c5031d1 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java @@ -7,13 +7,15 @@ import org.eclipse.lsp4j.SemanticTokens; import ai.vespa.schemals.common.ClientLogger; +import ai.vespa.schemals.common.StringUtils; import ai.vespa.schemals.context.EventDocumentContext; import ai.vespa.schemals.lsp.common.semantictokens.CommonSemanticTokens; import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenMarker; +import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenUtils; import ai.vespa.schemals.parser.yqlplus.ast.pipeline_step; import ai.vespa.schemals.tree.Node; -import ai.vespa.schemals.tree.YQLNode; import ai.vespa.schemals.tree.Node.LanguageType; +import ai.vespa.schemals.tree.YQLNode; public class YQLPlusSemanticTokens { @@ -88,8 +90,14 @@ public static SemanticTokens getSemanticTokens(EventDocumentContext context) { return new SemanticTokens(new ArrayList<>()); } + List comments = SemanticTokenUtils.convertCommentRanges( + StringUtils.findSingleLineComments(context.document.getCurrentContent(), "//", context.logger) + ); + List markers = traverseCST(node, context.logger); - List compactMarkers = SemanticTokenMarker.concatCompactForm(markers); + List compactMarkers = SemanticTokenMarker.concatCompactForm( + SemanticTokenUtils.mergeSemanticTokenMarkers(comments, markers) + ); return new SemanticTokens(compactMarkers); } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/parser/GeneralTokenSource.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/parser/GeneralTokenSource.java deleted file mode 100644 index 43680403f699..000000000000 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/parser/GeneralTokenSource.java +++ /dev/null @@ -1,18 +0,0 @@ -package ai.vespa.schemals.parser; - -/** - * Interface to capture logic needed from the different parser's generated TokenSource classes. - */ -public interface GeneralTokenSource extends CharSequence { - public String getInputSource(); - public String getText(int startOffset, int endOffset); - public String toString(); - public CharSequence subSequence(int start, int end); - public char charAt(int pos); - public int length(); - public int getCodePointColumnFromOffset(int pos); - public int getLineEndOffset(int lineNumber); - public int getLineFromOffset(int pos); - public int getLineStartOffset(int lineNumber); - public void setInputSource(String inputSource); -} diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifyDirtySchemaNodes.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifyDirtySchemaNodes.java index e56f961d715b..45ab7d408d13 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifyDirtySchemaNodes.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifyDirtySchemaNodes.java @@ -8,7 +8,6 @@ import org.eclipse.lsp4j.Range; import ai.vespa.schemals.parser.ParseException; -import ai.vespa.schemals.parser.GeneralTokenSource; import ai.vespa.schemals.parser.Token.ParseExceptionSource; import ai.vespa.schemals.schemadocument.parser.Identifier; import ai.vespa.schemals.schemadocument.parser.IdentifyDirtyNodes; @@ -46,8 +45,11 @@ public List identify(SchemaNode node) { ParseExceptionSource parseException = node.getParseExceptionSource(); if (parseException != null) { - GeneralTokenSource tokenSource = node.getTokenSource(); - Range range = CSTUtils.getRangeFromOffsets(tokenSource, parseException.beginOffset, parseException.endOffset); + Range range = CSTUtils.getRangeFromOffsets( + node.getOriginalSchemaNode().getTokenSource(), + parseException.beginOffset, + parseException.endOffset + ); String message = getParseExceptionMessage(parseException.parseException); ret.add(new SchemaDiagnostic.Builder() .setRange(range) diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolReferences.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolReferences.java index db6f236f95af..927c4f8aa572 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolReferences.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolReferences.java @@ -20,7 +20,6 @@ import ai.vespa.schemals.index.Symbol; import ai.vespa.schemals.index.Symbol.SymbolStatus; import ai.vespa.schemals.index.Symbol.SymbolType; -import ai.vespa.schemals.parser.TokenSource; import ai.vespa.schemals.parser.ast.COLON; import ai.vespa.schemals.parser.ast.FIELD; import ai.vespa.schemals.parser.ast.annotationRefDataType; @@ -301,7 +300,7 @@ private ArrayList handleFieldReference(SchemaNode identifierNode) { newEnd += subfields[i].length() + 1; identifierStr newASTNode = new identifierStr(); - newASTNode.setTokenSource((TokenSource)identifierNode.getTokenSource()); + newASTNode.setTokenSource(identifierNode.getOriginalSchemaNode().getTokenSource()); newASTNode.setBeginOffset(identifierNode.getOriginalSchemaNode().getBeginOffset()); newASTNode.setEndOffset(identifierNode.getOriginalSchemaNode().getEndOffset()); @@ -378,7 +377,7 @@ private ArrayList handleImportField(SchemaNode identifierNode) { newEnd += subfields[1].length() + 1; identifierStr newASTNode = new identifierStr(); - newASTNode.setTokenSource((TokenSource)identifierNode.getTokenSource()); + newASTNode.setTokenSource(identifierNode.getOriginalSchemaNode().getTokenSource()); newASTNode.setBeginOffset(identifierNode.getOriginalSchemaNode().getBeginOffset()); newASTNode.setEndOffset(identifierNode.getOriginalSchemaNode().getEndOffset()); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/CSTUtils.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/CSTUtils.java index 92e044fa6306..a27829a6dafa 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/CSTUtils.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/CSTUtils.java @@ -13,7 +13,6 @@ import ai.vespa.schemals.index.Symbol; import ai.vespa.schemals.index.Symbol.SymbolStatus; import ai.vespa.schemals.index.Symbol.SymbolType; -import ai.vespa.schemals.parser.GeneralTokenSource; import ai.vespa.schemals.parser.TokenSource; import ai.vespa.schemals.parser.ast.documentElm; import ai.vespa.schemals.parser.ast.functionElm; @@ -26,14 +25,14 @@ public class CSTUtils { - public static Position getPositionFromOffset(GeneralTokenSource tokenSource, int offset) { + public static Position getPositionFromOffset(TokenSource tokenSource, int offset) { int line = tokenSource.getLineFromOffset(offset) - 1; int startOfLineOffset = tokenSource.getLineStartOffset(line + 1); int column = offset - startOfLineOffset; return new Position(line, column); } - public static Range getRangeFromOffsets(GeneralTokenSource tokenSource, int beginOffset, int endOffset) { + public static Range getRangeFromOffsets(TokenSource tokenSource, int beginOffset, int endOffset) { Position begin = getPositionFromOffset(tokenSource, beginOffset); Position end = getPositionFromOffset(tokenSource, endOffset); return new Range(begin, end); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java index 5a18a688cb4d..9f5b166286c2 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/Node.java @@ -9,7 +9,6 @@ import ai.vespa.schemals.index.Symbol; import ai.vespa.schemals.index.Symbol.SymbolType; -import ai.vespa.schemals.parser.GeneralTokenSource; public abstract class Node implements Iterable { @@ -123,8 +122,6 @@ public Node getPrevious() { return parent.get(parentIndex - 1); } - public abstract GeneralTokenSource getTokenSource(); - /** * Returns the next SchemaNode of the sibilings of the node. * If there is no next node, it returns the next node of the parent node. diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java index 03ca222d9a61..a919ad1ca841 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java @@ -18,7 +18,6 @@ import ai.vespa.schemals.tree.indexinglanguage.ILUtils; import ai.vespa.schemals.tree.rankingexpression.RankNode; import ai.vespa.schemals.tree.rankingexpression.RankingExpressionUtils; -import ai.vespa.schemals.parser.GeneralTokenSource; import ai.vespa.schemals.parser.ast.expression; import ai.vespa.schemals.parser.ast.featureListElm; @@ -388,18 +387,6 @@ public ParseExceptionSource getParseExceptionSource() { return null; } - public GeneralTokenSource getTokenSource() { - if (language == LanguageType.SCHEMA) { - return originalSchemaNode.getTokenSource(); - } else if (language == LanguageType.RANK_EXPRESSION) { - return originalRankExpressionNode.getTokenSource(); - } else if (language == LanguageType.INDEXING) { - return originalIndexingNode.getTokenSource(); - } - - return null; - } - public void setRankNode(RankNode node) { rankNode = Optional.of(node); } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java index f054a9f645d3..9029c24bd420 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/YQLNode.java @@ -1,9 +1,11 @@ package ai.vespa.schemals.tree; +import java.util.ArrayList; +import java.util.List; + import org.eclipse.lsp4j.Position; import org.eclipse.lsp4j.Range; -import ai.vespa.schemals.parser.GeneralTokenSource; import ai.vespa.schemals.parser.grouping.ast.request; import ai.vespa.schemals.parser.yqlplus.Node; import ai.vespa.schemals.tree.YQL.YQLUtils; @@ -143,14 +145,4 @@ public String toString() { Position end = range.getEnd(); return "YQLNode(" + getASTClass() + ", " + start.getLine() + "," + start.getCharacter() + "->" + end.getLine() + "," + end.getCharacter() + ")"; } - - @Override - public GeneralTokenSource getTokenSource() { - if (language == LanguageType.YQLPlus) { - return originalYQLNode.getTokenSource(); - } else if (language == LanguageType.GROUPING) { - return originalGroupingNode.getTokenSource(); - } - return null; - } } From c5a827fc0ccd4a6315523de9e59c54abe5e308dc Mon Sep 17 00:00:00 2001 From: Magnus Eide-Fredriksen Date: Fri, 28 Mar 2025 14:50:22 +0100 Subject: [PATCH 4/7] chore(lsp): test for semantic token partitioning --- .../ai/vespa/schemals/common/StringUtils.java | 3 +- .../semantictokens/SchemaSemanticTokens.java | 24 +++--- .../semantictokens/YQLPlusSemanticTokens.java | 2 +- .../test/java/ai/vespa/schemals/LSPTest.java | 85 +++++++++++++++++++ .../src/test/sdfiles/single/semantictoken.sd | 25 ++++++ 5 files changed, 126 insertions(+), 13 deletions(-) create mode 100644 integration/schema-language-server/language-server/src/test/sdfiles/single/semantictoken.sd diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java index 015dc30dbab7..5ea287e471de 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java @@ -128,7 +128,7 @@ public static boolean isInsideComment(String content, Position pos) { return false; } - public static List findSingleLineComments(String content, String commentMarker, ClientLogger debug) { + public static List findSingleLineComments(String content, String commentMarker) { List commentRanges = new ArrayList<>(); if (content == null) return commentRanges; @@ -154,7 +154,6 @@ public static List findSingleLineComments(String content, String commentM commentEndOffset = content.length(); } - debug.info("Comment found at offset: " + nextComment); Position commentStart = new Position(currentLineNum, nextComment - currentLineStart); Position commentEnd = new Position(currentLineNum, commentEndOffset - currentLineStart); commentRanges.add(new Range(commentStart, commentEnd)); diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java index a643884f1048..0fe37cdee9d5 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java @@ -249,26 +249,30 @@ private static List findSemanticMarkersForSymbol(SchemaNode return ret; } - public static SemanticTokens getSemanticTokens(EventDocumentContext context) { - + /* + * Useful for testing since SemanticTokenMarker is more interpretable than the compact form + */ + public static List getSemanticTokenMarkers(EventDocumentContext context) { if (context.document == null) { - return new SemanticTokens(new ArrayList<>()); + return List.of(); } SchemaNode node = context.document.getRootNode(); if (node == null) { - return new SemanticTokens(new ArrayList<>()); + return List.of(); } List comments = SemanticTokenUtils.convertCommentRanges( - StringUtils.findSingleLineComments(context.document.getCurrentContent(), "#", context.logger) + StringUtils.findSingleLineComments(context.document.getCurrentContent(), "#") ); - List markers = traverseCST(node, context.logger); - List compactMarkers = SemanticTokenMarker.concatCompactForm( - SemanticTokenUtils.mergeSemanticTokenMarkers(markers, comments) - ); + var markers = SemanticTokenUtils.mergeSemanticTokenMarkers(traverseCST(node, context.logger), comments); + return markers; + } - return new SemanticTokens(compactMarkers); + public static SemanticTokens getSemanticTokens(EventDocumentContext context) { + return new SemanticTokens(SemanticTokenMarker.concatCompactForm( + getSemanticTokenMarkers(context) + )); } } diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java index 8b8a2c5031d1..7463935398e6 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java @@ -91,7 +91,7 @@ public static SemanticTokens getSemanticTokens(EventDocumentContext context) { } List comments = SemanticTokenUtils.convertCommentRanges( - StringUtils.findSingleLineComments(context.document.getCurrentContent(), "//", context.logger) + StringUtils.findSingleLineComments(context.document.getCurrentContent(), "//") ); List markers = traverseCST(node, context.logger); diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java index 4b1a498f53bd..d17bdd513279 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java @@ -17,10 +17,13 @@ import com.yahoo.io.IOUtils; import ai.vespa.schemals.common.ClientLogger; +import ai.vespa.schemals.context.EventDocumentContext; import ai.vespa.schemals.context.EventPositionContext; import ai.vespa.schemals.context.InvalidContextException; import ai.vespa.schemals.index.SchemaIndex; +import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenMarker; import ai.vespa.schemals.lsp.schema.definition.SchemaDefinition; +import ai.vespa.schemals.lsp.schema.semantictokens.SchemaSemanticTokens; import ai.vespa.schemals.schemadocument.DocumentManager; import ai.vespa.schemals.schemadocument.SchemaDocumentScheduler; import ai.vespa.schemals.schemadocument.parser.schema.IdentifySymbolDefinition; @@ -91,4 +94,86 @@ void definitionTest() throws IOException, InvalidContextException { assertEquals(testPair.resultRange(), result.get(0).getRange(), "Definition request returned wrong range for position " + testPair.startPosition().toString()); } } + + /* + * Test if partition of semantic tokens is as expected on a test file. + */ + @Test + void semanticTokenSchemaPartitionTest() throws IOException, InvalidContextException { + // This list is generated from running on a working build on the file tested on below. + List semanticTokenTestFileRanges = List.of( + new Range(new Position(0, 0), new Position(0, 30)), + new Range(new Position(1, 0), new Position(1, 6)), + new Range(new Position(1, 7), new Position(1, 20)), + new Range(new Position(2, 4), new Position(2, 33)), + new Range(new Position(3, 4), new Position(3, 12)), + new Range(new Position(3, 13), new Position(3, 26)), + new Range(new Position(4, 8), new Position(4, 13)), + new Range(new Position(4, 14), new Position(4, 22)), + new Range(new Position(4, 23), new Position(4, 27)), + new Range(new Position(4, 28), new Position(4, 34)), + new Range(new Position(6, 0), new Position(6, 26)), + new Range(new Position(7, 8), new Position(7, 13)), + new Range(new Position(7, 14), new Position(7, 20)), + new Range(new Position(7, 21), new Position(7, 25)), + new Range(new Position(7, 26), new Position(7, 41)), + new Range(new Position(8, 12), new Position(8, 17)), + new Range(new Position(8, 19), new Position(8, 26)), + new Range(new Position(11, 8), new Position(11, 13)), + new Range(new Position(11, 14), new Position(11, 17)), + new Range(new Position(11, 18), new Position(11, 22)), + new Range(new Position(11, 23), new Position(11, 26)), + new Range(new Position(12, 21), new Position(12, 50)), + new Range(new Position(13, 12), new Position(13, 17)), + new Range(new Position(14, 16), new Position(14, 44)), + new Range(new Position(14, 46), new Position(14, 49)), + new Range(new Position(14, 50), new Position(14, 74)), + new Range(new Position(18, 4), new Position(18, 16)), + new Range(new Position(18, 17), new Position(18, 32)), + new Range(new Position(19, 8), new Position(19, 16)), + new Range(new Position(19, 17), new Position(19, 20)), + new Range(new Position(20, 12), new Position(20, 22)), + new Range(new Position(20, 24), new Position(20, 34)), + new Range(new Position(20, 35), new Position(20, 41)), + new Range(new Position(20, 43), new Position(20, 44)), + new Range(new Position(20, 45), new Position(20, 55)), + new Range(new Position(20, 56), new Position(20, 59)), + new Range(new Position(24, 0), new Position(24, 24)) + ); + + String fileName = "src/test/sdfiles/single/semantictoken.sd"; + File file = new File(fileName); + String fileURI = file.toURI().toString(); + String fileContent = IOUtils.readFile(file); + TestSchemaMessageHandler messageHandler = new TestSchemaMessageHandler(); + ClientLogger logger = new TestLogger(messageHandler); + SchemaIndex schemaIndex = new SchemaIndex(logger); + TestSchemaDiagnosticsHandler diagnosticsHandler = new TestSchemaDiagnosticsHandler(new ArrayList<>()); + SchemaDocumentScheduler scheduler = new SchemaDocumentScheduler(logger, diagnosticsHandler, schemaIndex, messageHandler); + + scheduler.openDocument(new TextDocumentItem(fileURI, "vespaSchema", 0, fileContent)); + + + DocumentManager document = scheduler.getDocument(fileURI); + EventDocumentContext context = new EventDocumentContext( + scheduler, + schemaIndex, + messageHandler, + document.getVersionedTextDocumentIdentifier() + ); + + SchemaSemanticTokens.init(); + List computedMarkers = SchemaSemanticTokens.getSemanticTokenMarkers(context); + + assertEquals(semanticTokenTestFileRanges.size(), computedMarkers.size(), "Computed markers does not have the same size as expected."); + + for (int i = 0; i < computedMarkers.size(); ++i) { + Range expectedRange = semanticTokenTestFileRanges.get(i); + Range computedRange = computedMarkers.get(i).getRange(); + + assertEquals(expectedRange, computedRange, "If this test fails you should open " + fileURI + " with the Language Server running and inspect semantic tokens (syntax highlighting)."); + } + + scheduler.closeDocument(fileURI); + } } diff --git a/integration/schema-language-server/language-server/src/test/sdfiles/single/semantictoken.sd b/integration/schema-language-server/language-server/src/test/sdfiles/single/semantictoken.sd new file mode 100644 index 000000000000..27464c92931a --- /dev/null +++ b/integration/schema-language-server/language-server/src/test/sdfiles/single/semantictoken.sd @@ -0,0 +1,25 @@ +# Comment at beginning of file +schema semantictoken { + # Testing marking of comments + document semantictoken { + field testfile type string {} + +# comment at start of line + field totest type tensor() { + match: uncased + } + + field foo type int { + # Nested and indented comment + index { + dense-posting-list-threshold: 2.0 # comment at end of line + } + } + } + rank-profile semanticprofile { + function foo() { + expression: nativeRank(totest) + subqueries(foo) + } + } +} +# comment at end of file \ No newline at end of file From 5839435b77a9e2ccb7f370d102353e315c78cec5 Mon Sep 17 00:00:00 2001 From: Magnus Eide-Fredriksen Date: Fri, 28 Mar 2025 15:07:37 +0100 Subject: [PATCH 5/7] chore(lsp): test for YQLplus semantic token partitioning --- .../semantictokens/SchemaSemanticTokens.java | 3 +- .../semantictokens/YQLPlusSemanticTokens.java | 14 ++-- .../test/java/ai/vespa/schemals/LSPTest.java | 68 +++++++++++++++++++ .../ai/vespa/schemals/testutils/Utils.java | 1 - .../src/test/yqlfiles/semantictoken.yql | 17 +++++ 5 files changed, 94 insertions(+), 9 deletions(-) create mode 100644 integration/schema-language-server/language-server/src/test/yqlfiles/semantictoken.yql diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java index 0fe37cdee9d5..acfdda814da4 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java @@ -266,8 +266,7 @@ public static List getSemanticTokenMarkers(EventDocumentCon StringUtils.findSingleLineComments(context.document.getCurrentContent(), "#") ); - var markers = SemanticTokenUtils.mergeSemanticTokenMarkers(traverseCST(node, context.logger), comments); - return markers; + return SemanticTokenUtils.mergeSemanticTokenMarkers(traverseCST(node, context.logger), comments); } public static SemanticTokens getSemanticTokens(EventDocumentContext context) { diff --git a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java index 7463935398e6..a881723b4e05 100644 --- a/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java +++ b/integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java @@ -79,24 +79,26 @@ private static List traverseCST(YQLNode node, ClientLogger return ret; } - public static SemanticTokens getSemanticTokens(EventDocumentContext context) { - + public static List getSemanticTokenMarkers(EventDocumentContext context) { if (context.document == null) { - return new SemanticTokens(new ArrayList<>()); + return List.of(); } YQLNode node = context.document.getRootYQLNode(); if (node == null) { - return new SemanticTokens(new ArrayList<>()); + return List.of(); } List comments = SemanticTokenUtils.convertCommentRanges( StringUtils.findSingleLineComments(context.document.getCurrentContent(), "//") ); - List markers = traverseCST(node, context.logger); + return SemanticTokenUtils.mergeSemanticTokenMarkers(traverseCST(node, context.logger), comments); + } + + public static SemanticTokens getSemanticTokens(EventDocumentContext context) { List compactMarkers = SemanticTokenMarker.concatCompactForm( - SemanticTokenUtils.mergeSemanticTokenMarkers(comments, markers) + getSemanticTokenMarkers(context) ); return new SemanticTokens(compactMarkers); diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java index d17bdd513279..7cba02542d57 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java @@ -24,6 +24,7 @@ import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenMarker; import ai.vespa.schemals.lsp.schema.definition.SchemaDefinition; import ai.vespa.schemals.lsp.schema.semantictokens.SchemaSemanticTokens; +import ai.vespa.schemals.lsp.yqlplus.semantictokens.YQLPlusSemanticTokens; import ai.vespa.schemals.schemadocument.DocumentManager; import ai.vespa.schemals.schemadocument.SchemaDocumentScheduler; import ai.vespa.schemals.schemadocument.parser.schema.IdentifySymbolDefinition; @@ -176,4 +177,71 @@ void semanticTokenSchemaPartitionTest() throws IOException, InvalidContextExcept scheduler.closeDocument(fileURI); } + + @Test + void semanticTokenYQLPartitionTest() throws IOException, InvalidContextException { + List semanticTokenTestFileRanges = List.of( + new Range(new Position(0, 0), new Position(0, 31)), + new Range(new Position(1, 0), new Position(1, 39)), + new Range(new Position(2, 0), new Position(2, 6)), + new Range(new Position(2, 9), new Position(2, 13)), + new Range(new Position(2, 14), new Position(2, 22)), + new Range(new Position(2, 23), new Position(2, 28)), + new Range(new Position(2, 29), new Position(2, 33)), + new Range(new Position(2, 36), new Position(2, 39)), + new Range(new Position(3, 3), new Position(3, 19)), + new Range(new Position(4, 3), new Position(4, 8)), + new Range(new Position(4, 9), new Position(4, 13)), + new Range(new Position(5, 3), new Position(5, 31)), + new Range(new Position(6, 3), new Position(6, 8)), + new Range(new Position(6, 9), new Position(6, 10)), + new Range(new Position(6, 10), new Position(6, 15)), + new Range(new Position(7, 3), new Position(7, 48)), + new Range(new Position(8, 3), new Position(8, 7)), + new Range(new Position(9, 7), new Position(9, 13)), + new Range(new Position(9, 14), new Position(9, 19)), + new Range(new Position(12, 0), new Position(12, 30)), + new Range(new Position(14, 0), new Position(14, 20)), + new Range(new Position(15, 0), new Position(15, 6)), + new Range(new Position(15, 9), new Position(15, 13)), + new Range(new Position(15, 14), new Position(15, 17)), + new Range(new Position(15, 18), new Position(15, 23)), + new Range(new Position(15, 24), new Position(15, 28)), + new Range(new Position(16, 0), new Position(16, 16)) + ); + + String fileName = "src/test/yqlfiles/semantictoken.yql"; + File file = new File(fileName); + String fileURI = file.toURI().toString(); + String fileContent = IOUtils.readFile(file); + TestSchemaMessageHandler messageHandler = new TestSchemaMessageHandler(); + ClientLogger logger = new TestLogger(messageHandler); + SchemaIndex schemaIndex = new SchemaIndex(logger); + TestSchemaDiagnosticsHandler diagnosticsHandler = new TestSchemaDiagnosticsHandler(new ArrayList<>()); + SchemaDocumentScheduler scheduler = new SchemaDocumentScheduler(logger, diagnosticsHandler, schemaIndex, messageHandler); + + scheduler.openDocument(new TextDocumentItem(fileURI, "vespaYQL", 0, fileContent)); + + DocumentManager document = scheduler.getDocument(fileURI); + EventDocumentContext context = new EventDocumentContext( + scheduler, + schemaIndex, + messageHandler, + document.getVersionedTextDocumentIdentifier() + ); + + YQLPlusSemanticTokens.init(); + List computedMarkers = YQLPlusSemanticTokens.getSemanticTokenMarkers(context); + + assertEquals(semanticTokenTestFileRanges.size(), computedMarkers.size(), "Computed markers does not have the same size as expected."); + + for (int i = 0; i < computedMarkers.size(); ++i) { + Range expectedRange = semanticTokenTestFileRanges.get(i); + Range computedRange = computedMarkers.get(i).getRange(); + + assertEquals(expectedRange, computedRange, "If this test fails you should open " + fileURI + " with the Language Server running and inspect semantic tokens (syntax highlighting)."); + } + + scheduler.closeDocument(fileURI); + } } diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java index 75271428dd75..7ecd313b65c3 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java @@ -7,7 +7,6 @@ import org.eclipse.lsp4j.DiagnosticSeverity; import org.eclipse.lsp4j.Position; -import ai.vespa.schemals.SchemaMessageHandler; import ai.vespa.schemals.common.ClientLogger; import ai.vespa.schemals.context.ParseContext; import ai.vespa.schemals.index.SchemaIndex; diff --git a/integration/schema-language-server/language-server/src/test/yqlfiles/semantictoken.yql b/integration/schema-language-server/language-server/src/test/yqlfiles/semantictoken.yql new file mode 100644 index 000000000000..b9679e4611a9 --- /dev/null +++ b/integration/schema-language-server/language-server/src/test/yqlfiles/semantictoken.yql @@ -0,0 +1,17 @@ +// comment at beginning of file +// comments in yql parser start with // +select * from purchase where true | all( + // group by item + group(item) + // order by count descending + order(-count()) + // show the actual count value for each group + each( + output(count()) + ) +); +// comment after one statement + +// new yql statment: +select * from foo where true +// comment after \ No newline at end of file From 1e46fae8cbf2f7acfe91d512e1ea3a799646210e Mon Sep 17 00:00:00 2001 From: Magnus Eide-Fredriksen Date: Fri, 28 Mar 2025 15:39:50 +0100 Subject: [PATCH 6/7] chore(lsp): rewrite tests to reduce code duplication --- .../test/java/ai/vespa/schemals/LSPTest.java | 71 ++++++------------- .../ai/vespa/schemals/SchemaParserTest.java | 30 +++----- .../ai/vespa/schemals/testutils/Utils.java | 49 +++++++++++++ 3 files changed, 82 insertions(+), 68 deletions(-) diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java index 7cba02542d57..2b71113af123 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/LSPTest.java @@ -5,7 +5,6 @@ import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import org.eclipse.lsp4j.Location; @@ -16,23 +15,18 @@ import com.yahoo.io.IOUtils; -import ai.vespa.schemals.common.ClientLogger; import ai.vespa.schemals.context.EventDocumentContext; import ai.vespa.schemals.context.EventPositionContext; import ai.vespa.schemals.context.InvalidContextException; -import ai.vespa.schemals.index.SchemaIndex; import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenMarker; import ai.vespa.schemals.lsp.schema.definition.SchemaDefinition; import ai.vespa.schemals.lsp.schema.semantictokens.SchemaSemanticTokens; import ai.vespa.schemals.lsp.yqlplus.semantictokens.YQLPlusSemanticTokens; import ai.vespa.schemals.schemadocument.DocumentManager; -import ai.vespa.schemals.schemadocument.SchemaDocumentScheduler; import ai.vespa.schemals.schemadocument.parser.schema.IdentifySymbolDefinition; import ai.vespa.schemals.schemadocument.parser.schema.IdentifySymbolReferences; import ai.vespa.schemals.schemadocument.resolvers.SymbolReferenceResolver; -import ai.vespa.schemals.testutils.TestLogger; -import ai.vespa.schemals.testutils.TestSchemaDiagnosticsHandler; -import ai.vespa.schemals.testutils.TestSchemaMessageHandler; +import ai.vespa.schemals.testutils.Utils.TestContext; /** * LSPTest @@ -58,17 +52,14 @@ void definitionTest() throws IOException, InvalidContextException { File file = new File(fileName); String fileURI = file.toURI().toString(); String fileContent = IOUtils.readFile(file); - TestSchemaMessageHandler messageHandler = new TestSchemaMessageHandler(); - ClientLogger logger = new TestLogger(messageHandler); - SchemaIndex schemaIndex = new SchemaIndex(logger); - TestSchemaDiagnosticsHandler diagnosticsHandler = new TestSchemaDiagnosticsHandler(new ArrayList<>()); - SchemaDocumentScheduler scheduler = new SchemaDocumentScheduler(logger, diagnosticsHandler, schemaIndex, messageHandler); - scheduler.openDocument(new TextDocumentItem(fileURI, "vespaSchema", 0, fileContent)); + TestContext testContext = TestContext.create(); - assertNotEquals(null, scheduler.getDocument(fileURI), "Adding a document to the scheduler should create a DocumentManager for it."); + testContext.scheduler().openDocument(new TextDocumentItem(fileURI, "vespaSchema", 0, fileContent)); - DocumentManager document = scheduler.getDocument(fileURI); + assertNotEquals(null, testContext.scheduler().getDocument(fileURI), "Adding a document to the scheduler should create a DocumentManager for it."); + + DocumentManager document = testContext.scheduler().getDocument(fileURI); // A list of tests specific to the file read above, positions are 0-indexed. List definitionTests = List.of( @@ -82,11 +73,8 @@ void definitionTest() throws IOException, InvalidContextException { ); for (var testPair : definitionTests) { - EventPositionContext definitionContext = new EventPositionContext( - scheduler, - schemaIndex, - messageHandler, - document.getVersionedTextDocumentIdentifier(), + EventPositionContext definitionContext = testContext.getPositionContext( + document, testPair.startPosition() ); List result = SchemaDefinition.getDefinition(definitionContext); @@ -146,21 +134,14 @@ void semanticTokenSchemaPartitionTest() throws IOException, InvalidContextExcept File file = new File(fileName); String fileURI = file.toURI().toString(); String fileContent = IOUtils.readFile(file); - TestSchemaMessageHandler messageHandler = new TestSchemaMessageHandler(); - ClientLogger logger = new TestLogger(messageHandler); - SchemaIndex schemaIndex = new SchemaIndex(logger); - TestSchemaDiagnosticsHandler diagnosticsHandler = new TestSchemaDiagnosticsHandler(new ArrayList<>()); - SchemaDocumentScheduler scheduler = new SchemaDocumentScheduler(logger, diagnosticsHandler, schemaIndex, messageHandler); - scheduler.openDocument(new TextDocumentItem(fileURI, "vespaSchema", 0, fileContent)); + TestContext testContext = TestContext.create(); + testContext.scheduler().openDocument(new TextDocumentItem(fileURI, "vespaSchema", 0, fileContent)); - DocumentManager document = scheduler.getDocument(fileURI); - EventDocumentContext context = new EventDocumentContext( - scheduler, - schemaIndex, - messageHandler, - document.getVersionedTextDocumentIdentifier() + DocumentManager document = testContext.scheduler().getDocument(fileURI); + EventDocumentContext context = testContext.getDocumentContext( + document ); SchemaSemanticTokens.init(); @@ -175,7 +156,7 @@ void semanticTokenSchemaPartitionTest() throws IOException, InvalidContextExcept assertEquals(expectedRange, computedRange, "If this test fails you should open " + fileURI + " with the Language Server running and inspect semantic tokens (syntax highlighting)."); } - scheduler.closeDocument(fileURI); + testContext.scheduler().closeDocument(fileURI); } @Test @@ -214,21 +195,13 @@ void semanticTokenYQLPartitionTest() throws IOException, InvalidContextException File file = new File(fileName); String fileURI = file.toURI().toString(); String fileContent = IOUtils.readFile(file); - TestSchemaMessageHandler messageHandler = new TestSchemaMessageHandler(); - ClientLogger logger = new TestLogger(messageHandler); - SchemaIndex schemaIndex = new SchemaIndex(logger); - TestSchemaDiagnosticsHandler diagnosticsHandler = new TestSchemaDiagnosticsHandler(new ArrayList<>()); - SchemaDocumentScheduler scheduler = new SchemaDocumentScheduler(logger, diagnosticsHandler, schemaIndex, messageHandler); - - scheduler.openDocument(new TextDocumentItem(fileURI, "vespaYQL", 0, fileContent)); - - DocumentManager document = scheduler.getDocument(fileURI); - EventDocumentContext context = new EventDocumentContext( - scheduler, - schemaIndex, - messageHandler, - document.getVersionedTextDocumentIdentifier() - ); + + TestContext testContext = TestContext.create(); + + testContext.scheduler().openDocument(new TextDocumentItem(fileURI, "vespaYQL", 0, fileContent)); + + DocumentManager document = testContext.scheduler().getDocument(fileURI); + EventDocumentContext context = testContext.getDocumentContext(document); YQLPlusSemanticTokens.init(); List computedMarkers = YQLPlusSemanticTokens.getSemanticTokenMarkers(context); @@ -242,6 +215,6 @@ void semanticTokenYQLPartitionTest() throws IOException, InvalidContextException assertEquals(expectedRange, computedRange, "If this test fails you should open " + fileURI + " with the Language Server running and inspect semantic tokens (syntax highlighting)."); } - scheduler.closeDocument(fileURI); + testContext.scheduler().closeDocument(fileURI); } } diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/SchemaParserTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/SchemaParserTest.java index 1b761a7a7b3e..162f2cceb567 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/SchemaParserTest.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/SchemaParserTest.java @@ -15,15 +15,12 @@ import com.yahoo.io.IOUtils; -import ai.vespa.schemals.common.ClientLogger; import ai.vespa.schemals.common.FileUtils; import ai.vespa.schemals.context.ParseContext; -import ai.vespa.schemals.index.SchemaIndex; import ai.vespa.schemals.schemadocument.SchemaDocument; import ai.vespa.schemals.schemadocument.SchemaDocument.ParseResult; -import ai.vespa.schemals.schemadocument.SchemaDocumentScheduler; - -import ai.vespa.schemals.testutils.*; +import ai.vespa.schemals.testutils.Utils; +import ai.vespa.schemals.testutils.Utils.TestContext; public class SchemaParserTest { @@ -61,18 +58,13 @@ void checkFileFails(String fileName, int expectedErrors) throws Exception { void checkDirectoryParses(String directoryPath) throws Exception { String directoryURI = new File(directoryPath).toURI().toString(); - TestSchemaMessageHandler messageHandler = new TestSchemaMessageHandler(); - ClientLogger logger = new TestLogger(messageHandler); - SchemaIndex schemaIndex = new SchemaIndex(logger); - List diagnostics = new ArrayList<>(); - SchemaDiagnosticsHandler diagnosticsHandler = new TestSchemaDiagnosticsHandler(diagnostics); - SchemaDocumentScheduler scheduler = new SchemaDocumentScheduler(logger, diagnosticsHandler, schemaIndex, messageHandler); + TestContext testContext = TestContext.withManagedDiagnostics(diagnostics); - scheduler.setupWorkspace(URI.create(directoryURI)); + testContext.scheduler().setupWorkspace(URI.create(directoryURI)); - List schemaFiles = FileUtils.findSchemaFiles(directoryURI, logger); - List rankProfileFiles = FileUtils.findRankProfileFiles(directoryURI, logger); + List schemaFiles = FileUtils.findSchemaFiles(directoryURI, testContext.logger()); + List rankProfileFiles = FileUtils.findRankProfileFiles(directoryURI, testContext.logger()); diagnostics.clear(); @@ -80,7 +72,7 @@ void checkDirectoryParses(String directoryPath) throws Exception { int numErrors = 0; for (String rankProfileURI : rankProfileFiles) { diagnostics.clear(); - var documentHandle = scheduler.getRankProfileDocument(rankProfileURI); + var documentHandle = testContext.scheduler().getRankProfileDocument(rankProfileURI); documentHandle.reparseContent(); testMessage += "\n File: " + rankProfileURI + Utils.constructDiagnosticMessage(diagnostics, 2); @@ -89,7 +81,7 @@ void checkDirectoryParses(String directoryPath) throws Exception { for (String schemaURI : schemaFiles) { diagnostics.clear(); - var documentHandle = scheduler.getSchemaDocument(schemaURI); + var documentHandle = testContext.scheduler().getSchemaDocument(schemaURI); documentHandle.reparseContent(); testMessage += "\n File: " + schemaURI + Utils.constructDiagnosticMessage(diagnostics, 2); @@ -160,7 +152,7 @@ Stream generateGoodFileTests() { "../../../config-model/src/test/derived/structandfieldset/test.sd", "../../../config-model/src/test/derived/structanyorder/structanyorder.sd", "../../../config-model/src/test/derived/structinheritance/simple.sd", - //"../../../config-model/src/test/derived/tensor/tensor.sd", + "../../../config-model/src/test/derived/tensor/tensor.sd", "../../../config-model/src/test/derived/tokenization/tokenization.sd", "../../../config-model/src/test/derived/types/types.sd", "../../../config-model/src/test/derived/uri_array/uri_array.sd", @@ -281,7 +273,7 @@ record BadFileTestCase(String filePath, int expectedErrors) {} Stream generateBadFileTests() { BadFileTestCase[] tests = new BadFileTestCase[] { new BadFileTestCase("../../../config-model/src/test/derived/inheritfromnull/inheritfromnull.sd", 1), - new BadFileTestCase("../../../config-model/src/test/derived/structinheritance/bad.sd", 1), // TODO: check that the error is correct + new BadFileTestCase("../../../config-model/src/test/derived/structinheritance/bad.sd", 1), new BadFileTestCase("src/test/sdfiles/single/rankprofilefuncs.sd", 2), new BadFileTestCase("../../../config-model/src/test/derived/function_arguments/test.sd", 3), new BadFileTestCase("../../../config-model/src/test/derived/flickr/flickrphotos.sd", 1), @@ -306,7 +298,7 @@ Stream generateBadFileTests() { new BadFileTestCase("../../../config-model/src/test/derived/slice/test.sd", 2), // TODO: slicing? - new BadFileTestCase("../../../config-model/src/test/examples/simple.sd", 5), // TODO: unused rank-profile functions should throw errors? Also rank-type doesntexist: ... in field? + new BadFileTestCase("../../../config-model/src/test/examples/simple.sd", 5), new BadFileTestCase("src/test/sdfiles/single/rankprofilefuncs.sd", 2), new BadFileTestCase("src/test/sdfiles/single/onnxmodel.sd", 1), diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java index 7ecd313b65c3..4a5f9b7a97bb 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java @@ -8,12 +8,61 @@ import org.eclipse.lsp4j.Position; import ai.vespa.schemals.common.ClientLogger; +import ai.vespa.schemals.context.EventDocumentContext; +import ai.vespa.schemals.context.EventPositionContext; +import ai.vespa.schemals.context.InvalidContextException; import ai.vespa.schemals.context.ParseContext; import ai.vespa.schemals.index.SchemaIndex; +import ai.vespa.schemals.schemadocument.DocumentManager; import ai.vespa.schemals.schemadocument.SchemaDocumentScheduler; public class Utils { + /* + * Sets up objects and datastructures normally set up by the language server on launch. + * Replaces the client-facing classes with test stubs. + */ + public record TestContext( + TestSchemaMessageHandler messageHandler, + ClientLogger logger, + SchemaIndex schemaIndex, + TestSchemaDiagnosticsHandler diagnosticsHandler, + SchemaDocumentScheduler scheduler) { + + public static TestContext create() { + return withManagedDiagnostics(new ArrayList<>()); + } + + // Useful if produced diagnostics need to be cleared or otherwise managed by a test. + public static TestContext withManagedDiagnostics(List managedDiagnostics) { + TestSchemaMessageHandler messageHandler = new TestSchemaMessageHandler(); + TestLogger logger = new TestLogger(messageHandler); + SchemaIndex schemaIndex = new SchemaIndex(logger); + TestSchemaDiagnosticsHandler diagnosticsHandler = new TestSchemaDiagnosticsHandler(managedDiagnostics); + SchemaDocumentScheduler scheduler = new SchemaDocumentScheduler(logger, diagnosticsHandler, schemaIndex, messageHandler); + return new TestContext(messageHandler, logger, schemaIndex, diagnosticsHandler, scheduler); + } + + public EventDocumentContext getDocumentContext(DocumentManager document) throws InvalidContextException { + return new EventDocumentContext( + scheduler(), + schemaIndex(), + messageHandler(), + document.getVersionedTextDocumentIdentifier() + ); + } + + public EventPositionContext getPositionContext(DocumentManager document, Position position) throws InvalidContextException { + return new EventPositionContext( + scheduler(), + schemaIndex(), + messageHandler(), + document.getVersionedTextDocumentIdentifier(), + position + ); + } + } + public static long countErrors(List diagnostics) { return diagnostics.stream() .filter(diag -> diag.getSeverity() == DiagnosticSeverity.Error || diag.getSeverity() == null) From 02b4cb13fcd610107b70c5c1df9c5614dba620df Mon Sep 17 00:00:00 2001 From: Magnus Eide-Fredriksen Date: Fri, 28 Mar 2025 15:47:22 +0100 Subject: [PATCH 7/7] chore(lsp): remove unused, old, hover test --- .../java/ai/vespa/schemals/HoverTest.java | 96 ------------------- .../ai/vespa/schemals/testutils/Utils.java | 2 +- .../src/test/sdfiles/single/hover.sd | 10 -- 3 files changed, 1 insertion(+), 107 deletions(-) delete mode 100644 integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/HoverTest.java diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/HoverTest.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/HoverTest.java deleted file mode 100644 index d89aa6405493..000000000000 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/HoverTest.java +++ /dev/null @@ -1,96 +0,0 @@ -package ai.vespa.schemals; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import org.eclipse.lsp4j.Hover; -import org.eclipse.lsp4j.Position; -import org.eclipse.lsp4j.TextDocumentIdentifier; -import org.eclipse.lsp4j.TextDocumentItem; -import org.junit.jupiter.api.Test; - -import com.yahoo.io.IOUtils; - -import ai.vespa.schemals.testutils.*; - -import ai.vespa.schemals.common.ClientLogger; -import ai.vespa.schemals.common.FileUtils; -import ai.vespa.schemals.context.ParseContext; -import ai.vespa.schemals.index.SchemaIndex; -import ai.vespa.schemals.lsp.schema.hover.SchemaHover; -import ai.vespa.schemals.schemadocument.SchemaDocument; -import ai.vespa.schemals.schemadocument.SchemaDocument.ParseResult; -import ai.vespa.schemals.schemadocument.SchemaDocumentScheduler; - -import ai.vespa.schemals.tree.CSTUtils; -import ai.vespa.schemals.tree.SchemaNode; - -import ai.vespa.schemals.context.EventPositionContext; -import ai.vespa.schemals.schemadocument.DocumentManager; - - -/** - * HoverTest - */ -public class HoverTest { - @Test - public void hoverTest() throws IOException { - // TODO: uncomment this test and fix hover when new HTML parsing is done - /* - String fileName = "src/test/sdfiles/single/hover.sd"; - File file = new File(fileName); - String fileURI = file.toURI().toString(); - String fileContent = IOUtils.readFile(file); - SchemaMessageHandler messageHandler = new TestSchemaMessageHandler(); - ClientLogger logger = new TestLogger(messageHandler); - SchemaIndex schemaIndex = new SchemaIndex(logger); - TestSchemaDiagnosticsHandler diagnosticsHandler = new TestSchemaDiagnosticsHandler(new ArrayList<>()); - SchemaDocumentScheduler scheduler = new SchemaDocumentScheduler(logger, diagnosticsHandler, schemaIndex, messageHandler); - - scheduler.openDocument(new TextDocumentItem(fileURI, "vespaSchema", 0, fileContent)); - - assertNotEquals(null, scheduler.getDocument(fileURI), "Adding a document to the scheduler should create a DocumentManager for it."); - - DocumentManager document = scheduler.getDocument(fileURI); - - assertNotEquals(null, document.getRootNode(), "Parsing the hover file should be successful!"); - - List lines = fileContent.lines().toList(); - - for (int lineNum = 0; lineNum < lines.size(); ++lineNum) { - int nonWhiteSpace = 0; - - String line = lines.get(lineNum); - - for (nonWhiteSpace = 0; nonWhiteSpace < line.length(); ++nonWhiteSpace) { - if (!Character.isWhitespace(line.charAt(nonWhiteSpace)))break; - } - - if (nonWhiteSpace == line.length()) continue; - - if (line.charAt(nonWhiteSpace) == '}') continue; - - Position hoverPosition = new Position(lineNum, nonWhiteSpace); - - EventPositionContext hoverContext = new EventPositionContext( - scheduler, - schemaIndex, - messageHandler, - document.getVersionedTextDocumentIdentifier(), - hoverPosition - ); - - Hover hoverResult = SchemaHover.getHover(hoverContext); - - assertNotEquals(null, hoverResult, - "Failed to get hover information for " + fileName + ", line " + (lineNum+1) + ": " + line.substring(nonWhiteSpace)); - } - */ - } -} diff --git a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java index 4a5f9b7a97bb..1d8af731dc23 100644 --- a/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java +++ b/integration/schema-language-server/language-server/src/test/java/ai/vespa/schemals/testutils/Utils.java @@ -20,7 +20,7 @@ public class Utils { /* * Sets up objects and datastructures normally set up by the language server on launch. - * Replaces the client-facing classes with test stubs. + * Replaces the client-facing classes with test stubs. */ public record TestContext( TestSchemaMessageHandler messageHandler, diff --git a/integration/schema-language-server/language-server/src/test/sdfiles/single/hover.sd b/integration/schema-language-server/language-server/src/test/sdfiles/single/hover.sd index fad130363ad8..c1ce7f15cc09 100644 --- a/integration/schema-language-server/language-server/src/test/sdfiles/single/hover.sd +++ b/integration/schema-language-server/language-server/src/test/sdfiles/single/hover.sd @@ -1,7 +1,6 @@ schema hover { document hover { field name type weightedset { - alias: title attribute: fast-access bolding: on attribute { @@ -70,15 +69,6 @@ schema hover { index test { lower-bound: 2 } - onnx-model name { - file: a - input "input": a - output "output": b - gpu-device: 0 - intraop-threads: 1 - interop-threads: 2 - execution-mode: parallel - } rank-profile prof { constants { a: 0