Magnus/lsp yql comments

integration/schema-language-server/language-server/src/main/ccc/grouping/GroupingParser.ccc

@@ -47,7 +47,6 @@ INJECT GroupingParserLexer:
     }
 }
 
-
 TOKEN :
     <INTEGER: <DECIMAL> (["l","L"])? | <HEX> (["l","L"])? | <OCTAL> (["l","L"])?> |
         <#DECIMAL: ["1"-"9"] (["0"-"9"])*> |
@@ -179,6 +178,9 @@ TOKEN :
     <IDENTIFIER: ["A"-"Z","a"-"z"](["A"-"Z","a"-"z","0"-"9","_","@"])*>
 ;
 
+UNPARSED :
+  <COMMENT: (('//'|'#')(~['\n','\r'])* ('\r')? ('\n')?) | ( '/*' (~[])* '*/' ) >
+;
 
 // --------------------------------------------------------------------------------
 //

integration/schema-language-server/language-server/src/main/ccc/yqlplus/YQLPlus.ccc

@@ -125,7 +125,7 @@ TOKEN :
 ;
 
 UNPARSED :
-  <COMMENT: ('//' (~['\n','\r'])* ('\r')? ('\n')?) | ( '/*' (~[])* '*/' ) >
+  <COMMENT: (('//'|'#') (~['\n','\r'])* ('\r')? ('\n')?) | ( '/*' (~[])* '*/' ) >
 ;
 
 // --------------------------------------------------------------------------------

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/StringUtils.java

@@ -1,12 +1,13 @@
 package ai.vespa.schemals.common;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import org.eclipse.lsp4j.Position;
 import org.eclipse.lsp4j.Range;
 
+import ai.vespa.schemals.tree.CSTUtils;
 import ai.vespa.schemals.tree.Node;
-import ai.vespa.schemals.tree.SchemaNode;
 
 /**
  * StringUtils
@@ -127,6 +128,42 @@ public static boolean isInsideComment(String content, Position pos) {
         return false;
     }
 
+    public static List<Range> findSingleLineComments(String content, String commentMarker) {
+        List<Range> commentRanges = new ArrayList<>();
+        if (content == null) return commentRanges;
+
+
+        int currentLineNum = 0;
+        int currentLineStart = 0;
+        int nextComment = content.indexOf(commentMarker);
+        int nextNewLine = content.indexOf('\n');
+
+        // Iterate the text while keeping track of line offset information
+        while (nextComment != -1) {
+            // Consume lines before the comment
+            if (nextNewLine != -1 && nextNewLine < nextComment) {
+                currentLineStart = nextNewLine + 1;
+                nextNewLine = content.indexOf('\n', nextNewLine + 1);
+                currentLineNum++;
+                continue;
+            }
+
+            int commentEndOffset = nextNewLine;
+
+            if (nextNewLine == -1) {
+                commentEndOffset = content.length();
+            }
+
+            Position commentStart = new Position(currentLineNum, nextComment - currentLineStart);
+            Position commentEnd   = new Position(currentLineNum, commentEndOffset - currentLineStart);
+            commentRanges.add(new Range(commentStart, commentEnd));
+
+            nextComment = content.indexOf(commentMarker, commentEndOffset + 1);
+        }
+
+        return commentRanges;
+    }
+
     public static Position getStringPosition(String str) {
         int lines = 0;
         int column = str.length();

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/common/semantictokens/SemanticTokenUtils.java

@@ -0,0 +1,53 @@
+package ai.vespa.schemals.lsp.common.semantictokens;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.eclipse.lsp4j.Position;
+import org.eclipse.lsp4j.Range;
+
+import ai.vespa.schemals.tree.CSTUtils;
+
+public class SemanticTokenUtils {
+    public static List<SemanticTokenMarker> convertCommentRanges(List<Range> commentRanges) {
+        int commentTokenType = CommonSemanticTokens.getType("comment");
+        return commentRanges.stream()
+                            .map(range -> new SemanticTokenMarker(commentTokenType, range))
+                            .collect(Collectors.toList());
+    }
+
+    // This function assumes that both of the lists are sorted, and that no elements are overlapping
+    public static List<SemanticTokenMarker> mergeSemanticTokenMarkers(List<SemanticTokenMarker> lhs, List<SemanticTokenMarker> rhs) {
+        List<SemanticTokenMarker> ret = new ArrayList<>(lhs.size() + rhs.size());
+
+        int lhsIndex = 0;
+        int rhsIndex = 0;
+        while (
+            lhsIndex < lhs.size() &&
+            rhsIndex < rhs.size()
+        ) {
+            Position rhsPos = rhs.get(rhsIndex).getRange().getStart();
+            Position lhsPos = lhs.get(lhsIndex).getRange().getStart();
+
+            if (CSTUtils.positionLT(lhsPos, rhsPos)) {
+                ret.add(lhs.get(lhsIndex));
+                lhsIndex++;
+            } else {
+                ret.add(rhs.get(rhsIndex));
+                rhsIndex++;
+            }
+        }
+
+        for (int i = lhsIndex; i < lhs.size(); i++) {
+            ret.add(lhs.get(i));
+        }
+
+        for (int i = rhsIndex; i < rhs.size(); i++) {
+            ret.add(rhs.get(i));
+        }
+
+        return ret;
+    }
+
+}

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/semantictokens/SchemaSemanticTokens.java

@@ -7,24 +7,23 @@
 import java.util.Map;
 import java.util.Set;
 
-import org.eclipse.lsp4j.Position;
 import org.eclipse.lsp4j.Range;
 import org.eclipse.lsp4j.SemanticTokenModifiers;
 import org.eclipse.lsp4j.SemanticTokenTypes;
 import org.eclipse.lsp4j.SemanticTokens;
 
-import ai.vespa.schemals.tree.CSTUtils;
 import ai.vespa.schemals.tree.Node;
 import ai.vespa.schemals.tree.SchemaNode;
 import ai.vespa.schemals.tree.Node.LanguageType;
 import ai.vespa.schemals.common.ClientLogger;
+import ai.vespa.schemals.common.StringUtils;
 import ai.vespa.schemals.context.EventDocumentContext;
 import ai.vespa.schemals.index.Symbol.SymbolStatus;
 import ai.vespa.schemals.index.Symbol.SymbolType;
 import ai.vespa.schemals.lsp.common.semantictokens.CommonSemanticTokens;
 import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenMarker;
+import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenUtils;
 import ai.vespa.schemals.parser.Token.TokenType;
-import ai.vespa.schemals.parser.TokenSource;
 import ai.vespa.schemals.parser.ast.FILTER;
 import ai.vespa.schemals.parser.ast.RANK_TYPE;
 import ai.vespa.schemals.parser.ast.bool;
@@ -250,98 +249,29 @@ private static List<SemanticTokenMarker> findSemanticMarkersForSymbol(SchemaNode
         return ret;
     }
 
-    private static ArrayList<SemanticTokenMarker> convertCommentRanges(ArrayList<Range> comments) {
-        ArrayList<SemanticTokenMarker> ret = new ArrayList<>();
-
-        int tokenType = CommonSemanticTokens.getType("comment");
-
-        for (Range range : comments) {
-            ret.add(new SemanticTokenMarker(tokenType, range));
-        }
-
-        return ret;
-    }
-
-    private static ArrayList<SemanticTokenMarker> findComments(SchemaNode rootNode) {
-        TokenSource tokenSource = rootNode.getTokenSource();
-        String source = tokenSource.toString();
-        ArrayList<Range> ret = new ArrayList<>();
-
-        int index = source.indexOf("#");
-        while (index >= 0) {
-            Position start = CSTUtils.getPositionFromOffset(tokenSource, index);
-            if (CSTUtils.getLeafNodeAtPosition(rootNode, start) != null) {
-                index = source.indexOf("#", index + 1);
-                continue;
-            }
-
-            index = source.indexOf("\n", index + 1);
-
-            if (index < 0) {
-                index = source.length() - 1;
-            }
-
-            Position end = CSTUtils.getPositionFromOffset(tokenSource, index);
-
-            ret.add(new Range(start, end));
-
-            index = source.indexOf("#", index + 1);
-        }
-
-        return convertCommentRanges(ret);
-    }
-
-    // This function assumes that both of the lists are sorted, and that no elements are overlapping
-    private static ArrayList<SemanticTokenMarker> mergeSemanticTokenMarkers(ArrayList<SemanticTokenMarker> lhs, ArrayList<SemanticTokenMarker> rhs) {
-        ArrayList<SemanticTokenMarker> ret = new ArrayList<>(lhs.size() + rhs.size());
-
-        int lhsIndex = 0;
-        int rhsIndex = 0;
-        while (
-            lhsIndex < lhs.size() &&
-            rhsIndex < rhs.size()
-        ) {
-            Position rhsPos = rhs.get(rhsIndex).getRange().getStart();
-            Position lhsPos = lhs.get(lhsIndex).getRange().getStart();
-
-            if (CSTUtils.positionLT(lhsPos, rhsPos)) {
-                ret.add(lhs.get(lhsIndex));
-                lhsIndex++;
-            } else {
-                ret.add(rhs.get(rhsIndex));
-                rhsIndex++;
-            }
-        }
-
-        for (int i = lhsIndex; i < lhs.size(); i++) {
-            ret.add(lhs.get(i));
-        }
-
-        for (int i = rhsIndex; i < rhs.size(); i++) {
-            ret.add(rhs.get(i));
-        }
-
-        return ret;
-    }
-
-    public static SemanticTokens getSemanticTokens(EventDocumentContext context) {
-
+    /*
+     * Useful for testing since SemanticTokenMarker is more interpretable than the compact form
+     */
+    public static List<SemanticTokenMarker> getSemanticTokenMarkers(EventDocumentContext context) {
         if (context.document == null) {
-            return new SemanticTokens(new ArrayList<>());
+            return List.of();
         }
 
         SchemaNode node = context.document.getRootNode();
         if (node == null) {
-            return new SemanticTokens(new ArrayList<>());
+            return List.of();
         }
 
-        ArrayList<SemanticTokenMarker> comments = findComments(context.document.getRootNode());
-
-        ArrayList<SemanticTokenMarker> markers = traverseCST(node, context.logger);
-        List<Integer> compactMarkers = SemanticTokenMarker.concatCompactForm(
-            mergeSemanticTokenMarkers(markers, comments)
+        List<SemanticTokenMarker> comments = SemanticTokenUtils.convertCommentRanges(
+            StringUtils.findSingleLineComments(context.document.getCurrentContent(), "#")
         );
 
-        return new SemanticTokens(compactMarkers);
+        return SemanticTokenUtils.mergeSemanticTokenMarkers(traverseCST(node, context.logger), comments);
+    }
+
+    public static SemanticTokens getSemanticTokens(EventDocumentContext context) {
+        return new SemanticTokens(SemanticTokenMarker.concatCompactForm(
+            getSemanticTokenMarkers(context)
+        ));
     }
 }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/yqlplus/semantictokens/YQLPlusSemanticTokens.java

@@ -7,13 +7,15 @@
 import org.eclipse.lsp4j.SemanticTokens;
 
 import ai.vespa.schemals.common.ClientLogger;
+import ai.vespa.schemals.common.StringUtils;
 import ai.vespa.schemals.context.EventDocumentContext;
 import ai.vespa.schemals.lsp.common.semantictokens.CommonSemanticTokens;
 import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenMarker;
+import ai.vespa.schemals.lsp.common.semantictokens.SemanticTokenUtils;
 import ai.vespa.schemals.parser.yqlplus.ast.pipeline_step;
 import ai.vespa.schemals.tree.Node;
-import ai.vespa.schemals.tree.YQLNode;
 import ai.vespa.schemals.tree.Node.LanguageType;
+import ai.vespa.schemals.tree.YQLNode;
 
 public class YQLPlusSemanticTokens {
 
@@ -77,19 +79,27 @@ private static List<SemanticTokenMarker> traverseCST(YQLNode node, ClientLogger
         return ret;
     }
 
-    public static SemanticTokens getSemanticTokens(EventDocumentContext context) {
-
+    public static List<SemanticTokenMarker> getSemanticTokenMarkers(EventDocumentContext context) {
         if (context.document == null) {
-            return new SemanticTokens(new ArrayList<>());
+            return List.of();
         }
 
         YQLNode node = context.document.getRootYQLNode();
         if (node == null) {
-            return new SemanticTokens(new ArrayList<>());
+            return List.of();
         }
 
-        List<SemanticTokenMarker> markers = traverseCST(node, context.logger);
-        List<Integer> compactMarkers = SemanticTokenMarker.concatCompactForm(markers);
+        List<SemanticTokenMarker> comments = SemanticTokenUtils.convertCommentRanges(
+            StringUtils.findSingleLineComments(context.document.getCurrentContent(), "//")
+        );
+
+        return SemanticTokenUtils.mergeSemanticTokenMarkers(traverseCST(node, context.logger), comments);
+    }
+
+    public static SemanticTokens getSemanticTokens(EventDocumentContext context) {
+        List<Integer> compactMarkers = SemanticTokenMarker.concatCompactForm(
+            getSemanticTokenMarkers(context)
+        );
 
         return new SemanticTokens(compactMarkers);
     }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifyDirtySchemaNodes.java

@@ -8,7 +8,6 @@
 import org.eclipse.lsp4j.Range;
 
 import ai.vespa.schemals.parser.ParseException;
-import ai.vespa.schemals.parser.TokenSource;
 import ai.vespa.schemals.parser.Token.ParseExceptionSource;
 import ai.vespa.schemals.schemadocument.parser.Identifier;
 import ai.vespa.schemals.schemadocument.parser.IdentifyDirtyNodes;
@@ -46,8 +45,11 @@ public List<Diagnostic> identify(SchemaNode node) {
         ParseExceptionSource parseException = node.getParseExceptionSource();
 
         if (parseException != null) {
-            TokenSource tokenSource = node.getTokenSource();
-            Range range = CSTUtils.getRangeFromOffsets(tokenSource, parseException.beginOffset, parseException.endOffset);
+            Range range = CSTUtils.getRangeFromOffsets(
+                node.getOriginalSchemaNode().getTokenSource(), 
+                parseException.beginOffset, 
+                parseException.endOffset
+            );
             String message = getParseExceptionMessage(parseException.parseException);
             ret.add(new SchemaDiagnostic.Builder()
                 .setRange(range)

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/parser/schema/IdentifySymbolReferences.java

@@ -300,7 +300,7 @@ private ArrayList<Diagnostic> handleFieldReference(SchemaNode identifierNode) {
             newEnd += subfields[i].length() + 1;
 
             identifierStr newASTNode = new identifierStr();
-            newASTNode.setTokenSource(identifierNode.getTokenSource());
+            newASTNode.setTokenSource(identifierNode.getOriginalSchemaNode().getTokenSource());
             newASTNode.setBeginOffset(identifierNode.getOriginalSchemaNode().getBeginOffset());
             newASTNode.setEndOffset(identifierNode.getOriginalSchemaNode().getEndOffset());
 
@@ -377,7 +377,7 @@ private ArrayList<Diagnostic> handleImportField(SchemaNode identifierNode) {
         newEnd += subfields[1].length() + 1;
 
         identifierStr newASTNode = new identifierStr();
-        newASTNode.setTokenSource(identifierNode.getTokenSource());
+        newASTNode.setTokenSource(identifierNode.getOriginalSchemaNode().getTokenSource());
         newASTNode.setBeginOffset(identifierNode.getOriginalSchemaNode().getBeginOffset());
         newASTNode.setEndOffset(identifierNode.getOriginalSchemaNode().getEndOffset());
 

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/CSTUtils.java

@@ -43,8 +43,7 @@ public static Range getNodeRange(ai.vespa.schemals.parser.Node node) {
         try {
             return getRangeFromOffsets(tokenSource, node.getBeginOffset(), node.getEndOffset());
         } catch(Exception e) {
-            // TODO: something happens when offsets are bad
-        }        
+        }
         return new Range(new Position(0, 0), new Position(0, 0));
     }
 

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/tree/SchemaNode.java

@@ -387,14 +387,6 @@ public ParseExceptionSource getParseExceptionSource() {
         return null;
     }
 
-    public TokenSource getTokenSource() {
-        if (language == LanguageType.SCHEMA) {
-            return originalSchemaNode.getTokenSource();
-        }
-
-        return null;
-    }
-
     public void setRankNode(RankNode node) {
         rankNode = Optional.of(node);
     }