Migrate go language module to new framework

language-antlr-utils/src/main/java/de/jplag/antlr/AbstractVisitor.java

@@ -25,7 +25,7 @@ public abstract class AbstractVisitor<T> {
 
     private final Predicate<T> condition;
     private final List<Consumer<HandlerData<T>>> entryHandlers;
-    private TokenType entryTokenType;
+    protected TokenType entryTokenType;
     private Function<T, CodeSemantics> entrySemantics;
 
     /**
@@ -118,15 +118,24 @@ boolean matches(T entity) {
      * Enter a given entity, injecting the needed dependencies.
      */
     void enter(HandlerData<T> data) {
+        handleEnter(data, this::extractEnterToken, this::extractEnterToken);
+    }
+
+    protected void handleEnter(HandlerData<T> data, Function<T, Token> extractStartToken, Function<T, Token> extractEndToken) {
         if (entryTokenType == null && entrySemantics != null) {
             logger.warn("Received semantics, but no token type, so no token was generated and the semantics discarded");
         }
-        addToken(data, entryTokenType, entrySemantics, this::extractEnterToken);  // addToken takes null token types
+        addToken(data, entryTokenType, entrySemantics, extractStartToken, extractEndToken);  // addToken takes null token types
         entryHandlers.forEach(handler -> handler.accept(data));
     }
 
     void addToken(HandlerData<T> data, TokenType tokenType, Function<T, CodeSemantics> semantics, Function<T, Token> extractToken) {
-        data.collector().addToken(tokenType, semantics, data.entity(), extractToken, data.variableRegistry());
+        addToken(data, tokenType, semantics, extractToken, extractToken);
+    }
+
+    void addToken(HandlerData<T> data, TokenType tokenType, Function<T, CodeSemantics> semantics, Function<T, Token> extractStartToken,
+            Function<T, Token> extractEndToken) {
+        data.collector().addToken(tokenType, semantics, data.entity(), extractStartToken, extractEndToken, data.variableRegistry());
     }
 
     abstract Token extractEnterToken(T entity);

language-antlr-utils/src/main/java/de/jplag/antlr/ContextDelegateVisitor.java

@@ -0,0 +1,19 @@
+package de.jplag.antlr;
+
+import java.util.function.Function;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+
+public class ContextDelegateVisitor<T, V extends ParserRuleContext> extends DelegateVisitor<T, V> {
+    private final ContextVisitor<V> contextVisitor;
+
+    public ContextDelegateVisitor(ContextVisitor<V> delegate, Function<T, V> mapper) {
+        super(delegate, mapper);
+        this.contextVisitor = delegate;
+    }
+
+    @Override
+    public void delegateExit(HandlerData<T> parentData) {
+        this.contextVisitor.exit(parentData.derive(this.mapper));
+    }
+}

language-antlr-utils/src/main/java/de/jplag/antlr/ContextVisitor.java

@@ -10,6 +10,7 @@
 
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.tree.TerminalNode;
 
 import de.jplag.TokenType;
 import de.jplag.semantics.CodeSemantics;
@@ -23,10 +24,15 @@ public class ContextVisitor<T extends ParserRuleContext> extends AbstractVisitor
     private final List<Consumer<HandlerData<T>>> exitHandlers;
     private TokenType exitTokenType;
     private Function<T, CodeSemantics> exitSemantics;
+    private boolean lengthAsRange;
+
+    private DelegateVisitor<T, ?> delegate;
 
     ContextVisitor(Predicate<T> condition) {
         super(condition);
         this.exitHandlers = new ArrayList<>();
+        this.lengthAsRange = false;
+        this.delegate = null;
     }
 
     /**
@@ -59,6 +65,53 @@ public ContextVisitor<T> mapExit(TokenType tokenType) {
         return this;
     }
 
+    /**
+     * Behaves like mapEnter, but the created token will range from the beginning of this context to the end instead of only
+     * marking the beginning.
+     * @param tokenType The type of token to crate
+     * @return Self
+     */
+    public AbstractVisitor<T> mapRange(TokenType tokenType) {
+        this.entryTokenType = tokenType;
+        this.lengthAsRange = true;
+        return this;
+    }
+
+    /**
+     * Delegates calls to this visitor to a derived visitor. The mapper function is used to determine the delegated token.
+     * This invalidated all mapping happening inside this visitor. You need to configure the new visitor to do so.
+     * @param mapper The mapper function
+     */
+    public TerminalVisitor delegateTerminal(Function<T, TerminalNode> mapper) {
+        TerminalVisitor delegateVisitor = new TerminalVisitor(ignore -> true);
+        this.delegate = new DelegateVisitor<>(delegateVisitor, parentData -> mapper.apply(parentData).getSymbol());
+        return delegateVisitor;
+    }
+
+    /**
+     * Delegates calls to this visitor to a derived visitor. The mapper function is used to determine the delegated token.
+     * This invalidated all mapping happening inside this visitor. You need to configure the new visitor to do so. Visits
+     * the terminal upon exiting this context
+     * @param mapper The mapper function
+     */
+    public TerminalVisitor delegateTerminalExit(Function<T, TerminalNode> mapper) {
+        TerminalVisitor delegateVisitor = new TerminalVisitor(ignore -> true);
+        this.delegate = new DelegateVisitor<>(delegateVisitor, parentData -> mapper.apply(parentData).getSymbol());
+        this.delegate.mapOnExit();
+        return delegateVisitor;
+    }
+
+    /**
+     * Delegates calls to this visitor to a derived visitor. The mapper function is used to determine the delegated token.
+     * This invalidated all mapping happening inside this visitor. You need to configure the new visitor to do so.
+     * @param mapper The mapper function
+     */
+    public <V extends ParserRuleContext> ContextVisitor<V> delegateContext(Function<T, V> mapper) {
+        ContextVisitor<V> visitor = new ContextVisitor<>(ignore -> true);
+        this.delegate = new ContextDelegateVisitor<>(visitor, mapper);
+        return visitor;
+    }
+
     /**
      * Tell the visitor that it should generate a token upon entering and one upon exiting the entity. Should only be
      * invoked once per visitor.
@@ -132,12 +185,40 @@ public ContextVisitor<T> addClassScope() {
      * Exit a given entity, injecting the needed dependencies.
      */
     void exit(HandlerData<T> data) {
+        if (this.delegate != null) {
+            this.delegate.delegateExit(data);
+            return;
+        }
+
         addToken(data, exitTokenType, exitSemantics, ParserRuleContext::getStop);
         exitHandlers.forEach(handler -> handler.accept(data));
     }
 
+    @Override
+    void enter(HandlerData<T> data) {
+        if (this.delegate != null) {
+            this.delegate.delegateEnter(data);
+            return;
+        }
+
+        if (this.lengthAsRange) {
+            this.handleEnter(data, this::extractEnterToken, ParserRuleContext::getStop);
+        } else {
+            super.enter(data);
+        }
+    }
+
     @Override
     Token extractEnterToken(T entity) {
         return entity.getStart();
     }
+
+    @Override
+    boolean matches(T entity) {
+        if (this.delegate != null && !this.delegate.isPresent(entity)) {
+            return false;
+        }
+
+        return super.matches(entity);
+    }
 }

language-antlr-utils/src/main/java/de/jplag/antlr/DelegateVisitor.java

@@ -0,0 +1,39 @@
+package de.jplag.antlr;
+
+import java.util.function.Function;
+
+public class DelegateVisitor<T, V> {
+    private final AbstractVisitor<V> delegate;
+    protected final Function<T, V> mapper;
+    private boolean mapOnExit;
+
+    public DelegateVisitor(AbstractVisitor<V> delegate, Function<T, V> mapper) {
+        this.delegate = delegate;
+        this.mapper = mapper;
+        this.mapOnExit = false;
+    }
+
+    public void delegateEnter(HandlerData<T> parentData) {
+        if (!this.mapOnExit) {
+            this.delegate.enter(parentData.derive(this.mapper));
+        }
+    }
+
+    public void mapOnExit() {
+        this.mapOnExit = true;
+    }
+
+    public void delegateExit(HandlerData<T> parentData) {
+        if (this.mapOnExit) {
+            this.delegate.enter(parentData.derive(this.mapper));
+        }
+    }
+
+    public boolean isPresent(T entity) {
+        try {
+            return this.mapper.apply(entity) != null;
+        } catch (Exception e) { // If something goes wrong during mapping, the delegate is not present
+            return false;
+        }
+    }
+}

language-antlr-utils/src/main/java/de/jplag/antlr/HandlerData.java

@@ -1,9 +1,14 @@
 package de.jplag.antlr;
 
+import java.util.function.Function;
+
 import de.jplag.semantics.VariableRegistry;
 
 /**
  * Holds the data passed to the (quasi-static) listeners.
  */
-record HandlerData<T>(T entity, VariableRegistry variableRegistry, TokenCollector collector) {
+public record HandlerData<T>(T entity, VariableRegistry variableRegistry, TokenCollector collector) {
+    public <V> HandlerData<V> derive(Function<T, V> mapper) {
+        return new HandlerData<>(mapper.apply(entity), variableRegistry, collector);
+    }
 }

language-antlr-utils/src/main/java/de/jplag/antlr/TokenCollector.java

@@ -40,14 +40,16 @@ List<Token> getTokens() {
     }
 
     <T> void addToken(TokenType jplagType, Function<T, CodeSemantics> semanticsSupplier, T entity,
-            Function<T, org.antlr.v4.runtime.Token> extractToken, VariableRegistry variableRegistry) {
+            Function<T, org.antlr.v4.runtime.Token> extractStartToken, Function<T, org.antlr.v4.runtime.Token> extractEndToken,
+            VariableRegistry variableRegistry) {
         if (jplagType == null) {
             return;
         }
-        org.antlr.v4.runtime.Token antlrToken = extractToken.apply(entity);
+        org.antlr.v4.runtime.Token antlrToken = extractStartToken.apply(entity);
+        org.antlr.v4.runtime.Token antlrEndToken = extractEndToken.apply(entity);
         int line = antlrToken.getLine();
         int column = antlrToken.getCharPositionInLine() + 1;
-        int length = antlrToken.getText().length();
+        int length = (antlrEndToken.getStopIndex() - antlrToken.getStartIndex()) + 1;
         Token token;
         if (extractsSemantics) {
             if (semanticsSupplier == null) {

languages/golang/pom.xml

@@ -14,6 +14,11 @@
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
         </dependency>
+        <dependency>
+            <groupId>de.jplag</groupId>
+            <artifactId>language-antlr-utils</artifactId>
+            <version>${revision}</version>
+        </dependency>
     </dependencies>
 
     <build>

languages/golang/src/main/java/de/jplag/golang/GoLanguage.java

@@ -1,25 +1,18 @@
 package de.jplag.golang;
 
-import java.io.File;
-import java.util.List;
-import java.util.Set;
-
 import org.kohsuke.MetaInfServices;
 
-import de.jplag.ParsingException;
-import de.jplag.Token;
+import de.jplag.antlr.AbstractAntlrLanguage;
 
 @MetaInfServices(de.jplag.Language.class)
-public class GoLanguage implements de.jplag.Language {
-
+public class GoLanguage extends AbstractAntlrLanguage {
     private static final String NAME = "Go Parser";
     private static final String IDENTIFIER = "go";
     private static final int DEFAULT_MIN_TOKEN_MATCH = 8;
     private static final String[] FILE_EXTENSIONS = {".go"};
-    private final GoParserAdapter parserAdapter;
 
     public GoLanguage() {
-        this.parserAdapter = new GoParserAdapter();
+        super(new GoParserAdapter());
     }
 
     @Override
@@ -41,9 +34,4 @@ public String getIdentifier() {
     public int minimumTokenMatch() {
         return DEFAULT_MIN_TOKEN_MATCH;
     }
-
-    @Override
-    public List<Token> parse(Set<File> files, boolean normalize) throws ParsingException {
-        return parserAdapter.parse(files);
-    }
 }