phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Start removing TokenManager wrappers

Browse Source
This commit is contained in:
Clément Fournier 2020-03-20 16:32:09 +01:00
parent e9b65823bf
commit 368573e7c7
5 changed files with 51 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
javacc-wrapper.xml
View File

@ -359,7 +359,7 @@ public final class ${token-constants-name} \{${line.separator}
<replace>
<replacetoken>/** Literal token values. */</replacetoken>
<replacevalue>
<replacevalue><![CDATA[
/** Returns a string describing the given token kind.
* Returns null if the kind is unknown.
*
@ -369,16 +369,31 @@ public final class ${token-constants-name} \{${line.separator}
*/
public static @org.checkerframework.checker.nullness.qual.Nullable
String describe(int kind) {
return kind &lt; 0 || kind >= tokenImage.length
return kind < 0 || kind >= tokenImage.length
? null
: tokenImage[kind];
}
public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.CharStream cs) {
return new %%%TOKEN_MGR_NAME%%%(cs);
}
private static final
</replacevalue>
]]> </replacevalue>
<fileset file="${token-constants-file}" />
</replace>
<replaceregexp>
<regexp pattern="%%%TOKEN_MGR_NAME%%%" />
<substitution expression="${tokenmgr-name}" />
<fileset file="${token-constants-file}" />
</replaceregexp>
<replaceregexp flags="g">
<regexp pattern="%%%API_PACK%%%" />
<substitution expression="${ast-api-package}" />
<fileset file="${token-constants-file}" />
</replaceregexp>
</target>

17
pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java
View File

@ -5,6 +5,9 @@
package net.sourceforge.pmd.cpd.internal;
import java.io.IOException;
import java.io.Reader;
import org.apache.commons.io.input.CharSequenceReader;
import net.sourceforge.pmd.cpd.SourceCode;
import net.sourceforge.pmd.cpd.TokenEntry;
@ -13,12 +16,24 @@ import net.sourceforge.pmd.cpd.Tokens;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.util.IOUtil;
public abstract class JavaCCTokenizer implements Tokenizer {
protected abstract TokenManager<JavaccToken> getLexerForSource(SourceCode sourceCode);
protected TokenManager<JavaccToken> getLexerForSource(SourceCode sourceCode) {
Reader reader = IOUtil.skipBOM(new CharSequenceReader(sourceCode.getCodeBuffer()));
return makeLexerImpl(makeCharStream(reader));
}
protected CharStream makeCharStream(Reader sourceCode) {
return CharStreamFactory.simpleCharStream(sourceCode);
}
protected abstract TokenManager<JavaccToken> makeLexerImpl(CharStream sourceCode);
protected TokenFilter<JavaccToken> getTokenFilter(TokenManager<JavaccToken> tokenManager) {
return new JavaCCTokenFilter(tokenManager);

15
pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java
View File

@ -5,7 +5,7 @@
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.io.StringReader;
import java.io.Reader;
import java.util.Deque;
import java.util.LinkedList;
import java.util.Properties;
@ -14,9 +14,10 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.java.ast.JavaTokenKinds;
import net.sourceforge.pmd.lang.java.ast.JavaTokenManager;
public class JavaTokenizer extends JavaCCTokenizer {
@ -42,9 +43,13 @@ public class JavaTokenizer extends JavaCCTokenizer {
}
@Override
protected TokenManager<JavaccToken> getLexerForSource(SourceCode sourceCode) {
final StringBuilder stringBuilder = sourceCode.getCodeBuffer();
return new JavaTokenManager(new StringReader(stringBuilder.toString()));
protected CharStream makeCharStream(Reader sourceCode) {
return CharStreamFactory.javaCharStream(sourceCode);
}
@Override
protected TokenManager<JavaccToken> makeLexerImpl(CharStream sourceCode) {
return JavaTokenKinds.newTokenManager(sourceCode);
}
@Override

28
pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenManager.java
View File

@ -1,28 +0,0 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang.java.ast;
import java.io.Reader;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
/**
* Java Token Manager implementation.
*/
public class JavaTokenManager implements TokenManager<JavaccToken> {
private final JavaParserImplTokenManager tokenManager;
public JavaTokenManager(Reader source) {
tokenManager = new JavaParserImplTokenManager(CharStreamFactory.javaCharStream(source));
}
@Override
public JavaccToken getNextToken() {
return tokenManager.getNextToken();
}
}

7
pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/ast/MatlabTokenManager.java
View File

@ -7,8 +7,10 @@ package net.sourceforge.pmd.lang.matlab.ast;
import java.io.Reader;
import net.sourceforge.pmd.annotation.InternalApi;
import net.sourceforge.pmd.cpd.SourceCode;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
/**
* Matlab Token Manager implementation.
@ -26,4 +28,9 @@ public class MatlabTokenManager implements TokenManager {
return tokenManager.getNextToken();
}
public static TokenManager<JavaccToken> create(Reader reader) {
return new MatlabParserImplTokenManager(CharStreamFactory.simpleCharStream(reader));
}
}