phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] Add a generic JavaCCTokenizer for CPD

Browse Source
This commit is contained in:
Andreas Dangel
2019-01-26 11:24:32 +01:00
parent a79d4aeb4f
commit 08b7dd49dd
13 changed files with 154 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java
View File

@ -4,37 +4,20 @@
package net.sourceforge.pmd.cpd;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.cpd.token.TokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.matlab.MatlabTokenManager;
import net.sourceforge.pmd.lang.matlab.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
/**
* The Matlab Tokenizer.
*/
public class MatlabTokenizer implements Tokenizer {
public class MatlabTokenizer extends JavaCCTokenizer {
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
protected TokenManager getLexerForSource(SourceCode sourceCode) {
StringBuilder buffer = sourceCode.getCodeBuffer();
try (Reader reader = IOUtil.skipBOM(new StringReader(buffer.toString()))) {
MatlabTokenManager tokenManager = new MatlabTokenManager(reader);
tokenManager.setFileName(sourceCode.getFileName());
final TokenFilter tokenFilter = new JavaCCTokenFilter(tokenManager);
Token currentToken = (Token) tokenFilter.getNextToken();
while (currentToken != null) {
tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
currentToken = (Token) tokenFilter.getNextToken();
}
} catch (IOException err) {
err.printStackTrace();
} finally {
tokenEntries.add(TokenEntry.getEOF());
}
return new MatlabTokenManager(IOUtil.skipBOM(new StringReader(buffer.toString())));
}
}