phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'tiobe/UTF8_BOM_support_for_JSP_tokenizer' of https://github.com/tiobe/pmd into pr-29

Browse Source
This commit is contained in:
Andreas Dangel
2016-01-07 10:58:05 +01:00
parent eee4bf2550 a735fb057e
commit 4a90118222
1 changed files with 26 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java
View File

@ -3,27 +3,41 @@
*/
package net.sourceforge.pmd.cpd;
import java.io.Reader;
import java.io.StringReader;
import org.apache.commons.io.IOUtils;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.jsp.JspLanguageModule;
import net.sourceforge.pmd.lang.jsp.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
public class JSPTokenizer implements Tokenizer {
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer();
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JspLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), new StringReader(buffer.toString()));
Token currentToken = (Token) tokenMgr.getNextToken();
while (currentToken.image.length() > 0) {
tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(),
currentToken.beginLine));
currentToken = (Token) tokenMgr.getNextToken();
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer();
LanguageVersionHandler languageVersionHandler =
LanguageRegistry.getLanguage(JspLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
Reader reader = null;
try {
reader = new StringReader(buffer.toString());
reader = IOUtil.skipBOM(reader);
TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), reader);
Token currentToken = (Token) tokenMgr.getNextToken();
while (currentToken.image.length() > 0) {
tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(),
currentToken.beginLine));
currentToken = (Token) tokenMgr.getNextToken();
}
} finally {
IOUtils.closeQuietly(reader);
}
tokenEntries.add(TokenEntry.getEOF());
}
tokenEntries.add(TokenEntry.getEOF());
}
}