hiifong/pmd
1
0
Fork 0
forked from phoedos/pmd
Code Pull Requests Activity

Added support for files with UTF-8 BOM to JSP tokenizer.

Browse Source
This commit is contained in:
Jan van Nunen
2015-12-10 15:39:29 +01:00
parent 6a3e7ce6ec
commit a735fb057e
1 changed files with 26 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java
View File

@ -3,27 +3,41 @@
*/
package net.sourceforge.pmd.cpd;
import java.io.Reader;
import java.io.StringReader;
import org.apache.commons.io.IOUtils;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.LanguageVersionHandler;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.jsp.JspLanguageModule;
import net.sourceforge.pmd.lang.jsp.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
public class JSPTokenizer implements Tokenizer {
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer();
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(JspLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), new StringReader(buffer.toString()));
Token currentToken = (Token) tokenMgr.getNextToken();
while (currentToken.image.length() > 0) {
tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(),
currentToken.beginLine));
currentToken = (Token) tokenMgr.getNextToken();
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer();
LanguageVersionHandler languageVersionHandler =
LanguageRegistry.getLanguage(JspLanguageModule.NAME).getDefaultVersion().getLanguageVersionHandler();
Reader reader = null;
try {
reader = new StringReader(buffer.toString());
reader = IOUtil.skipBOM(reader);
TokenManager tokenMgr = languageVersionHandler.getParser(languageVersionHandler.getDefaultParserOptions())
.getTokenManager(sourceCode.getFileName(), reader);
Token currentToken = (Token) tokenMgr.getNextToken();
while (currentToken.image.length() > 0) {
tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(),
currentToken.beginLine));
currentToken = (Token) tokenMgr.getNextToken();
}
} finally {
IOUtils.closeQuietly(reader);
}
tokenEntries.add(TokenEntry.getEOF());
}
tokenEntries.add(TokenEntry.getEOF());
}
}