phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#1252 implement BOM detection differently for cpp

Browse Source
This commit is contained in:
Andreas Dangel
2014-12-12 17:45:38 +01:00
parent 8fb2f0a09f
commit 6b3b9d0ccc
2 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pmd-cpp/etc/grammar/cpp.jj
View File

@ -133,8 +133,6 @@ PARSER_END(CppParser)
SKIP : SKIP :
{ {
"\ufeff" // UTF Byte Order Mark
|
" " " "
| |
"\f" "\f"

5
pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java
View File

@ -5,6 +5,7 @@ package net.sourceforge.pmd.cpd;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.IOException; import java.io.IOException;
import java.io.Reader;
import java.io.StringReader; import java.io.StringReader;
import java.util.Properties; import java.util.Properties;
@ -15,6 +16,7 @@ import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.cpp.CppLanguageModule; import net.sourceforge.pmd.lang.cpp.CppLanguageModule;
import net.sourceforge.pmd.lang.cpp.ast.Token; import net.sourceforge.pmd.lang.cpp.ast.Token;
import net.sourceforge.pmd.util.IOUtil;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
@ -50,11 +52,12 @@ public class CPPTokenizer implements Tokenizer {
@Override @Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder buffer = sourceCode.getCodeBuffer(); StringBuilder buffer = sourceCode.getCodeBuffer();
StringReader reader = null; Reader reader = null;
try { try {
LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(CppLanguageModule.NAME) LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(CppLanguageModule.NAME)
.getDefaultVersion().getLanguageVersionHandler(); .getDefaultVersion().getLanguageVersionHandler();
reader = new StringReader(maybeSkipBlocks(buffer.toString())); reader = new StringReader(maybeSkipBlocks(buffer.toString()));
reader = IOUtil.skipBOM(reader);
TokenManager tokenManager = languageVersionHandler.getParser( TokenManager tokenManager = languageVersionHandler.getParser(
languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader); languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
Token currentToken = (Token) tokenManager.getNextToken(); Token currentToken = (Token) tokenManager.getNextToken();