forked from phoedos/pmd
Fix build
This commit is contained in:
justfile
pmd-ant/src/test/resources/net/sourceforge/pmd/ant/xml
pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex
pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal
pmd-core/src
main/java/net/sourceforge/pmd
cpd
internal
lang/impl
test/java/net/sourceforge/pmd
pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp
pmd-cs/src
main/java/net/sourceforge/pmd/lang/cs
test/java/net/sourceforge/pmd/lang/cs/cpd
pmd-dart/src/main/java/net/sourceforge/pmd/lang/dart
pmd-fortran/src/main/java/net/sourceforge/pmd/lang/fortran
pmd-gherkin/src/main/java/net/sourceforge/pmd/lang/gherkin
pmd-go/src/main/java/net/sourceforge/pmd/lang/go
pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy
pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast
pmd-java/src
main/java/net/sourceforge/pmd/lang/java/cpd
test/java/net/sourceforge/pmd/lang/java/cpd
pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript
pmd-kotlin/src
main/java/net/sourceforge/pmd/lang/kotlin
test/java/net/sourceforge/pmd/lang/kotlin/cpd
pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua
7
justfile
7
justfile
@ -20,6 +20,9 @@ alias i := install
|
||||
cleanInstallEverything *FLAGS:
|
||||
mvnd clean install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}}
|
||||
|
||||
installEverything *FLAGS:
|
||||
mvnd install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}}
|
||||
|
||||
testCore *FLAGS:
|
||||
mvnd test checkstyle:check pmd:check -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl pmd-core {{FLAGS}}
|
||||
|
||||
@ -46,6 +49,6 @@ lintChanged DIFF="master" *FLAGS="":
|
||||
lint projects="pmd-java":
|
||||
mvnd checkstyle:check pmd:check -pl {{projects}} -fae
|
||||
|
||||
lintAll:
|
||||
mvnd checkstyle:check pmd:check -fae
|
||||
lintAll *FLAGS:
|
||||
mvnd checkstyle:check pmd:check -fae {{FLAGS}}
|
||||
|
||||
|
@ -8,7 +8,7 @@
|
||||
<target name="all" depends="testBasic"/>
|
||||
|
||||
<target name="testBasic">
|
||||
<cpd minimumTokenCount="20" outputFile="${pmd.home}/target/cpd.ant.tests" language="cpddummy">
|
||||
<cpd minimumTokenCount="20" outputFile="${pmd.home}/target/cpd.ant.tests" language="dummy">
|
||||
<fileset dir="${pmd.home}/src/main/">
|
||||
<include name="**/*.java"/>
|
||||
</fileset>
|
||||
|
@ -4,13 +4,13 @@
|
||||
|
||||
package net.sourceforge.pmd.lang.apex;
|
||||
|
||||
import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.Language;
|
||||
import net.sourceforge.pmd.lang.LanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguageProcessor;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.LanguageRegistry;
|
||||
import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer;
|
||||
|
||||
import apex.jorje.services.Version;
|
||||
|
||||
|
@ -4,32 +4,60 @@
|
||||
|
||||
package net.sourceforge.pmd.lang.apex.cpd;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.antlr.v4.runtime.CharStream;
|
||||
import org.antlr.runtime.ANTLRReaderStream;
|
||||
import org.antlr.runtime.ANTLRStringStream;
|
||||
import org.antlr.runtime.Lexer;
|
||||
import org.antlr.runtime.Token;
|
||||
|
||||
import net.sourceforge.pmd.cpd.TokenFactory;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
|
||||
import net.sourceforge.pmd.lang.apex.ApexJorjeLogging;
|
||||
import net.sourceforge.pmd.lang.apex.ApexLanguageProperties;
|
||||
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
|
||||
import net.sourceforge.pmd.lang.ast.TokenMgrError;
|
||||
import net.sourceforge.pmd.lang.document.TextDocument;
|
||||
|
||||
import apex.jorje.parser.impl.ApexLexer;
|
||||
|
||||
public class ApexTokenizer implements Tokenizer {
|
||||
|
||||
public class ApexTokenizer extends AntlrTokenizer {
|
||||
private final boolean caseSensitive;
|
||||
|
||||
public ApexTokenizer(ApexLanguageProperties properties) {
|
||||
this.caseSensitive = properties.getProperty(Tokenizer.CPD_CASE_SENSITIVE);
|
||||
ApexJorjeLogging.disableLogging();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getImage(AntlrToken token) {
|
||||
if (caseSensitive) {
|
||||
return token.getImage();
|
||||
public void tokenize(TextDocument document, TokenFactory tokenEntries) throws IOException {
|
||||
|
||||
ANTLRStringStream ass = new ANTLRReaderStream(document.newReader());
|
||||
ApexLexer lexer = new ApexLexer(ass) {
|
||||
@Override
|
||||
public void emitErrorMessage(String msg) {
|
||||
throw new TokenMgrError(getLine(), getCharPositionInLine(), getSourceName(), msg, null);
|
||||
}
|
||||
};
|
||||
|
||||
Token token = lexer.nextToken();
|
||||
|
||||
while (token.getType() != Token.EOF) {
|
||||
if (token.getChannel() != Lexer.HIDDEN) {
|
||||
String tokenText = token.getText();
|
||||
if (!caseSensitive) {
|
||||
tokenText = tokenText.toLowerCase(Locale.ROOT);
|
||||
}
|
||||
tokenEntries.recordToken(
|
||||
tokenText,
|
||||
token.getLine(),
|
||||
token.getCharPositionInLine() + 1,
|
||||
token.getLine(),
|
||||
token.getCharPositionInLine() + tokenText.length() + 1
|
||||
);
|
||||
}
|
||||
token = lexer.nextToken();
|
||||
}
|
||||
return token.getImage().toLowerCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected org.antlr.v4.runtime.Lexer getLexerForSource(CharStream charStream) {
|
||||
return new com.nawforce.runtime.parsers.ApexLexer(charStream);
|
||||
}
|
||||
}
|
||||
|
@ -124,7 +124,7 @@ public class CpdCommand extends AbstractAnalysisPmdSubcommand {
|
||||
|
||||
final CPDConfiguration configuration = toConfiguration();
|
||||
|
||||
try (CpdAnalysis cpd = new CpdAnalysis(configuration)){
|
||||
try (CpdAnalysis cpd = new CpdAnalysis(configuration)) {
|
||||
|
||||
MutableBoolean hasViolations = new MutableBoolean();
|
||||
cpd.performAnalysis(report -> hasViolations.setValue(!report.getMatches().isEmpty()));
|
||||
|
@ -62,8 +62,8 @@ public class AnyTokenizer implements Tokenizer {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
|
||||
Chars text = sourceCode.getText();
|
||||
public void tokenize(TextDocument document, TokenFactory tokens) {
|
||||
Chars text = document.getText();
|
||||
Matcher matcher = pattern.matcher(text);
|
||||
int lineNo = 1;
|
||||
int lastLineStart = 0;
|
||||
|
@ -62,11 +62,11 @@ import javax.swing.table.TableModel;
|
||||
|
||||
import net.sourceforge.pmd.PMDVersion;
|
||||
import net.sourceforge.pmd.cpd.renderer.CPDReportRenderer;
|
||||
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.Language;
|
||||
import net.sourceforge.pmd.lang.LanguageModuleBase.LanguageMetadata;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.LanguageRegistry;
|
||||
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
|
||||
|
||||
public class GUI implements CPDListener {
|
||||
|
||||
|
@ -26,10 +26,8 @@ public interface Tokenizer {
|
||||
PropertyDescriptor<Boolean> CPD_ANONYMIZE_IDENTIFIERS =
|
||||
PropertyFactory.booleanProperty("cpdAnonymizeIdentifiers")
|
||||
.defaultValue(false)
|
||||
.desc("Anonymize identifiers. They are still part of the token stream but all literals appear to have the same value.")
|
||||
.desc("Anonymize identifiers. They are still part of the token stream but all identifiers appear to have the same value.")
|
||||
.build();
|
||||
|
||||
|
||||
PropertyDescriptor<Boolean> CPD_IGNORE_IMPORTS =
|
||||
PropertyFactory.booleanProperty("cpdIgnoreImports")
|
||||
.defaultValue(true)
|
||||
@ -41,48 +39,17 @@ public interface Tokenizer {
|
||||
.defaultValue(false)
|
||||
.desc("Ignore metadata such as Java annotations or C# attributes.")
|
||||
.build();
|
||||
|
||||
|
||||
PropertyDescriptor<Boolean> CPD_CASE_SENSITIVE =
|
||||
PropertyFactory.booleanProperty("cpdCaseSensitive")
|
||||
.defaultValue(true)
|
||||
.defaultValue(false)
|
||||
.desc("Whether CPD should ignore the case of tokens. Affects all tokens.")
|
||||
.build();
|
||||
|
||||
|
||||
String IGNORE_LITERALS = "ignore_literals";
|
||||
String IGNORE_IDENTIFIERS = "ignore_identifiers";
|
||||
String IGNORE_ANNOTATIONS = "ignore_annotations";
|
||||
|
||||
/**
|
||||
* Ignore sequences of literals (e.g, <code>0,0,0,0...</code>).
|
||||
*/
|
||||
String OPTION_IGNORE_LITERAL_SEQUENCES = "net.sourceforge.pmd.cpd.Tokenizer.skipLiteralSequences";
|
||||
/**
|
||||
* Ignore using directives in C#. The default value is <code>false</code>.
|
||||
*/
|
||||
String IGNORE_USINGS = "ignore_usings";
|
||||
|
||||
/**
|
||||
* Enables or disabled skipping of blocks like a pre-processor. It is a
|
||||
* boolean property. The default value is <code>true</code>.
|
||||
*
|
||||
* @see #OPTION_SKIP_BLOCKS_PATTERN
|
||||
*/
|
||||
String OPTION_SKIP_BLOCKS = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocks";
|
||||
/**
|
||||
* Configures the pattern, to find the blocks to skip. It is a string
|
||||
* property and contains of two parts, separated by {@code |}. The first
|
||||
* part is the start pattern, the second part is the ending pattern. Default
|
||||
* value is "{@code #if 0|#endif}".
|
||||
*
|
||||
* @see #DEFAULT_SKIP_BLOCKS_PATTERN
|
||||
*/
|
||||
String OPTION_SKIP_BLOCKS_PATTERN = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocksPattern";
|
||||
|
||||
@Deprecated // TODO what to do with this?
|
||||
String DEFAULT_SKIP_BLOCKS_PATTERN = "#if 0|#endif";
|
||||
|
||||
void tokenize(TextDocument sourceCode, TokenFactory tokens) throws IOException;
|
||||
void tokenize(TextDocument document, TokenFactory tokens) throws IOException;
|
||||
|
||||
static void tokenize(Tokenizer tokenizer, TextDocument textDocument, Tokens tokens) throws IOException {
|
||||
try (TokenFactory tf = TokenFactory.forFile(textDocument, tokens)) {
|
||||
|
@ -1,68 +0,0 @@
|
||||
/*
|
||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
*/
|
||||
|
||||
package net.sourceforge.pmd.internal;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Map;
|
||||
import java.util.ServiceConfigurationError;
|
||||
import java.util.ServiceLoader;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import net.sourceforge.pmd.annotation.InternalApi;
|
||||
|
||||
@InternalApi
|
||||
public abstract class LanguageServiceBase<T> {
|
||||
|
||||
protected interface NameExtractor<T> {
|
||||
String getName(T language);
|
||||
}
|
||||
|
||||
protected final Set<T> languages;
|
||||
protected final Map<String, T> languagesByName;
|
||||
protected final Map<String, T> languagesByTerseName;
|
||||
|
||||
protected LanguageServiceBase(final Class<T> serviceType, final Comparator<T> comparator,
|
||||
final NameExtractor<T> nameExtractor, final NameExtractor<T> terseNameExtractor) {
|
||||
Set<T> sortedLangs = new TreeSet<>(comparator);
|
||||
// Use current class' classloader instead of the threads context classloader, see https://github.com/pmd/pmd/issues/1788
|
||||
ServiceLoader<T> languageLoader = ServiceLoader.load(serviceType, getClass().getClassLoader());
|
||||
Iterator<T> iterator = languageLoader.iterator();
|
||||
|
||||
while (true) {
|
||||
// this loop is weird, but both hasNext and next may throw ServiceConfigurationError,
|
||||
// it's more robust that way
|
||||
try {
|
||||
if (iterator.hasNext()) {
|
||||
T language = iterator.next();
|
||||
sortedLangs.add(language);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
} catch (UnsupportedClassVersionError | ServiceConfigurationError e) {
|
||||
// Some languages require java8 and are therefore only available
|
||||
// if java8 or later is used as runtime.
|
||||
System.err.println("Ignoring language for PMD: " + e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
// using a linked hash map to maintain insertion order
|
||||
languages = Collections.unmodifiableSet(new LinkedHashSet<>(sortedLangs));
|
||||
|
||||
// TODO there may be languages with duplicate names
|
||||
Map<String, T> byName = new LinkedHashMap<>();
|
||||
Map<String, T> byTerseName = new LinkedHashMap<>();
|
||||
for (T language : sortedLangs) {
|
||||
byName.put(nameExtractor.getName(language), language);
|
||||
byTerseName.put(terseNameExtractor.getName(language), language);
|
||||
}
|
||||
languagesByName = Collections.unmodifiableMap(byName);
|
||||
languagesByTerseName = Collections.unmodifiableMap(byTerseName);
|
||||
}
|
||||
}
|
@ -2,9 +2,11 @@
|
||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
*/
|
||||
|
||||
package net.sourceforge.pmd.lang;
|
||||
package net.sourceforge.pmd.lang.impl;
|
||||
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.LanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
|
||||
/**
|
||||
* Base class for language modules that only support CPD and not PMD.
|
@ -1,4 +1,4 @@
|
||||
/**
|
||||
/*
|
||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
*/
|
||||
|
||||
@ -13,12 +13,10 @@ import java.util.Iterator;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import net.sourceforge.pmd.lang.DummyLanguageModule;
|
||||
import net.sourceforge.pmd.lang.Language;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.document.Chars;
|
||||
import net.sourceforge.pmd.lang.document.TextDocument;
|
||||
import net.sourceforge.pmd.lang.document.TextFile;
|
||||
import net.sourceforge.pmd.lang.java.JavaLanguageModule;
|
||||
|
||||
class MatchAlgorithmTest {
|
||||
|
||||
@ -38,15 +36,15 @@ class MatchAlgorithmTest {
|
||||
|
||||
@Test
|
||||
void testSimple() throws IOException {
|
||||
Language java = JavaLanguageModule.getInstance();
|
||||
Tokenizer tokenizer = java.createCpdTokenizer(java.newPropertyBundle());
|
||||
String fileName = "Foo.java";
|
||||
TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, java.getDefaultVersion());
|
||||
Language dummy = DummyLanguageModule.getInstance();
|
||||
Tokenizer tokenizer = dummy.createCpdTokenizer(dummy.newPropertyBundle());
|
||||
String fileName = "Foo.dummy";
|
||||
TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, dummy.getDefaultVersion());
|
||||
SourceManager sourceManager = new SourceManager(listOf(textFile));
|
||||
Tokens tokens = new Tokens();
|
||||
TextDocument sourceCode = sourceManager.get(textFile);
|
||||
Tokenizer.tokenize(tokenizer, sourceCode, tokens);
|
||||
assertEquals(41, tokens.size());
|
||||
assertEquals(44, tokens.size());
|
||||
|
||||
MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5);
|
||||
matchAlgorithm.findMatches();
|
||||
@ -61,34 +59,10 @@ class MatchAlgorithmTest {
|
||||
|
||||
assertEquals(3, mark1.getBeginLine());
|
||||
assertEquals(fileName, mark1.getFilename());
|
||||
assertEquals(Chars.wrap(LINE_3), sourceManager.getSlice(mark1));
|
||||
assertEquals(LINE_3 + "\n", sourceManager.getSlice(mark1).toString());
|
||||
|
||||
assertEquals(4, mark2.getBeginLine());
|
||||
assertEquals(fileName, mark2.getFilename());
|
||||
assertEquals(Chars.wrap(LINE_4), sourceManager.getSlice(mark2));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testIgnore() throws IOException {
|
||||
Language java = JavaLanguageModule.getInstance();
|
||||
LanguagePropertyBundle bundle = java.newPropertyBundle();
|
||||
bundle.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, true);
|
||||
bundle.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, true);
|
||||
Tokenizer tokenizer = java.createCpdTokenizer(bundle);
|
||||
TextDocument sourceCode = TextDocument.readOnlyString(getSampleCode(), "Foo.java", java.getDefaultVersion());
|
||||
Tokens tokens = new Tokens();
|
||||
Tokenizer.tokenize(tokenizer, sourceCode, tokens);
|
||||
|
||||
MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5);
|
||||
matchAlgorithm.findMatches();
|
||||
Iterator<Match> matches = matchAlgorithm.matches();
|
||||
Match match = matches.next();
|
||||
assertFalse(matches.hasNext());
|
||||
|
||||
Iterator<Mark> marks = match.iterator();
|
||||
marks.next();
|
||||
marks.next();
|
||||
marks.next();
|
||||
assertFalse(marks.hasNext());
|
||||
assertEquals(LINE_4 + "\n", sourceManager.getSlice(mark2).toString());
|
||||
}
|
||||
}
|
@ -7,6 +7,7 @@ package net.sourceforge.pmd.lang;
|
||||
import java.util.Objects;
|
||||
|
||||
import net.sourceforge.pmd.RuleViolation;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.ast.DummyNode;
|
||||
import net.sourceforge.pmd.lang.ast.DummyNode.DummyRootNode;
|
||||
import net.sourceforge.pmd.lang.ast.ParseException;
|
||||
@ -45,6 +46,14 @@ public class DummyLanguageModule extends SimpleLanguageModuleBase {
|
||||
return (DummyLanguageModule) Objects.requireNonNull(LanguageRegistry.PMD.getLanguageByFullName(NAME));
|
||||
}
|
||||
|
||||
@Override
|
||||
public LanguagePropertyBundle newPropertyBundle() {
|
||||
LanguagePropertyBundle bundle = super.newPropertyBundle();
|
||||
bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_LITERALS);
|
||||
bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS);
|
||||
return bundle;
|
||||
}
|
||||
|
||||
public LanguageVersion getVersionWhereParserThrows() {
|
||||
return getVersion(PARSER_THROWS);
|
||||
}
|
||||
|
@ -6,9 +6,9 @@ package net.sourceforge.pmd.lang.cpp;
|
||||
|
||||
import net.sourceforge.pmd.cpd.CPPTokenizer;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.LanguageRegistry;
|
||||
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.properties.PropertyDescriptor;
|
||||
import net.sourceforge.pmd.properties.PropertyFactory;
|
||||
|
||||
@ -19,7 +19,7 @@ public class CppLanguageModule extends CpdOnlyLanguageModuleBase {
|
||||
|
||||
|
||||
public static final PropertyDescriptor<String> CPD_SKIP_BLOCKS =
|
||||
PropertyFactory.stringProperty("cpdSkipBlocksPattern")
|
||||
PropertyFactory.stringProperty("cpdSkipBlocksPattern")
|
||||
.defaultValue("#if 0|#endif")
|
||||
.desc("Specifies a start and end delimiter for CPD to completely ignore. "
|
||||
+ "The delimiters are separated by a pipe |. The default skips code "
|
||||
|
@ -4,11 +4,11 @@
|
||||
|
||||
package net.sourceforge.pmd.lang.cs;
|
||||
|
||||
import net.sourceforge.pmd.cpd.CsTokenizer;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.LanguageRegistry;
|
||||
import net.sourceforge.pmd.lang.cs.cpd.CsTokenizer;
|
||||
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
|
||||
|
||||
/**
|
||||
* Defines the Language module for C#.
|
||||
|
@ -1,12 +1,13 @@
|
||||
/**
|
||||
/*
|
||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
*/
|
||||
|
||||
package net.sourceforge.pmd.cpd;
|
||||
package net.sourceforge.pmd.lang.cs.cpd;
|
||||
|
||||
import org.antlr.v4.runtime.CharStream;
|
||||
import org.antlr.v4.runtime.Lexer;
|
||||
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
|
||||
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
|
||||
import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;
|
@ -2,28 +2,22 @@
|
||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
*/
|
||||
|
||||
package net.sourceforge.pmd.cpd;
|
||||
package net.sourceforge.pmd.lang.cs.cpd;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.checkerframework.checker.nullness.qual.NonNull;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
|
||||
import net.sourceforge.pmd.lang.ast.TokenMgrError;
|
||||
|
||||
class CsTokenizerTest extends CpdTextComparisonTest {
|
||||
|
||||
CsTokenizerTest() {
|
||||
super(".cs");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getResourcePrefix() {
|
||||
return "../lang/cs/cpd/testdata";
|
||||
super("cs", ".cs");
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -107,23 +101,28 @@ class CsTokenizerTest extends CpdTextComparisonTest {
|
||||
doTest("attributes", "_ignored", skipAttributes());
|
||||
}
|
||||
|
||||
private Properties ignoreUsings() {
|
||||
private LanguagePropertyConfig ignoreUsings() {
|
||||
return properties(true, false, false);
|
||||
}
|
||||
|
||||
private Properties skipLiteralSequences() {
|
||||
private LanguagePropertyConfig skipLiteralSequences() {
|
||||
return properties(false, true, false);
|
||||
}
|
||||
|
||||
private Properties skipAttributes() {
|
||||
private LanguagePropertyConfig skipAttributes() {
|
||||
return properties(false, false, true);
|
||||
}
|
||||
|
||||
private Properties properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) {
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty(Tokenizer.IGNORE_USINGS, Boolean.toString(ignoreUsings));
|
||||
properties.setProperty(Tokenizer.OPTION_IGNORE_LITERAL_SEQUENCES, Boolean.toString(ignoreLiteralSequences));
|
||||
properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAttributes));
|
||||
return properties;
|
||||
@Override
|
||||
public @NonNull LanguagePropertyConfig defaultProperties() {
|
||||
return properties(false, false, false);
|
||||
}
|
||||
|
||||
private LanguagePropertyConfig properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) {
|
||||
return properties -> {
|
||||
properties.setProperty(Tokenizer.CPD_IGNORE_IMPORTS, ignoreUsings);
|
||||
properties.setProperty(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES, ignoreLiteralSequences);
|
||||
properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAttributes);
|
||||
};
|
||||
}
|
||||
}
|
@ -4,10 +4,10 @@
|
||||
|
||||
package net.sourceforge.pmd.lang.dart;
|
||||
|
||||
import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer;
|
||||
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
|
||||
|
||||
/**
|
||||
* Language implementation for Dart
|
||||
|
@ -6,8 +6,8 @@ package net.sourceforge.pmd.lang.fortran;
|
||||
|
||||
import net.sourceforge.pmd.cpd.AnyTokenizer;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
|
||||
|
||||
/**
|
||||
* Language implementation for Fortran
|
||||
|
@ -5,9 +5,9 @@
|
||||
package net.sourceforge.pmd.lang.gherkin;
|
||||
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.gherkin.cpd.GherkinTokenizer;
|
||||
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
|
||||
|
||||
/**
|
||||
* Language implementation for Gherkin.
|
||||
|
@ -5,9 +5,9 @@
|
||||
package net.sourceforge.pmd.lang.go;
|
||||
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.go.cpd.GoTokenizer;
|
||||
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
|
||||
|
||||
public class GoLanguageModule extends CpdOnlyLanguageModuleBase {
|
||||
|
||||
|
@ -4,10 +4,10 @@
|
||||
|
||||
package net.sourceforge.pmd.lang.groovy;
|
||||
|
||||
import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer;
|
||||
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
|
||||
|
||||
/**
|
||||
* Language implementation for Groovy
|
||||
|
@ -22,8 +22,8 @@ import groovyjarjarantlr.TokenStreamException;
|
||||
public class GroovyTokenizer implements Tokenizer {
|
||||
|
||||
@Override
|
||||
public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
|
||||
GroovyLexer lexer = new GroovyLexer(sourceCode.newReader());
|
||||
public void tokenize(TextDocument document, TokenFactory tokens) {
|
||||
GroovyLexer lexer = new GroovyLexer(document.newReader());
|
||||
TokenStream tokenStream = lexer.plumb();
|
||||
|
||||
try {
|
||||
|
@ -19,13 +19,13 @@ import net.sourceforge.pmd.lang.html.HtmlLanguageModule;
|
||||
public class HtmlTokenizer implements Tokenizer {
|
||||
|
||||
@Override
|
||||
public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
|
||||
public void tokenize(TextDocument document, TokenFactory tokens) {
|
||||
HtmlLanguageModule html = HtmlLanguageModule.getInstance();
|
||||
|
||||
try (LanguageProcessor processor = html.createProcessor(html.newPropertyBundle())) {
|
||||
|
||||
ParserTask task = new ParserTask(
|
||||
sourceCode,
|
||||
document,
|
||||
SemanticErrorReporter.noop(),
|
||||
LanguageProcessorRegistry.singleton(processor)
|
||||
);
|
||||
|
@ -1,12 +1,15 @@
|
||||
/**
|
||||
/*
|
||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
*/
|
||||
|
||||
package net.sourceforge.pmd.cpd;
|
||||
package net.sourceforge.pmd.lang.java.cpd;
|
||||
|
||||
import java.util.Deque;
|
||||
import java.util.LinkedList;
|
||||
|
||||
import net.sourceforge.pmd.cpd.TokenEntry;
|
||||
import net.sourceforge.pmd.cpd.TokenFactory;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
|
||||
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
|
||||
import net.sourceforge.pmd.lang.TokenManager;
|
@ -1,27 +1,20 @@
|
||||
/**
|
||||
/*
|
||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
*/
|
||||
|
||||
package net.sourceforge.pmd.cpd;
|
||||
package net.sourceforge.pmd.lang.java.cpd;
|
||||
|
||||
import java.util.Properties;
|
||||
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
|
||||
import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
|
||||
import net.sourceforge.pmd.lang.java.JavaLanguageModule;
|
||||
|
||||
// TODO - enable test
|
||||
@Disabled("Needs to be enabled after java-grammar changes are finalized")
|
||||
class JavaTokenizerTest extends CpdTextComparisonTest {
|
||||
|
||||
JavaTokenizerTest() {
|
||||
super(".java");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getResourcePrefix() {
|
||||
return "../lang/java/cpd/testdata";
|
||||
super(JavaLanguageModule.getInstance(), ".java");
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -85,32 +78,32 @@ class JavaTokenizerTest extends CpdTextComparisonTest {
|
||||
}
|
||||
|
||||
|
||||
private static Properties ignoreAnnotations() {
|
||||
private static LanguagePropertyConfig ignoreAnnotations() {
|
||||
return properties(true, false, false);
|
||||
}
|
||||
|
||||
private static Properties ignoreIdents() {
|
||||
private static LanguagePropertyConfig ignoreIdents() {
|
||||
return properties(false, false, true);
|
||||
}
|
||||
|
||||
private static Properties ignoreLiterals() {
|
||||
private static LanguagePropertyConfig ignoreLiterals() {
|
||||
return properties(false, true, false);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Properties defaultProperties() {
|
||||
public LanguagePropertyConfig defaultProperties() {
|
||||
return properties(false, false, false);
|
||||
}
|
||||
|
||||
private static Properties properties(boolean ignoreAnnotations,
|
||||
boolean ignoreLiterals,
|
||||
boolean ignoreIdents) {
|
||||
Properties properties = new Properties();
|
||||
properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAnnotations));
|
||||
properties.setProperty(Tokenizer.IGNORE_IDENTIFIERS, Boolean.toString(ignoreIdents));
|
||||
properties.setProperty(Tokenizer.IGNORE_LITERALS, Boolean.toString(ignoreLiterals));
|
||||
return properties;
|
||||
private static LanguagePropertyConfig properties(boolean ignoreAnnotations,
|
||||
boolean ignoreLiterals,
|
||||
boolean ignoreIdents) {
|
||||
return properties -> {
|
||||
properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAnnotations);
|
||||
properties.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, ignoreIdents);
|
||||
properties.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, ignoreLiterals);
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -4,12 +4,12 @@
|
||||
|
||||
package net.sourceforge.pmd.lang.ecmascript;
|
||||
|
||||
import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.Language;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.LanguageRegistry;
|
||||
import net.sourceforge.pmd.lang.ecmascript.ast.EcmascriptParser;
|
||||
import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer;
|
||||
import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase;
|
||||
|
||||
/**
|
||||
|
@ -5,10 +5,10 @@
|
||||
package net.sourceforge.pmd.lang.kotlin;
|
||||
|
||||
import net.sourceforge.pmd.annotation.Experimental;
|
||||
import net.sourceforge.pmd.cpd.KotlinTokenizer;
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.kotlin.cpd.KotlinTokenizer;
|
||||
|
||||
/**
|
||||
* Language Module for Kotlin
|
||||
|
@ -1,16 +1,16 @@
|
||||
/**
|
||||
/*
|
||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
*/
|
||||
|
||||
package net.sourceforge.pmd.cpd;
|
||||
package net.sourceforge.pmd.lang.kotlin.cpd;
|
||||
|
||||
import org.antlr.v4.runtime.CharStream;
|
||||
import org.antlr.v4.runtime.Lexer;
|
||||
|
||||
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
|
||||
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
|
||||
import net.sourceforge.pmd.lang.TokenManager;
|
||||
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
|
||||
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
|
||||
import net.sourceforge.pmd.lang.kotlin.ast.KotlinLexer;
|
||||
|
||||
/**
|
||||
@ -24,7 +24,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) {
|
||||
protected TokenManager<AntlrToken> filterTokenStream(TokenManager<AntlrToken> tokenManager) {
|
||||
return new KotlinTokenFilter(tokenManager);
|
||||
}
|
||||
|
||||
@ -40,7 +40,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
|
||||
private boolean discardingPackageAndImport = false;
|
||||
private boolean discardingNL = false;
|
||||
|
||||
/* default */ KotlinTokenFilter(final AntlrTokenManager tokenManager) {
|
||||
/* default */ KotlinTokenFilter(final TokenManager<AntlrToken> tokenManager) {
|
||||
super(tokenManager);
|
||||
}
|
||||
|
@ -1,8 +1,8 @@
|
||||
/**
|
||||
/*
|
||||
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
|
||||
*/
|
||||
|
||||
package net.sourceforge.pmd.cpd;
|
||||
package net.sourceforge.pmd.lang.kotlin.cpd;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@ -14,11 +14,6 @@ class KotlinTokenizerTest extends CpdTextComparisonTest {
|
||||
super("kotlin", ".kt");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getResourcePrefix() {
|
||||
return "../lang/kotlin/cpd/testdata";
|
||||
}
|
||||
|
||||
@Test
|
||||
void testComments() {
|
||||
doTest("comment");
|
@ -5,8 +5,8 @@
|
||||
package net.sourceforge.pmd.lang.lua;
|
||||
|
||||
import net.sourceforge.pmd.cpd.Tokenizer;
|
||||
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
|
||||
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
|
||||
import net.sourceforge.pmd.lang.lua.cpd.LuaTokenizer;
|
||||
|
||||
/**
|
||||
@ -18,8 +18,15 @@ public class LuaLanguageModule extends CpdOnlyLanguageModuleBase {
|
||||
super(LanguageMetadata.withId("lua").name("Lua").extensions("lua"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public LanguagePropertyBundle newPropertyBundle() {
|
||||
LanguagePropertyBundle bundle = super.newPropertyBundle();
|
||||
bundle.definePropertyDescriptor(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES);
|
||||
return bundle;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Tokenizer createCpdTokenizer(LanguagePropertyBundle bundle) {
|
||||
return new LuaTokenizer();
|
||||
return new LuaTokenizer(bundle);
|
||||
}
|
||||
}
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user