1
0
forked from phoedos/pmd

Fix build

This commit is contained in:
Clément Fournier
2023-02-12 18:21:54 +01:00
parent fb9f49624d
commit ddbfc90c14
40 changed files with 174 additions and 276 deletions
justfile
pmd-ant/src/test/resources/net/sourceforge/pmd/ant/xml
pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex
pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal
pmd-core/src
pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp
pmd-cs/src
main/java/net/sourceforge/pmd/lang/cs
test/java/net/sourceforge/pmd/lang/cs/cpd
pmd-dart/src/main/java/net/sourceforge/pmd/lang/dart
pmd-fortran/src/main/java/net/sourceforge/pmd/lang/fortran
pmd-gherkin/src/main/java/net/sourceforge/pmd/lang/gherkin
pmd-go/src/main/java/net/sourceforge/pmd/lang/go
pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy
pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast
pmd-java/src
main/java/net/sourceforge/pmd/lang/java/cpd
test/java/net/sourceforge/pmd/lang/java/cpd
pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript
pmd-kotlin/src
main/java/net/sourceforge/pmd/lang/kotlin
test/java/net/sourceforge/pmd/lang/kotlin/cpd
pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua

@ -20,6 +20,9 @@ alias i := install
cleanInstallEverything *FLAGS:
mvnd clean install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}}
installEverything *FLAGS:
mvnd install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}}
testCore *FLAGS:
mvnd test checkstyle:check pmd:check -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl pmd-core {{FLAGS}}
@ -46,6 +49,6 @@ lintChanged DIFF="master" *FLAGS="":
lint projects="pmd-java":
mvnd checkstyle:check pmd:check -pl {{projects}} -fae
lintAll:
mvnd checkstyle:check pmd:check -fae
lintAll *FLAGS:
mvnd checkstyle:check pmd:check -fae {{FLAGS}}

@ -8,7 +8,7 @@
<target name="all" depends="testBasic"/>
<target name="testBasic">
<cpd minimumTokenCount="20" outputFile="${pmd.home}/target/cpd.ant.tests" language="cpddummy">
<cpd minimumTokenCount="20" outputFile="${pmd.home}/target/cpd.ant.tests" language="dummy">
<fileset dir="${pmd.home}/src/main/">
<include name="**/*.java"/>
</fileset>

@ -4,13 +4,13 @@
package net.sourceforge.pmd.lang.apex;
import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.Language;
import net.sourceforge.pmd.lang.LanguageModuleBase;
import net.sourceforge.pmd.lang.LanguageProcessor;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer;
import apex.jorje.services.Version;

@ -4,32 +4,60 @@
package net.sourceforge.pmd.lang.apex.cpd;
import java.io.IOException;
import java.util.Locale;
import org.antlr.v4.runtime.CharStream;
import org.antlr.runtime.ANTLRReaderStream;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.Lexer;
import org.antlr.runtime.Token;
import net.sourceforge.pmd.cpd.TokenFactory;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
import net.sourceforge.pmd.lang.apex.ApexJorjeLogging;
import net.sourceforge.pmd.lang.apex.ApexLanguageProperties;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.document.TextDocument;
import apex.jorje.parser.impl.ApexLexer;
public class ApexTokenizer implements Tokenizer {
public class ApexTokenizer extends AntlrTokenizer {
private final boolean caseSensitive;
public ApexTokenizer(ApexLanguageProperties properties) {
this.caseSensitive = properties.getProperty(Tokenizer.CPD_CASE_SENSITIVE);
ApexJorjeLogging.disableLogging();
}
@Override
protected String getImage(AntlrToken token) {
if (caseSensitive) {
return token.getImage();
public void tokenize(TextDocument document, TokenFactory tokenEntries) throws IOException {
ANTLRStringStream ass = new ANTLRReaderStream(document.newReader());
ApexLexer lexer = new ApexLexer(ass) {
@Override
public void emitErrorMessage(String msg) {
throw new TokenMgrError(getLine(), getCharPositionInLine(), getSourceName(), msg, null);
}
};
Token token = lexer.nextToken();
while (token.getType() != Token.EOF) {
if (token.getChannel() != Lexer.HIDDEN) {
String tokenText = token.getText();
if (!caseSensitive) {
tokenText = tokenText.toLowerCase(Locale.ROOT);
}
tokenEntries.recordToken(
tokenText,
token.getLine(),
token.getCharPositionInLine() + 1,
token.getLine(),
token.getCharPositionInLine() + tokenText.length() + 1
);
}
token = lexer.nextToken();
}
return token.getImage().toLowerCase(Locale.ROOT);
}
@Override
protected org.antlr.v4.runtime.Lexer getLexerForSource(CharStream charStream) {
return new com.nawforce.runtime.parsers.ApexLexer(charStream);
}
}

@ -124,7 +124,7 @@ public class CpdCommand extends AbstractAnalysisPmdSubcommand {
final CPDConfiguration configuration = toConfiguration();
try (CpdAnalysis cpd = new CpdAnalysis(configuration)){
try (CpdAnalysis cpd = new CpdAnalysis(configuration)) {
MutableBoolean hasViolations = new MutableBoolean();
cpd.performAnalysis(report -> hasViolations.setValue(!report.getMatches().isEmpty()));

@ -62,8 +62,8 @@ public class AnyTokenizer implements Tokenizer {
}
@Override
public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
Chars text = sourceCode.getText();
public void tokenize(TextDocument document, TokenFactory tokens) {
Chars text = document.getText();
Matcher matcher = pattern.matcher(text);
int lineNo = 1;
int lastLineStart = 0;

@ -62,11 +62,11 @@ import javax.swing.table.TableModel;
import net.sourceforge.pmd.PMDVersion;
import net.sourceforge.pmd.cpd.renderer.CPDReportRenderer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.Language;
import net.sourceforge.pmd.lang.LanguageModuleBase.LanguageMetadata;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
public class GUI implements CPDListener {

@ -26,10 +26,8 @@ public interface Tokenizer {
PropertyDescriptor<Boolean> CPD_ANONYMIZE_IDENTIFIERS =
PropertyFactory.booleanProperty("cpdAnonymizeIdentifiers")
.defaultValue(false)
.desc("Anonymize identifiers. They are still part of the token stream but all literals appear to have the same value.")
.desc("Anonymize identifiers. They are still part of the token stream but all identifiers appear to have the same value.")
.build();
PropertyDescriptor<Boolean> CPD_IGNORE_IMPORTS =
PropertyFactory.booleanProperty("cpdIgnoreImports")
.defaultValue(true)
@ -41,48 +39,17 @@ public interface Tokenizer {
.defaultValue(false)
.desc("Ignore metadata such as Java annotations or C# attributes.")
.build();
PropertyDescriptor<Boolean> CPD_CASE_SENSITIVE =
PropertyFactory.booleanProperty("cpdCaseSensitive")
.defaultValue(true)
.defaultValue(false)
.desc("Whether CPD should ignore the case of tokens. Affects all tokens.")
.build();
String IGNORE_LITERALS = "ignore_literals";
String IGNORE_IDENTIFIERS = "ignore_identifiers";
String IGNORE_ANNOTATIONS = "ignore_annotations";
/**
* Ignore sequences of literals (e.g, <code>0,0,0,0...</code>).
*/
String OPTION_IGNORE_LITERAL_SEQUENCES = "net.sourceforge.pmd.cpd.Tokenizer.skipLiteralSequences";
/**
* Ignore using directives in C#. The default value is <code>false</code>.
*/
String IGNORE_USINGS = "ignore_usings";
/**
* Enables or disabled skipping of blocks like a pre-processor. It is a
* boolean property. The default value is <code>true</code>.
*
* @see #OPTION_SKIP_BLOCKS_PATTERN
*/
String OPTION_SKIP_BLOCKS = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocks";
/**
* Configures the pattern, to find the blocks to skip. It is a string
* property and contains of two parts, separated by {@code |}. The first
* part is the start pattern, the second part is the ending pattern. Default
* value is "{@code #if 0|#endif}".
*
* @see #DEFAULT_SKIP_BLOCKS_PATTERN
*/
String OPTION_SKIP_BLOCKS_PATTERN = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocksPattern";
@Deprecated // TODO what to do with this?
String DEFAULT_SKIP_BLOCKS_PATTERN = "#if 0|#endif";
void tokenize(TextDocument sourceCode, TokenFactory tokens) throws IOException;
void tokenize(TextDocument document, TokenFactory tokens) throws IOException;
static void tokenize(Tokenizer tokenizer, TextDocument textDocument, Tokens tokens) throws IOException {
try (TokenFactory tf = TokenFactory.forFile(textDocument, tokens)) {

@ -1,68 +0,0 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.internal;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.TreeSet;
import net.sourceforge.pmd.annotation.InternalApi;
@InternalApi
public abstract class LanguageServiceBase<T> {
protected interface NameExtractor<T> {
String getName(T language);
}
protected final Set<T> languages;
protected final Map<String, T> languagesByName;
protected final Map<String, T> languagesByTerseName;
protected LanguageServiceBase(final Class<T> serviceType, final Comparator<T> comparator,
final NameExtractor<T> nameExtractor, final NameExtractor<T> terseNameExtractor) {
Set<T> sortedLangs = new TreeSet<>(comparator);
// Use current class' classloader instead of the threads context classloader, see https://github.com/pmd/pmd/issues/1788
ServiceLoader<T> languageLoader = ServiceLoader.load(serviceType, getClass().getClassLoader());
Iterator<T> iterator = languageLoader.iterator();
while (true) {
// this loop is weird, but both hasNext and next may throw ServiceConfigurationError,
// it's more robust that way
try {
if (iterator.hasNext()) {
T language = iterator.next();
sortedLangs.add(language);
} else {
break;
}
} catch (UnsupportedClassVersionError | ServiceConfigurationError e) {
// Some languages require java8 and are therefore only available
// if java8 or later is used as runtime.
System.err.println("Ignoring language for PMD: " + e.toString());
}
}
// using a linked hash map to maintain insertion order
languages = Collections.unmodifiableSet(new LinkedHashSet<>(sortedLangs));
// TODO there may be languages with duplicate names
Map<String, T> byName = new LinkedHashMap<>();
Map<String, T> byTerseName = new LinkedHashMap<>();
for (T language : sortedLangs) {
byName.put(nameExtractor.getName(language), language);
byTerseName.put(terseNameExtractor.getName(language), language);
}
languagesByName = Collections.unmodifiableMap(byName);
languagesByTerseName = Collections.unmodifiableMap(byTerseName);
}
}

@ -2,9 +2,11 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang;
package net.sourceforge.pmd.lang.impl;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.LanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
/**
* Base class for language modules that only support CPD and not PMD.

@ -1,4 +1,4 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
@ -13,12 +13,10 @@ import java.util.Iterator;
import org.junit.jupiter.api.Test;
import net.sourceforge.pmd.lang.DummyLanguageModule;
import net.sourceforge.pmd.lang.Language;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.TextDocument;
import net.sourceforge.pmd.lang.document.TextFile;
import net.sourceforge.pmd.lang.java.JavaLanguageModule;
class MatchAlgorithmTest {
@ -38,15 +36,15 @@ class MatchAlgorithmTest {
@Test
void testSimple() throws IOException {
Language java = JavaLanguageModule.getInstance();
Tokenizer tokenizer = java.createCpdTokenizer(java.newPropertyBundle());
String fileName = "Foo.java";
TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, java.getDefaultVersion());
Language dummy = DummyLanguageModule.getInstance();
Tokenizer tokenizer = dummy.createCpdTokenizer(dummy.newPropertyBundle());
String fileName = "Foo.dummy";
TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, dummy.getDefaultVersion());
SourceManager sourceManager = new SourceManager(listOf(textFile));
Tokens tokens = new Tokens();
TextDocument sourceCode = sourceManager.get(textFile);
Tokenizer.tokenize(tokenizer, sourceCode, tokens);
assertEquals(41, tokens.size());
assertEquals(44, tokens.size());
MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5);
matchAlgorithm.findMatches();
@ -61,34 +59,10 @@ class MatchAlgorithmTest {
assertEquals(3, mark1.getBeginLine());
assertEquals(fileName, mark1.getFilename());
assertEquals(Chars.wrap(LINE_3), sourceManager.getSlice(mark1));
assertEquals(LINE_3 + "\n", sourceManager.getSlice(mark1).toString());
assertEquals(4, mark2.getBeginLine());
assertEquals(fileName, mark2.getFilename());
assertEquals(Chars.wrap(LINE_4), sourceManager.getSlice(mark2));
}
@Test
void testIgnore() throws IOException {
Language java = JavaLanguageModule.getInstance();
LanguagePropertyBundle bundle = java.newPropertyBundle();
bundle.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, true);
bundle.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, true);
Tokenizer tokenizer = java.createCpdTokenizer(bundle);
TextDocument sourceCode = TextDocument.readOnlyString(getSampleCode(), "Foo.java", java.getDefaultVersion());
Tokens tokens = new Tokens();
Tokenizer.tokenize(tokenizer, sourceCode, tokens);
MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5);
matchAlgorithm.findMatches();
Iterator<Match> matches = matchAlgorithm.matches();
Match match = matches.next();
assertFalse(matches.hasNext());
Iterator<Mark> marks = match.iterator();
marks.next();
marks.next();
marks.next();
assertFalse(marks.hasNext());
assertEquals(LINE_4 + "\n", sourceManager.getSlice(mark2).toString());
}
}

@ -7,6 +7,7 @@ package net.sourceforge.pmd.lang;
import java.util.Objects;
import net.sourceforge.pmd.RuleViolation;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.ast.DummyNode;
import net.sourceforge.pmd.lang.ast.DummyNode.DummyRootNode;
import net.sourceforge.pmd.lang.ast.ParseException;
@ -45,6 +46,14 @@ public class DummyLanguageModule extends SimpleLanguageModuleBase {
return (DummyLanguageModule) Objects.requireNonNull(LanguageRegistry.PMD.getLanguageByFullName(NAME));
}
@Override
public LanguagePropertyBundle newPropertyBundle() {
LanguagePropertyBundle bundle = super.newPropertyBundle();
bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_LITERALS);
bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS);
return bundle;
}
public LanguageVersion getVersionWhereParserThrows() {
return getVersion(PARSER_THROWS);
}

@ -6,9 +6,9 @@ package net.sourceforge.pmd.lang.cpp;
import net.sourceforge.pmd.cpd.CPPTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.properties.PropertyDescriptor;
import net.sourceforge.pmd.properties.PropertyFactory;
@ -19,7 +19,7 @@ public class CppLanguageModule extends CpdOnlyLanguageModuleBase {
public static final PropertyDescriptor<String> CPD_SKIP_BLOCKS =
PropertyFactory.stringProperty("cpdSkipBlocksPattern")
PropertyFactory.stringProperty("cpdSkipBlocksPattern")
.defaultValue("#if 0|#endif")
.desc("Specifies a start and end delimiter for CPD to completely ignore. "
+ "The delimiters are separated by a pipe |. The default skips code "

@ -4,11 +4,11 @@
package net.sourceforge.pmd.lang.cs;
import net.sourceforge.pmd.cpd.CsTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.cs.cpd.CsTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Defines the Language module for C#.

@ -1,12 +1,13 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.cs.cpd;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;

@ -2,28 +2,22 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.cs.cpd;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.Properties;
import org.jetbrains.annotations.NotNull;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.junit.jupiter.api.Test;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
class CsTokenizerTest extends CpdTextComparisonTest {
CsTokenizerTest() {
super(".cs");
}
@Override
protected String getResourcePrefix() {
return "../lang/cs/cpd/testdata";
super("cs", ".cs");
}
@Test
@ -107,23 +101,28 @@ class CsTokenizerTest extends CpdTextComparisonTest {
doTest("attributes", "_ignored", skipAttributes());
}
private Properties ignoreUsings() {
private LanguagePropertyConfig ignoreUsings() {
return properties(true, false, false);
}
private Properties skipLiteralSequences() {
private LanguagePropertyConfig skipLiteralSequences() {
return properties(false, true, false);
}
private Properties skipAttributes() {
private LanguagePropertyConfig skipAttributes() {
return properties(false, false, true);
}
private Properties properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) {
Properties properties = new Properties();
properties.setProperty(Tokenizer.IGNORE_USINGS, Boolean.toString(ignoreUsings));
properties.setProperty(Tokenizer.OPTION_IGNORE_LITERAL_SEQUENCES, Boolean.toString(ignoreLiteralSequences));
properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAttributes));
return properties;
@Override
public @NonNull LanguagePropertyConfig defaultProperties() {
return properties(false, false, false);
}
private LanguagePropertyConfig properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) {
return properties -> {
properties.setProperty(Tokenizer.CPD_IGNORE_IMPORTS, ignoreUsings);
properties.setProperty(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES, ignoreLiteralSequences);
properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAttributes);
};
}
}

@ -4,10 +4,10 @@
package net.sourceforge.pmd.lang.dart;
import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Language implementation for Dart

@ -6,8 +6,8 @@ package net.sourceforge.pmd.lang.fortran;
import net.sourceforge.pmd.cpd.AnyTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Language implementation for Fortran

@ -5,9 +5,9 @@
package net.sourceforge.pmd.lang.gherkin;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.gherkin.cpd.GherkinTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Language implementation for Gherkin.

@ -5,9 +5,9 @@
package net.sourceforge.pmd.lang.go;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.go.cpd.GoTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
public class GoLanguageModule extends CpdOnlyLanguageModuleBase {

@ -4,10 +4,10 @@
package net.sourceforge.pmd.lang.groovy;
import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Language implementation for Groovy

@ -22,8 +22,8 @@ import groovyjarjarantlr.TokenStreamException;
public class GroovyTokenizer implements Tokenizer {
@Override
public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
GroovyLexer lexer = new GroovyLexer(sourceCode.newReader());
public void tokenize(TextDocument document, TokenFactory tokens) {
GroovyLexer lexer = new GroovyLexer(document.newReader());
TokenStream tokenStream = lexer.plumb();
try {

@ -19,13 +19,13 @@ import net.sourceforge.pmd.lang.html.HtmlLanguageModule;
public class HtmlTokenizer implements Tokenizer {
@Override
public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
public void tokenize(TextDocument document, TokenFactory tokens) {
HtmlLanguageModule html = HtmlLanguageModule.getInstance();
try (LanguageProcessor processor = html.createProcessor(html.newPropertyBundle())) {
ParserTask task = new ParserTask(
sourceCode,
document,
SemanticErrorReporter.noop(),
LanguageProcessorRegistry.singleton(processor)
);

@ -1,12 +1,15 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.java.cpd;
import java.util.Deque;
import java.util.LinkedList;
import net.sourceforge.pmd.cpd.TokenEntry;
import net.sourceforge.pmd.cpd.TokenFactory;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.lang.TokenManager;

@ -1,27 +1,20 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.java.cpd;
import java.util.Properties;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
import net.sourceforge.pmd.lang.java.JavaLanguageModule;
// TODO - enable test
@Disabled("Needs to be enabled after java-grammar changes are finalized")
class JavaTokenizerTest extends CpdTextComparisonTest {
JavaTokenizerTest() {
super(".java");
}
@Override
protected String getResourcePrefix() {
return "../lang/java/cpd/testdata";
super(JavaLanguageModule.getInstance(), ".java");
}
@Test
@ -85,32 +78,32 @@ class JavaTokenizerTest extends CpdTextComparisonTest {
}
private static Properties ignoreAnnotations() {
private static LanguagePropertyConfig ignoreAnnotations() {
return properties(true, false, false);
}
private static Properties ignoreIdents() {
private static LanguagePropertyConfig ignoreIdents() {
return properties(false, false, true);
}
private static Properties ignoreLiterals() {
private static LanguagePropertyConfig ignoreLiterals() {
return properties(false, true, false);
}
@Override
public Properties defaultProperties() {
public LanguagePropertyConfig defaultProperties() {
return properties(false, false, false);
}
private static Properties properties(boolean ignoreAnnotations,
boolean ignoreLiterals,
boolean ignoreIdents) {
Properties properties = new Properties();
properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAnnotations));
properties.setProperty(Tokenizer.IGNORE_IDENTIFIERS, Boolean.toString(ignoreIdents));
properties.setProperty(Tokenizer.IGNORE_LITERALS, Boolean.toString(ignoreLiterals));
return properties;
private static LanguagePropertyConfig properties(boolean ignoreAnnotations,
boolean ignoreLiterals,
boolean ignoreIdents) {
return properties -> {
properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAnnotations);
properties.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, ignoreIdents);
properties.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, ignoreLiterals);
};
}

@ -4,12 +4,12 @@
package net.sourceforge.pmd.lang.ecmascript;
import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.Language;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.ecmascript.ast.EcmascriptParser;
import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer;
import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase;
/**

@ -5,10 +5,10 @@
package net.sourceforge.pmd.lang.kotlin;
import net.sourceforge.pmd.annotation.Experimental;
import net.sourceforge.pmd.cpd.KotlinTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase;
import net.sourceforge.pmd.lang.kotlin.cpd.KotlinTokenizer;
/**
* Language Module for Kotlin

@ -1,16 +1,16 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.kotlin.cpd;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer;
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
import net.sourceforge.pmd.lang.kotlin.ast.KotlinLexer;
/**
@ -24,7 +24,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
}
@Override
protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) {
protected TokenManager<AntlrToken> filterTokenStream(TokenManager<AntlrToken> tokenManager) {
return new KotlinTokenFilter(tokenManager);
}
@ -40,7 +40,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
private boolean discardingPackageAndImport = false;
private boolean discardingNL = false;
/* default */ KotlinTokenFilter(final AntlrTokenManager tokenManager) {
/* default */ KotlinTokenFilter(final TokenManager<AntlrToken> tokenManager) {
super(tokenManager);
}

@ -1,8 +1,8 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.kotlin.cpd;
import org.junit.jupiter.api.Test;
@ -14,11 +14,6 @@ class KotlinTokenizerTest extends CpdTextComparisonTest {
super("kotlin", ".kt");
}
@Override
protected String getResourcePrefix() {
return "../lang/kotlin/cpd/testdata";
}
@Test
void testComments() {
doTest("comment");

@ -5,8 +5,8 @@
package net.sourceforge.pmd.lang.lua;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.lua.cpd.LuaTokenizer;
/**
@ -18,8 +18,15 @@ public class LuaLanguageModule extends CpdOnlyLanguageModuleBase {
super(LanguageMetadata.withId("lua").name("Lua").extensions("lua"));
}
@Override
public LanguagePropertyBundle newPropertyBundle() {
LanguagePropertyBundle bundle = super.newPropertyBundle();
bundle.definePropertyDescriptor(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES);
return bundle;
}
@Override
public Tokenizer createCpdTokenizer(LanguagePropertyBundle bundle) {
return new LuaTokenizer();
return new LuaTokenizer(bundle);
}
}

Some files were not shown because too many files have changed in this diff Show More