phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix build

Browse Source
This commit is contained in:
Clément Fournier
2023-02-12 18:21:54 +01:00
parent fb9f49624d
commit ddbfc90c14
40 changed files with 174 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
justfile
View File

@ -20,6 +20,9 @@ alias i := install
cleanInstallEverything *FLAGS:
mvnd clean install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}}
installEverything *FLAGS:
mvnd install -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -fae {{FLAGS}}
testCore *FLAGS:
mvnd test checkstyle:check pmd:check -Dmaven.javadoc.skip -Dkotlin.compiler.incremental -pl pmd-core {{FLAGS}}
@ -46,6 +49,6 @@ lintChanged DIFF="master" *FLAGS="":
lint projects="pmd-java":
mvnd checkstyle:check pmd:check -pl {{projects}} -fae
lintAll:
mvnd checkstyle:check pmd:check -fae
lintAll *FLAGS:
mvnd checkstyle:check pmd:check -fae {{FLAGS}}

2
pmd-ant/src/test/resources/net/sourceforge/pmd/ant/xml/cpdtasktest.xml
View File

@ -8,7 +8,7 @@
<target name="all" depends="testBasic"/>
<target name="testBasic">
<cpd minimumTokenCount="20" outputFile="${pmd.home}/target/cpd.ant.tests" language="cpddummy">
<cpd minimumTokenCount="20" outputFile="${pmd.home}/target/cpd.ant.tests" language="dummy">
<fileset dir="${pmd.home}/src/main/">
<include name="**/*.java"/>
</fileset>

2
pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ApexLanguageModule.java
View File

@ -4,13 +4,13 @@
package net.sourceforge.pmd.lang.apex;
import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.Language;
import net.sourceforge.pmd.lang.LanguageModuleBase;
import net.sourceforge.pmd.lang.LanguageProcessor;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.apex.cpd.ApexTokenizer;
import apex.jorje.services.Version;

52
pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/cpd/ApexTokenizer.java
View File

@ -4,32 +4,60 @@
package net.sourceforge.pmd.lang.apex.cpd;
import java.io.IOException;
import java.util.Locale;
import org.antlr.v4.runtime.CharStream;
import org.antlr.runtime.ANTLRReaderStream;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.Lexer;
import org.antlr.runtime.Token;
import net.sourceforge.pmd.cpd.TokenFactory;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
import net.sourceforge.pmd.lang.apex.ApexJorjeLogging;
import net.sourceforge.pmd.lang.apex.ApexLanguageProperties;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.document.TextDocument;
import apex.jorje.parser.impl.ApexLexer;
public class ApexTokenizer implements Tokenizer {
public class ApexTokenizer extends AntlrTokenizer {
private final boolean caseSensitive;
public ApexTokenizer(ApexLanguageProperties properties) {
this.caseSensitive = properties.getProperty(Tokenizer.CPD_CASE_SENSITIVE);
ApexJorjeLogging.disableLogging();
}
@Override
protected String getImage(AntlrToken token) {
if (caseSensitive) {
return token.getImage();
}
return token.getImage().toLowerCase(Locale.ROOT);
}
public void tokenize(TextDocument document, TokenFactory tokenEntries) throws IOException {
ANTLRStringStream ass = new ANTLRReaderStream(document.newReader());
ApexLexer lexer = new ApexLexer(ass) {
@Override
protected org.antlr.v4.runtime.Lexer getLexerForSource(CharStream charStream) {
return new com.nawforce.runtime.parsers.ApexLexer(charStream);
public void emitErrorMessage(String msg) {
throw new TokenMgrError(getLine(), getCharPositionInLine(), getSourceName(), msg, null);
}
};
Token token = lexer.nextToken();
while (token.getType() != Token.EOF) {
if (token.getChannel() != Lexer.HIDDEN) {
String tokenText = token.getText();
if (!caseSensitive) {
tokenText = tokenText.toLowerCase(Locale.ROOT);
}
tokenEntries.recordToken(
tokenText,
token.getLine(),
token.getCharPositionInLine() + 1,
token.getLine(),
token.getCharPositionInLine() + tokenText.length() + 1
);
}
token = lexer.nextToken();
}
}
}

2
pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/CpdCommand.java
View File

@ -124,7 +124,7 @@ public class CpdCommand extends AbstractAnalysisPmdSubcommand {
final CPDConfiguration configuration = toConfiguration();
try (CpdAnalysis cpd = new CpdAnalysis(configuration)){
try (CpdAnalysis cpd = new CpdAnalysis(configuration)) {
MutableBoolean hasViolations = new MutableBoolean();
cpd.performAnalysis(report -> hasViolations.setValue(!report.getMatches().isEmpty()));

4
pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java
View File

@ -62,8 +62,8 @@ public class AnyTokenizer implements Tokenizer {
}
@Override
public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
Chars text = sourceCode.getText();
public void tokenize(TextDocument document, TokenFactory tokens) {
Chars text = document.getText();
Matcher matcher = pattern.matcher(text);
int lineNo = 1;
int lastLineStart = 0;

2
pmd-core/src/main/java/net/sourceforge/pmd/cpd/GUI.java
View File

@ -62,11 +62,11 @@ import javax.swing.table.TableModel;
import net.sourceforge.pmd.PMDVersion;
import net.sourceforge.pmd.cpd.renderer.CPDReportRenderer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.Language;
import net.sourceforge.pmd.lang.LanguageModuleBase.LanguageMetadata;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
public class GUI implements CPDListener {

41
pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java
View File

@ -26,10 +26,8 @@ public interface Tokenizer {
PropertyDescriptor<Boolean> CPD_ANONYMIZE_IDENTIFIERS =
PropertyFactory.booleanProperty("cpdAnonymizeIdentifiers")
.defaultValue(false)
.desc("Anonymize identifiers. They are still part of the token stream but all literals appear to have the same value.")
.desc("Anonymize identifiers. They are still part of the token stream but all identifiers appear to have the same value.")
.build();
PropertyDescriptor<Boolean> CPD_IGNORE_IMPORTS =
PropertyFactory.booleanProperty("cpdIgnoreImports")
.defaultValue(true)
@ -41,48 +39,17 @@ public interface Tokenizer {
.defaultValue(false)
.desc("Ignore metadata such as Java annotations or C# attributes.")
.build();
PropertyDescriptor<Boolean> CPD_CASE_SENSITIVE =
PropertyFactory.booleanProperty("cpdCaseSensitive")
.defaultValue(true)
.defaultValue(false)
.desc("Whether CPD should ignore the case of tokens. Affects all tokens.")
.build();
String IGNORE_LITERALS = "ignore_literals";
String IGNORE_IDENTIFIERS = "ignore_identifiers";
String IGNORE_ANNOTATIONS = "ignore_annotations";
/**
* Ignore sequences of literals (e.g, <code>0,0,0,0...</code>).
*/
String OPTION_IGNORE_LITERAL_SEQUENCES = "net.sourceforge.pmd.cpd.Tokenizer.skipLiteralSequences";
/**
* Ignore using directives in C#. The default value is <code>false</code>.
*/
String IGNORE_USINGS = "ignore_usings";
/**
* Enables or disabled skipping of blocks like a pre-processor. It is a
* boolean property. The default value is <code>true</code>.
*
* @see #OPTION_SKIP_BLOCKS_PATTERN
*/
String OPTION_SKIP_BLOCKS = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocks";
/**
* Configures the pattern, to find the blocks to skip. It is a string
* property and contains of two parts, separated by {@code |}. The first
* part is the start pattern, the second part is the ending pattern. Default
* value is "{@code #if 0|#endif}".
*
* @see #DEFAULT_SKIP_BLOCKS_PATTERN
*/
String OPTION_SKIP_BLOCKS_PATTERN = "net.sourceforge.pmd.cpd.Tokenizer.skipBlocksPattern";
@Deprecated // TODO what to do with this?
String DEFAULT_SKIP_BLOCKS_PATTERN = "#if 0|#endif";
void tokenize(TextDocument sourceCode, TokenFactory tokens) throws IOException;
void tokenize(TextDocument document, TokenFactory tokens) throws IOException;
static void tokenize(Tokenizer tokenizer, TextDocument textDocument, Tokens tokens) throws IOException {
try (TokenFactory tf = TokenFactory.forFile(textDocument, tokens)) {

68
pmd-core/src/main/java/net/sourceforge/pmd/internal/LanguageServiceBase.java
View File

@ -1,68 +0,0 @@
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.internal;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.ServiceConfigurationError;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.TreeSet;
import net.sourceforge.pmd.annotation.InternalApi;
@InternalApi
public abstract class LanguageServiceBase<T> {
protected interface NameExtractor<T> {
String getName(T language);
}
protected final Set<T> languages;
protected final Map<String, T> languagesByName;
protected final Map<String, T> languagesByTerseName;
protected LanguageServiceBase(final Class<T> serviceType, final Comparator<T> comparator,
final NameExtractor<T> nameExtractor, final NameExtractor<T> terseNameExtractor) {
Set<T> sortedLangs = new TreeSet<>(comparator);
// Use current class' classloader instead of the threads context classloader, see https://github.com/pmd/pmd/issues/1788
ServiceLoader<T> languageLoader = ServiceLoader.load(serviceType, getClass().getClassLoader());
Iterator<T> iterator = languageLoader.iterator();
while (true) {
// this loop is weird, but both hasNext and next may throw ServiceConfigurationError,
// it's more robust that way
try {
if (iterator.hasNext()) {
T language = iterator.next();
sortedLangs.add(language);
} else {
break;
}
} catch (UnsupportedClassVersionError | ServiceConfigurationError e) {
// Some languages require java8 and are therefore only available
// if java8 or later is used as runtime.
System.err.println("Ignoring language for PMD: " + e.toString());
}
}
// using a linked hash map to maintain insertion order
languages = Collections.unmodifiableSet(new LinkedHashSet<>(sortedLangs));
// TODO there may be languages with duplicate names
Map<String, T> byName = new LinkedHashMap<>();
Map<String, T> byTerseName = new LinkedHashMap<>();
for (T language : sortedLangs) {
byName.put(nameExtractor.getName(language), language);
byTerseName.put(terseNameExtractor.getName(language), language);
}
languagesByName = Collections.unmodifiableMap(byName);
languagesByTerseName = Collections.unmodifiableMap(byTerseName);
}
}

4
pmd-core/src/main/java/net/sourceforge/pmd/lang/CpdOnlyLanguageModuleBase.java → pmd-core/src/main/java/net/sourceforge/pmd/lang/impl/CpdOnlyLanguageModuleBase.java
View File

@ -2,9 +2,11 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.lang;
package net.sourceforge.pmd.lang.impl;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.LanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
/**
* Base class for language modules that only support CPD and not PMD.

44
pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java → pmd-core/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java
View File

@ -1,4 +1,4 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
@ -13,12 +13,10 @@ import java.util.Iterator;
import org.junit.jupiter.api.Test;
import net.sourceforge.pmd.lang.DummyLanguageModule;
import net.sourceforge.pmd.lang.Language;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.document.TextDocument;
import net.sourceforge.pmd.lang.document.TextFile;
import net.sourceforge.pmd.lang.java.JavaLanguageModule;
class MatchAlgorithmTest {
@ -38,15 +36,15 @@ class MatchAlgorithmTest {
@Test
void testSimple() throws IOException {
Language java = JavaLanguageModule.getInstance();
Tokenizer tokenizer = java.createCpdTokenizer(java.newPropertyBundle());
String fileName = "Foo.java";
TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, java.getDefaultVersion());
Language dummy = DummyLanguageModule.getInstance();
Tokenizer tokenizer = dummy.createCpdTokenizer(dummy.newPropertyBundle());
String fileName = "Foo.dummy";
TextFile textFile = TextFile.forCharSeq(getSampleCode(), fileName, dummy.getDefaultVersion());
SourceManager sourceManager = new SourceManager(listOf(textFile));
Tokens tokens = new Tokens();
TextDocument sourceCode = sourceManager.get(textFile);
Tokenizer.tokenize(tokenizer, sourceCode, tokens);
assertEquals(41, tokens.size());
assertEquals(44, tokens.size());
MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5);
matchAlgorithm.findMatches();
@ -61,34 +59,10 @@ class MatchAlgorithmTest {
assertEquals(3, mark1.getBeginLine());
assertEquals(fileName, mark1.getFilename());
assertEquals(Chars.wrap(LINE_3), sourceManager.getSlice(mark1));
assertEquals(LINE_3 + "\n", sourceManager.getSlice(mark1).toString());
assertEquals(4, mark2.getBeginLine());
assertEquals(fileName, mark2.getFilename());
assertEquals(Chars.wrap(LINE_4), sourceManager.getSlice(mark2));
}
@Test
void testIgnore() throws IOException {
Language java = JavaLanguageModule.getInstance();
LanguagePropertyBundle bundle = java.newPropertyBundle();
bundle.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, true);
bundle.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, true);
Tokenizer tokenizer = java.createCpdTokenizer(bundle);
TextDocument sourceCode = TextDocument.readOnlyString(getSampleCode(), "Foo.java", java.getDefaultVersion());
Tokens tokens = new Tokens();
Tokenizer.tokenize(tokenizer, sourceCode, tokens);
MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5);
matchAlgorithm.findMatches();
Iterator<Match> matches = matchAlgorithm.matches();
Match match = matches.next();
assertFalse(matches.hasNext());
Iterator<Mark> marks = match.iterator();
marks.next();
marks.next();
marks.next();
assertFalse(marks.hasNext());
assertEquals(LINE_4 + "\n", sourceManager.getSlice(mark2).toString());
}
}

9
pmd-core/src/test/java/net/sourceforge/pmd/lang/DummyLanguageModule.java
View File

@ -7,6 +7,7 @@ package net.sourceforge.pmd.lang;
import java.util.Objects;
import net.sourceforge.pmd.RuleViolation;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.ast.DummyNode;
import net.sourceforge.pmd.lang.ast.DummyNode.DummyRootNode;
import net.sourceforge.pmd.lang.ast.ParseException;
@ -45,6 +46,14 @@ public class DummyLanguageModule extends SimpleLanguageModuleBase {
return (DummyLanguageModule) Objects.requireNonNull(LanguageRegistry.PMD.getLanguageByFullName(NAME));
}
@Override
public LanguagePropertyBundle newPropertyBundle() {
LanguagePropertyBundle bundle = super.newPropertyBundle();
bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_LITERALS);
bundle.definePropertyDescriptor(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS);
return bundle;
}
public LanguageVersion getVersionWhereParserThrows() {
return getVersion(PARSER_THROWS);
}

2
pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/CppLanguageModule.java
View File

@ -6,9 +6,9 @@ package net.sourceforge.pmd.lang.cpp;
import net.sourceforge.pmd.cpd.CPPTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.properties.PropertyDescriptor;
import net.sourceforge.pmd.properties.PropertyFactory;

4
pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/CsLanguageModule.java
View File

@ -4,11 +4,11 @@
package net.sourceforge.pmd.lang.cs;
import net.sourceforge.pmd.cpd.CsTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.cs.cpd.CsTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Defines the Language module for C#.

5
pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java → pmd-cs/src/main/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizer.java
View File

@ -1,12 +1,13 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.cs.cpd;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter;

39
pmd-cs/src/test/java/net/sourceforge/pmd/cpd/CsTokenizerTest.java → pmd-cs/src/test/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizerTest.java
View File

@ -2,28 +2,22 @@
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.cs.cpd;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.Properties;
import org.jetbrains.annotations.NotNull;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.junit.jupiter.api.Test;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
class CsTokenizerTest extends CpdTextComparisonTest {
CsTokenizerTest() {
super(".cs");
}
@Override
protected String getResourcePrefix() {
return "../lang/cs/cpd/testdata";
super("cs", ".cs");
}
@Test
@ -107,23 +101,28 @@ class CsTokenizerTest extends CpdTextComparisonTest {
doTest("attributes", "_ignored", skipAttributes());
}
private Properties ignoreUsings() {
private LanguagePropertyConfig ignoreUsings() {
return properties(true, false, false);
}
private Properties skipLiteralSequences() {
private LanguagePropertyConfig skipLiteralSequences() {
return properties(false, true, false);
}
private Properties skipAttributes() {
private LanguagePropertyConfig skipAttributes() {
return properties(false, false, true);
}
private Properties properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) {
Properties properties = new Properties();
properties.setProperty(Tokenizer.IGNORE_USINGS, Boolean.toString(ignoreUsings));
properties.setProperty(Tokenizer.OPTION_IGNORE_LITERAL_SEQUENCES, Boolean.toString(ignoreLiteralSequences));
properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAttributes));
return properties;
@Override
public @NonNull LanguagePropertyConfig defaultProperties() {
return properties(false, false, false);
}
private LanguagePropertyConfig properties(boolean ignoreUsings, boolean ignoreLiteralSequences, boolean ignoreAttributes) {
return properties -> {
properties.setProperty(Tokenizer.CPD_IGNORE_IMPORTS, ignoreUsings);
properties.setProperty(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES, ignoreLiteralSequences);
properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAttributes);
};
}
}

4
pmd-dart/src/main/java/net/sourceforge/pmd/lang/dart/DartLanguageModule.java
View File

@ -4,10 +4,10 @@
package net.sourceforge.pmd.lang.dart;
import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.dart.cpd.DartTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Language implementation for Dart

2
pmd-fortran/src/main/java/net/sourceforge/pmd/lang/fortran/FortranLanguageModule.java
View File

@ -6,8 +6,8 @@ package net.sourceforge.pmd.lang.fortran;
import net.sourceforge.pmd.cpd.AnyTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Language implementation for Fortran

2
pmd-gherkin/src/main/java/net/sourceforge/pmd/lang/gherkin/GherkinLanguageModule.java
View File

@ -5,9 +5,9 @@
package net.sourceforge.pmd.lang.gherkin;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.gherkin.cpd.GherkinTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Language implementation for Gherkin.

2
pmd-go/src/main/java/net/sourceforge/pmd/lang/go/GoLanguageModule.java
View File

@ -5,9 +5,9 @@
package net.sourceforge.pmd.lang.go;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.go.cpd.GoTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
public class GoLanguageModule extends CpdOnlyLanguageModuleBase {

4
pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/GroovyLanguageModule.java
View File

@ -4,10 +4,10 @@
package net.sourceforge.pmd.lang.groovy;
import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.groovy.cpd.GroovyTokenizer;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
/**
* Language implementation for Groovy

4
pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/cpd/GroovyTokenizer.java
View File

@ -22,8 +22,8 @@ import groovyjarjarantlr.TokenStreamException;
public class GroovyTokenizer implements Tokenizer {
@Override
public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
GroovyLexer lexer = new GroovyLexer(sourceCode.newReader());
public void tokenize(TextDocument document, TokenFactory tokens) {
GroovyLexer lexer = new GroovyLexer(document.newReader());
TokenStream tokenStream = lexer.plumb();
try {

4
pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java
View File

@ -19,13 +19,13 @@ import net.sourceforge.pmd.lang.html.HtmlLanguageModule;
public class HtmlTokenizer implements Tokenizer {
@Override
public void tokenize(TextDocument sourceCode, TokenFactory tokens) {
public void tokenize(TextDocument document, TokenFactory tokens) {
HtmlLanguageModule html = HtmlLanguageModule.getInstance();
try (LanguageProcessor processor = html.createProcessor(html.newPropertyBundle())) {
ParserTask task = new ParserTask(
sourceCode,
document,
SemanticErrorReporter.noop(),
LanguageProcessorRegistry.singleton(processor)
);

7
pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java → pmd-java/src/main/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizer.java
View File

@ -1,12 +1,15 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.java.cpd;
import java.util.Deque;
import java.util.LinkedList;
import net.sourceforge.pmd.cpd.TokenEntry;
import net.sourceforge.pmd.cpd.TokenFactory;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer;
import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter;
import net.sourceforge.pmd.lang.TokenManager;

39
pmd-java/src/test/java/net/sourceforge/pmd/cpd/JavaTokenizerTest.java → pmd-java/src/test/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizerTest.java
View File

@ -1,27 +1,20 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.java.cpd;
import java.util.Properties;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
import net.sourceforge.pmd.lang.java.JavaLanguageModule;
// TODO - enable test
@Disabled("Needs to be enabled after java-grammar changes are finalized")
class JavaTokenizerTest extends CpdTextComparisonTest {
JavaTokenizerTest() {
super(".java");
}
@Override
protected String getResourcePrefix() {
return "../lang/java/cpd/testdata";
super(JavaLanguageModule.getInstance(), ".java");
}
@Test
@ -85,32 +78,32 @@ class JavaTokenizerTest extends CpdTextComparisonTest {
}
private static Properties ignoreAnnotations() {
private static LanguagePropertyConfig ignoreAnnotations() {
return properties(true, false, false);
}
private static Properties ignoreIdents() {
private static LanguagePropertyConfig ignoreIdents() {
return properties(false, false, true);
}
private static Properties ignoreLiterals() {
private static LanguagePropertyConfig ignoreLiterals() {
return properties(false, true, false);
}
@Override
public Properties defaultProperties() {
public LanguagePropertyConfig defaultProperties() {
return properties(false, false, false);
}
private static Properties properties(boolean ignoreAnnotations,
private static LanguagePropertyConfig properties(boolean ignoreAnnotations,
boolean ignoreLiterals,
boolean ignoreIdents) {
Properties properties = new Properties();
properties.setProperty(Tokenizer.IGNORE_ANNOTATIONS, Boolean.toString(ignoreAnnotations));
properties.setProperty(Tokenizer.IGNORE_IDENTIFIERS, Boolean.toString(ignoreIdents));
properties.setProperty(Tokenizer.IGNORE_LITERALS, Boolean.toString(ignoreLiterals));
return properties;
return properties -> {
properties.setProperty(Tokenizer.CPD_IGNORE_METADATA, ignoreAnnotations);
properties.setProperty(Tokenizer.CPD_ANONYMIZE_IDENTIFIERS, ignoreIdents);
properties.setProperty(Tokenizer.CPD_ANONYMIZE_LITERALS, ignoreLiterals);
};
}

2
pmd-javascript/src/main/java/net/sourceforge/pmd/lang/ecmascript/EcmascriptLanguageModule.java
View File

@ -4,12 +4,12 @@
package net.sourceforge.pmd.lang.ecmascript;
import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.Language;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.LanguageRegistry;
import net.sourceforge.pmd.lang.ecmascript.ast.EcmascriptParser;
import net.sourceforge.pmd.lang.ecmascript.cpd.EcmascriptTokenizer;
import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase;
/**

2
pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/KotlinLanguageModule.java
View File

@ -5,10 +5,10 @@
package net.sourceforge.pmd.lang.kotlin;
import net.sourceforge.pmd.annotation.Experimental;
import net.sourceforge.pmd.cpd.KotlinTokenizer;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.impl.SimpleLanguageModuleBase;
import net.sourceforge.pmd.lang.kotlin.cpd.KotlinTokenizer;
/**
* Language Module for Kotlin

10
pmd-kotlin/src/main/java/net/sourceforge/pmd/cpd/KotlinTokenizer.java → pmd-kotlin/src/main/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizer.java
View File

@ -1,16 +1,16 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.kotlin.cpd;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.Lexer;
import net.sourceforge.pmd.cpd.internal.AntlrTokenizer;
import net.sourceforge.pmd.cpd.token.AntlrTokenFilter;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
import net.sourceforge.pmd.lang.kotlin.ast.KotlinLexer;
/**
@ -24,7 +24,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
}
@Override
protected AntlrTokenFilter getTokenFilter(final AntlrTokenManager tokenManager) {
protected TokenManager<AntlrToken> filterTokenStream(TokenManager<AntlrToken> tokenManager) {
return new KotlinTokenFilter(tokenManager);
}
@ -40,7 +40,7 @@ public class KotlinTokenizer extends AntlrTokenizer {
private boolean discardingPackageAndImport = false;
private boolean discardingNL = false;
/* default */ KotlinTokenFilter(final AntlrTokenManager tokenManager) {
/* default */ KotlinTokenFilter(final TokenManager<AntlrToken> tokenManager) {
super(tokenManager);
}

9
pmd-kotlin/src/test/java/net/sourceforge/pmd/cpd/KotlinTokenizerTest.java → pmd-kotlin/src/test/java/net/sourceforge/pmd/lang/kotlin/cpd/KotlinTokenizerTest.java
View File

@ -1,8 +1,8 @@
/**
/*
* BSD-style license; for more info see http://pmd.sourceforge.net/license.html
*/
package net.sourceforge.pmd.cpd;
package net.sourceforge.pmd.lang.kotlin.cpd;
import org.junit.jupiter.api.Test;
@ -14,11 +14,6 @@ class KotlinTokenizerTest extends CpdTextComparisonTest {
super("kotlin", ".kt");
}
@Override
protected String getResourcePrefix() {
return "../lang/kotlin/cpd/testdata";
}
@Test
void testComments() {
doTest("comment");

11
pmd-lua/src/main/java/net/sourceforge/pmd/lang/lua/LuaLanguageModule.java
View File

@ -5,8 +5,8 @@
package net.sourceforge.pmd.lang.lua;
import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.impl.CpdOnlyLanguageModuleBase;
import net.sourceforge.pmd.lang.lua.cpd.LuaTokenizer;
/**
@ -18,8 +18,15 @@ public class LuaLanguageModule extends CpdOnlyLanguageModuleBase {
super(LanguageMetadata.withId("lua").name("Lua").extensions("lua"));
}
@Override
public LanguagePropertyBundle newPropertyBundle() {
LanguagePropertyBundle bundle = super.newPropertyBundle();
bundle.definePropertyDescriptor(Tokenizer.CPD_IGNORE_LITERAL_SEQUENCES);
return bundle;
}
@Override
public Tokenizer createCpdTokenizer(LanguagePropertyBundle bundle) {
return new LuaTokenizer();
return new LuaTokenizer(bundle);
}
}

Some files were not shown because too many files have changed in this diff Show More