From add597026cc3d6c2fb2885433aed0d61633f0772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 12 Feb 2023 15:28:34 +0100 Subject: [PATCH] Don't forget EOF token --- .../net/sourceforge/pmd/cpd/AnyTokenizer.java | 4 +-- .../net/sourceforge/pmd/cpd/CpdAnalysis.java | 20 ++++------- .../java/net/sourceforge/pmd/cpd/Mark.java | 2 +- .../net/sourceforge/pmd/cpd/TokenEntry.java | 35 ++++++++++++------- .../net/sourceforge/pmd/cpd/TokenFactory.java | 10 +++++- .../net/sourceforge/pmd/cpd/Tokenizer.java | 8 ++++- .../java/net/sourceforge/pmd/cpd/Tokens.java | 5 +++ .../pmd/cpd/internal/TokenizerBase.java | 4 +-- .../sourceforge/pmd/cpd/AnyTokenizerTest.java | 15 ++++---- .../sourceforge/pmd/cpd/TokenEntryTest.java | 2 +- .../sourceforge/pmd/cpd/CPPTokenizerTest.java | 2 +- .../sourceforge/pmd/cpd/GroovyTokenizer.java | 4 +-- .../pmd/lang/html/ast/HtmlTokenizer.java | 4 +-- .../pmd/cpd/MatchAlgorithmTest.java | 4 +-- .../pmd/cpd/test/CpdTextComparisonTest.kt | 15 ++++---- .../sourceforge/pmd/cpd/MatlabLanguage.java | 19 ---------- .../pmd/lang/matlab/MatlabLanguageModule.java | 29 +++++++++++++++ .../matlab}/cpd/MatlabTokenizer.java | 7 ++-- .../services/net.sourceforge.pmd.cpd.Language | 1 - .../net.sourceforge.pmd.lang.Language | 1 + .../matlab}/cpd/MatlabTokenizerTest.java | 11 ++---- 21 files changed, 118 insertions(+), 84 deletions(-) delete mode 100644 pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabLanguage.java create mode 100644 pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabLanguageModule.java rename pmd-matlab/src/main/java/net/sourceforge/pmd/{ => lang/matlab}/cpd/MatlabTokenizer.java (75%) delete mode 100644 pmd-matlab/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language create mode 100644 pmd-matlab/src/main/resources/META-INF/services/net.sourceforge.pmd.lang.Language rename pmd-matlab/src/test/java/net/sourceforge/pmd/{ => lang/matlab}/cpd/MatlabTokenizerTest.java (83%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java index 671644eae6..49000eaadd 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java @@ -62,7 +62,7 @@ public class AnyTokenizer implements Tokenizer { } @Override - public void tokenize(TextDocument sourceCode, TokenFactory tokenEntries) { + public void tokenize(TextDocument sourceCode, TokenFactory tokens) { Chars text = sourceCode.getText(); Matcher matcher = pattern.matcher(text); int lineNo = 1; @@ -87,7 +87,7 @@ public class AnyTokenizer implements Tokenizer { lineNo += StringUtil.lineNumberAt(image, image.length()) - 1; lastLineStart = matcher.start() + image.length() - ecol + 1; } - tokenEntries.recordToken(image, bline, bcol, lineNo, ecol); + tokens.recordToken(image, bline, bcol, lineNo, ecol); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java index 4d5c4eeccd..5b5cf439ac 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java @@ -112,20 +112,10 @@ public final class CpdAnalysis implements AutoCloseable { this.listener = cpdListener; } - private int doTokenize(TextDocument document, Tokenizer tokenizer, Tokens tokens) { + private int doTokenize(TextDocument document, Tokenizer tokenizer, Tokens tokens) throws IOException, TokenMgrError { LOGGER.trace("Tokenizing {}", document.getPathId()); int lastTokenSize = tokens.size(); - try { - tokenizer.tokenize(document, TokenFactory.forFile(document, tokens)); - } catch (IOException ioe) { - reporter.errorEx("Error while lexing.", ioe); - } catch (TokenMgrError e) { - e.setFileName(document.getDisplayName()); - reporter.errorEx("Error while lexing.", e); - throw e; - } finally { - tokens.addEof(); - } + Tokenizer.tokenize(tokenizer, document, tokens); return tokens.size() - lastTokenSize - 1; /* EOF */ } @@ -151,7 +141,11 @@ public final class CpdAnalysis implements AutoCloseable { int newTokens = doTokenize(textDocument, tokenizers.get(textFile.getLanguageVersion().getLanguage()), tokens); numberOfTokensPerFile.put(textDocument.getPathId(), newTokens); listener.addedFile(1); - } catch (TokenMgrError e) { + } catch (TokenMgrError | IOException e) { + if (e instanceof TokenMgrError) { + ((TokenMgrError) e).setFileName(textFile.getDisplayName()); + } + reporter.errorEx("Error while lexing.", e); // already reported savedState.restore(tokens); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java index 6e9c92213f..cc3519ed48 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java @@ -19,7 +19,7 @@ public class Mark implements Comparable { } public String getFilename() { - return this.token.getTokenSrcID(); + return this.token.getFileName(); } public int getBeginLine() { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java index 6c4ef2c9b8..ccfdbb11dc 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java @@ -8,7 +8,7 @@ public class TokenEntry implements Comparable { public static final TokenEntry EOF = new TokenEntry(); - private final String tokenSrcID; + private final String fileName; private final int beginLine; private final int beginColumn; private final int endColumn; @@ -18,15 +18,15 @@ public class TokenEntry implements Comparable { private TokenEntry() { this.identifier = 0; - this.tokenSrcID = "EOFMarker"; + this.fileName = "EOFMarker"; this.beginLine = -1; this.beginColumn = -1; this.endColumn = -1; } - TokenEntry(int imageId, String tokenSrcID, int beginLine, int beginColumn, int endLine, int endColumn, int index) { + TokenEntry(int imageId, String fileName, int beginLine, int beginColumn, int endLine, int endColumn, int index) { assert isOk(beginLine) && isOk(beginColumn) && isOk(endLine) && isOk(endColumn) : "Coordinates are 1-based"; - this.tokenSrcID = tokenSrcID; + this.fileName = fileName; this.beginLine = beginLine; this.beginColumn = beginColumn; this.endColumn = endColumn; @@ -40,8 +40,8 @@ public class TokenEntry implements Comparable { } - String getTokenSrcID() { - return tokenSrcID; + String getFileName() { + return fileName; } public int getBeginLine() { @@ -51,6 +51,7 @@ public class TokenEntry implements Comparable { /** * The column number where this token begins. * returns -1 if not available + * * @return the begin column number */ public int getBeginColumn() { @@ -60,17 +61,18 @@ public class TokenEntry implements Comparable { /** * The column number where this token ends. * returns -1 if not available + * * @return the end column number */ public int getEndColumn() { return endColumn; // TODO Java 1.8 make optional } - int getIdentifier() { + int getIdentifier() { return this.identifier; } - int getIndex() { + int getIndex() { return this.index; } @@ -79,7 +81,7 @@ public class TokenEntry implements Comparable { return hashCode; } - void setHashCode(int hashCode) { + void setHashCode(int hashCode) { this.hashCode = hashCode; } @@ -105,6 +107,18 @@ public class TokenEntry implements Comparable { return getIndex() - other.getIndex(); } + final void setImageIdentifier(int identifier) { + this.identifier = identifier; + } + + public String getImage(Tokens tokens) { + if (EOF.equals(this)) { + return "EOF"; + } + String image = tokens.imageFromId(this.identifier); + return image == null ? "--unknown--" : image; + } + @Override public String toString() { if (EOF.equals(this)) { @@ -113,7 +127,4 @@ public class TokenEntry implements Comparable { return Integer.toString(identifier); } - final void setImageIdentifier(int identifier) { - this.identifier = identifier; - } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenFactory.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenFactory.java index 796b40258c..84d4204901 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenFactory.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenFactory.java @@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.lang.document.FileLocation; import net.sourceforge.pmd.lang.document.TextDocument; -public interface TokenFactory { +public interface TokenFactory extends AutoCloseable { void recordToken(String image, int startLine, int startCol, int endLine, int endCol); @@ -19,6 +19,9 @@ public interface TokenFactory { TokenEntry peekLastToken(); + @Override + void close(); + static TokenFactory forFile(TextDocument file, Tokens sink) { return new TokenFactory() { final String name = file.getPathId(); @@ -37,6 +40,11 @@ public interface TokenFactory { public TokenEntry peekLastToken() { return sink.peekLastToken(); } + + @Override + public void close() { + sink.addEof(); + } }; } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java index 8abe88cd86..be20bf6570 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokenizer.java @@ -82,5 +82,11 @@ public interface Tokenizer { String DEFAULT_SKIP_BLOCKS_PATTERN = "#if 0|#endif"; - void tokenize(TextDocument sourceCode, TokenFactory tokenEntries) throws IOException; + void tokenize(TextDocument sourceCode, TokenFactory tokens) throws IOException; + + static void tokenize(Tokenizer tokenizer, TextDocument textDocument, Tokens tokens) throws IOException { + try (TokenFactory tf = TokenFactory.forFile(textDocument, tokens)) { + tokenizer.tokenize(textDocument, tf); + } + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java index 1e0841a125..b9b7fee8b2 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java @@ -9,6 +9,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; public class Tokens { @@ -32,6 +33,10 @@ public class Tokens { return images.computeIfAbsent(newImage, k -> images.size() + 1); } + String imageFromId(int i) { + return images.entrySet().stream().filter(it -> it.getValue() == i).findFirst().map(Entry::getKey).orElse(null); + } + public TokenEntry peekLastToken() { return get(size() - 1); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/TokenizerBase.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/TokenizerBase.java index 6168c2f58b..4d3d409b0f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/TokenizerBase.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/TokenizerBase.java @@ -30,11 +30,11 @@ public abstract class TokenizerBase> implements Tokeni } @Override - public void tokenize(TextDocument document, TokenFactory tokenEntries) throws IOException { + public void tokenize(TextDocument document, TokenFactory tokens) throws IOException { TokenManager tokenManager = filterTokenStream(makeLexerImpl(document)); T currentToken = tokenManager.getNextToken(); while (currentToken != null) { - processToken(tokenEntries, currentToken); + processToken(tokens, currentToken); currentToken = tokenManager.getNextToken(); } } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java index 05013bc139..18f6d2a76b 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.cpd; import static net.sourceforge.pmd.util.CollectionUtil.listOf; import static org.junit.jupiter.api.Assertions.assertEquals; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -18,19 +19,19 @@ import net.sourceforge.pmd.lang.document.TextDocument; class AnyTokenizerTest { @Test - void testMultiLineMacros() { + void testMultiLineMacros() throws IOException { AnyTokenizer tokenizer = new AnyTokenizer("//"); compareResult(tokenizer, TEST1, EXPECTED); } @Test - void testStringEscape() { + void testStringEscape() throws IOException { AnyTokenizer tokenizer = new AnyTokenizer("//"); compareResult(tokenizer, "a = \"oo\\n\"", listOf("a", "=", "\"oo\\n\"", "EOF")); } @Test - void testMultilineString() { + void testMultilineString() throws IOException { AnyTokenizer tokenizer = new AnyTokenizer("//"); Tokens tokens = compareResult(tokenizer, "a = \"oo\n\";", listOf("a", "=", "\"oo\n\"", ";", "EOF")); TokenEntry string = tokens.getTokens().get(2); @@ -50,11 +51,11 @@ class AnyTokenizerTest { * Tests that [core][cpd] AnyTokenizer doesn't count columns correctly #2760 is actually fixed. */ @Test - void testTokenPosition() { + void testTokenPosition() throws IOException { AnyTokenizer tokenizer = new AnyTokenizer(); TextDocument code = TextDocument.readOnlyString("a;\nbbbb\n;", "Foo.dummy", DummyLanguageModule.getInstance().getDefaultVersion()); Tokens tokens = new Tokens(); - tokenizer.tokenize(code, TokenFactory.forFile(code, tokens)); + Tokenizer.tokenize(tokenizer, code, tokens); TokenEntry bbbbToken = tokens.getTokens().get(2); assertEquals(2, bbbbToken.getBeginLine()); assertEquals(1, bbbbToken.getBeginColumn()); @@ -62,10 +63,10 @@ class AnyTokenizerTest { } - private Tokens compareResult(AnyTokenizer tokenizer, String source, List expectedImages) { + private Tokens compareResult(AnyTokenizer tokenizer, String source, List expectedImages) throws IOException { TextDocument code = TextDocument.readOnlyString(source, "Foo.dummy", DummyLanguageModule.getInstance().getDefaultVersion()); Tokens tokens = new Tokens(); - tokenizer.tokenize(code, TokenFactory.forFile(code, tokens)); + Tokenizer.tokenize(tokenizer, code, tokens); List tokenStrings = new ArrayList<>(); for (TokenEntry token : tokens.getTokens()) { diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/TokenEntryTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/TokenEntryTest.java index b393b0957f..c5248c722b 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/TokenEntryTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/TokenEntryTest.java @@ -16,7 +16,7 @@ class TokenEntryTest { tokens.addToken("public", "/var/Foo.java", 1, 2, 3, 4); TokenEntry mark = tokens.peekLastToken(); assertEquals(1, mark.getBeginLine()); - assertEquals("/var/Foo.java", mark.getTokenSrcID()); + assertEquals("/var/Foo.java", mark.getFileName()); assertEquals(0, mark.getIndex()); assertEquals(2, mark.getBeginColumn()); assertEquals(4, mark.getEndColumn()); diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java index 64cc27eb69..e0f03d28e6 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CPPTokenizerTest.java @@ -32,7 +32,7 @@ class CPPTokenizerTest extends CpdTextComparisonTest { @Test void testUTFwithBOM() { Tokenizer tokenizer = newTokenizer(dontSkipBlocks()); - Tokens tokens = tokenize(tokenizer, "\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n"); + Tokens tokens = tokenize(tokenizer, sourceCodeOf("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n")); assertEquals(15, tokens.size()); } diff --git a/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java b/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java index 1b9283c279..1c39b10da8 100644 --- a/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java +++ b/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java @@ -20,7 +20,7 @@ import groovyjarjarantlr.TokenStreamException; public class GroovyTokenizer implements Tokenizer { @Override - public void tokenize(TextDocument sourceCode, TokenFactory tokenEntries) { + public void tokenize(TextDocument sourceCode, TokenFactory tokens) { GroovyLexer lexer = new GroovyLexer(sourceCode.newReader()); TokenStream tokenStream = lexer.plumb(); @@ -42,7 +42,7 @@ public class GroovyTokenizer implements Tokenizer { lastLine = token.getLine(); // todo inaccurate } - tokenEntries.recordToken(tokenText, token.getLine(), token.getColumn(), lastLine, lastCol); + tokens.recordToken(tokenText, token.getLine(), token.getColumn(), lastLine, lastCol); token = tokenStream.nextToken(); } } catch (TokenStreamException err) { diff --git a/pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java b/pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java index 63fa4e45b6..530b2d4a12 100644 --- a/pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java +++ b/pmd-html/src/main/java/net/sourceforge/pmd/lang/html/ast/HtmlTokenizer.java @@ -19,7 +19,7 @@ import net.sourceforge.pmd.lang.html.HtmlLanguageModule; public class HtmlTokenizer implements Tokenizer { @Override - public void tokenize(TextDocument sourceCode, TokenFactory tokenEntries) { + public void tokenize(TextDocument sourceCode, TokenFactory tokens) { HtmlLanguageModule html = HtmlLanguageModule.getInstance(); try (LanguageProcessor processor = html.createProcessor(html.newPropertyBundle())) { @@ -33,7 +33,7 @@ public class HtmlTokenizer implements Tokenizer { HtmlParser parser = new HtmlParser(); ASTHtmlDocument root = parser.parse(task); - traverse(root, tokenEntries); + traverse(root, tokens); } catch (IOException e) { throw new UncheckedIOException(e); } catch (Exception e) { diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java index 9e680c83c6..4776e17029 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/cpd/MatchAlgorithmTest.java @@ -45,7 +45,7 @@ class MatchAlgorithmTest { SourceManager sourceManager = new SourceManager(listOf(textFile)); Tokens tokens = new Tokens(); TextDocument sourceCode = sourceManager.get(textFile); - tokenizer.tokenize(sourceCode, TokenFactory.forFile(sourceCode, tokens)); + Tokenizer.tokenize(tokenizer, sourceCode, tokens); assertEquals(41, tokens.size()); MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5); @@ -77,7 +77,7 @@ class MatchAlgorithmTest { Tokenizer tokenizer = java.createCpdTokenizer(bundle); TextDocument sourceCode = TextDocument.readOnlyString(getSampleCode(), "Foo.java", java.getDefaultVersion()); Tokens tokens = new Tokens(); - tokenizer.tokenize(sourceCode, TokenFactory.forFile(sourceCode, tokens)); + Tokenizer.tokenize(tokenizer, sourceCode, tokens); MatchAlgorithm matchAlgorithm = new MatchAlgorithm(tokens, 5); matchAlgorithm.findMatches(); diff --git a/pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt b/pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt index 0f983edf54..0532ae4de8 100644 --- a/pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt +++ b/pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt @@ -106,7 +106,7 @@ abstract class CpdTextComparisonTest( append('L').append(curLine).appendLine() } - formatLine(token).appendLine() + formatLine(token, tokens).appendLine() } } @@ -119,9 +119,9 @@ abstract class CpdTextComparisonTest( ) - private fun StringBuilder.formatLine(token: TokenEntry) = + private fun StringBuilder.formatLine(token: TokenEntry, tokens: Tokens) = formatLine( - escapedImage = escapeImage(token.toString()), + escapedImage = escapeImage(token.getImage(tokens)), bcol = token.beginColumn, ecol = token.endColumn ) @@ -167,11 +167,14 @@ abstract class CpdTextComparisonTest( private fun sourceCodeOf(fileData: FileData): TextDocument = TextDocument.readOnlyString(fileData.fileText, fileData.fileName, language.defaultVersion) + @JvmOverloads + fun sourceCodeOf(text: String, fileName: String = TextFile.UNKNOWN_FILENAME): FileData = + FileData(fileName = fileName, fileText = text) + fun tokenize(tokenizer: Tokenizer, fileData: FileData): Tokens = - Tokens().also { - val tokens = Tokens() + Tokens().also { tokens -> val source = sourceCodeOf(fileData) - tokenizer.tokenize(source, TokenFactory.forFile(source, tokens)) + Tokenizer.tokenize(tokenizer, source, tokens) } private companion object { diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabLanguage.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabLanguage.java deleted file mode 100644 index e4d4be1399..0000000000 --- a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabLanguage.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.cpd; - -/** - * Defines the Language module for Matlab - */ -public class MatlabLanguage extends AbstractLanguage { - - /** - * Creates a new instance of {@link MatlabLanguage} with the default - * extensions for matlab files. - */ - public MatlabLanguage() { - super("Matlab", "matlab", new MatlabTokenizer(), ".m"); - } -} diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabLanguageModule.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabLanguageModule.java new file mode 100644 index 0000000000..dc33c1970d --- /dev/null +++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/MatlabLanguageModule.java @@ -0,0 +1,29 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.matlab; + +import net.sourceforge.pmd.cpd.Tokenizer; +import net.sourceforge.pmd.lang.CpdOnlyLanguageModuleBase; +import net.sourceforge.pmd.lang.LanguagePropertyBundle; +import net.sourceforge.pmd.lang.matlab.cpd.MatlabTokenizer; + +/** + * Defines the Language module for Matlab + */ +public class MatlabLanguageModule extends CpdOnlyLanguageModuleBase { + + /** + * Creates a new instance of {@link MatlabLanguageModule} with the default + * extensions for matlab files. + */ + public MatlabLanguageModule() { + super(LanguageMetadata.withId("matlab").name("Matlab").extensions("m")); + } + + @Override + public Tokenizer createCpdTokenizer(LanguagePropertyBundle bundle) { + return new MatlabTokenizer(); + } +} diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/cpd/MatlabTokenizer.java similarity index 75% rename from pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java rename to pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/cpd/MatlabTokenizer.java index b2233923b3..1a275cd4ef 100644 --- a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java +++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/lang/matlab/cpd/MatlabTokenizer.java @@ -1,11 +1,12 @@ -/** +/* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.lang.matlab.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.document.TextDocument; import net.sourceforge.pmd.lang.matlab.ast.MatlabTokenKinds; @@ -17,6 +18,6 @@ public class MatlabTokenizer extends JavaCCTokenizer { @Override protected TokenManager makeLexerImpl(TextDocument doc) { - return MatlabTokenKinds.newTokenManager(doc); + return MatlabTokenKinds.newTokenManager(CharStream.create(doc)); } } diff --git a/pmd-matlab/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language b/pmd-matlab/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language deleted file mode 100644 index cbda640f65..0000000000 --- a/pmd-matlab/src/main/resources/META-INF/services/net.sourceforge.pmd.cpd.Language +++ /dev/null @@ -1 +0,0 @@ -net.sourceforge.pmd.cpd.MatlabLanguage diff --git a/pmd-matlab/src/main/resources/META-INF/services/net.sourceforge.pmd.lang.Language b/pmd-matlab/src/main/resources/META-INF/services/net.sourceforge.pmd.lang.Language new file mode 100644 index 0000000000..1e88740675 --- /dev/null +++ b/pmd-matlab/src/main/resources/META-INF/services/net.sourceforge.pmd.lang.Language @@ -0,0 +1 @@ +net.sourceforge.pmd.lang.matlab.MatlabLanguageModule diff --git a/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java b/pmd-matlab/src/test/java/net/sourceforge/pmd/lang/matlab/cpd/MatlabTokenizerTest.java similarity index 83% rename from pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java rename to pmd-matlab/src/test/java/net/sourceforge/pmd/lang/matlab/cpd/MatlabTokenizerTest.java index 73400c993e..68ed5fb0db 100644 --- a/pmd-matlab/src/test/java/net/sourceforge/pmd/cpd/MatlabTokenizerTest.java +++ b/pmd-matlab/src/test/java/net/sourceforge/pmd/lang/matlab/cpd/MatlabTokenizerTest.java @@ -1,8 +1,8 @@ -/** +/* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.cpd; +package net.sourceforge.pmd.lang.matlab.cpd; import org.junit.jupiter.api.Test; @@ -11,12 +11,7 @@ import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest; class MatlabTokenizerTest extends CpdTextComparisonTest { MatlabTokenizerTest() { - super(".m"); - } - - @Override - protected String getResourcePrefix() { - return "../lang/matlab/cpd/testdata"; + super("matlab", ".m"); } @Test