phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[core] Rename TokenMgrError to LexException

Browse Source 
See #4065
This commit is contained in:
Andreas Dangel 2024-01-11 15:49:21 +01:00
parent 2e7263af69
commit 55d91791c3
No known key found for this signature in database
GPG Key ID: 93450DF2DF9A3FA3
27 changed files with 54 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/pages/release_notes.md
View File

@ -159,6 +159,11 @@ The following previously deprecated classes have been removed:
If the current version is needed, then `Node.getTextDocument().getLanguageVersion()` can be used. This
is the version that has been selected via CLI `--use-version` parameter.
**Renamed classes**
* pmd-core
* {%jdoc_old core::lang.ast.TokenMgrError %} has been renamed to {% jdoc core::lang.ast.LexException %}
#### External Contributions
* [#4640](https://github.com/pmd/pmd/pull/4640): \[cli] Launch script fails if run via "bash pmd" - [Shai Bennathan](https://github.com/shai-bennathan) (@shai-bennathan)
* [#4673](https://github.com/pmd/pmd/pull/4673): \[javascript] CPD: Added support for decorator notation - [Wener](https://github.com/wener-tiobe) (@wener-tiobe)

7
javacc-wrapper.xml
View File

@ -280,6 +280,13 @@
<file name="${tokenmgr-file}" />
</replaceregexp>
<!-- Use own LexException instead of JavaCC's TokenMgrError -->
<replaceregexp>
<regexp pattern='throw new TokenMgrError\(EOFSeen' />
<substitution expression='throw new net.sourceforge.pmd.lang.ast.LexException(EOFSeen' />
<file name="${tokenmgr-file}" />
</replaceregexp>
<!-- Useless argument, also replace lex state ID with its name -->
<replaceregexp>
<regexp pattern='curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR\)' />

6
pmd-core/src/main/java/net/sourceforge/pmd/cpd/CpdAnalysis.java
View File

@ -23,7 +23,7 @@ import net.sourceforge.pmd.internal.util.IOUtil;
import net.sourceforge.pmd.lang.Language;
import net.sourceforge.pmd.lang.LanguagePropertyBundle;
import net.sourceforge.pmd.lang.ast.FileAnalysisException;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.LexException;
import net.sourceforge.pmd.lang.document.FileCollector;
import net.sourceforge.pmd.lang.document.FileId;
import net.sourceforge.pmd.lang.document.TextDocument;
@ -137,7 +137,7 @@ public final class CpdAnalysis implements AutoCloseable {
this.listener = cpdListener;
}
private int doTokenize(TextDocument document, Tokenizer tokenizer, Tokens tokens) throws IOException, TokenMgrError {
private int doTokenize(TextDocument document, Tokenizer tokenizer, Tokens tokens) throws IOException, LexException {
LOGGER.trace("Tokenizing {}", document.getFileId().getAbsolutePath());
int lastTokenSize = tokens.size();
Tokenizer.tokenize(tokenizer, document, tokens);
@ -170,7 +170,7 @@ public final class CpdAnalysis implements AutoCloseable {
int newTokens = doTokenize(textDocument, tokenizers.get(textFile.getLanguageVersion().getLanguage()), tokens);
numberOfTokensPerFile.put(textDocument.getFileId(), newTokens);
listener.addedFile(1);
} catch (TokenMgrError | IOException e) {
} catch (LexException | IOException e) {
if (e instanceof FileAnalysisException) { // NOPMD
((FileAnalysisException) e).setFileId(textFile.getFileId());
}

4
pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenFactory.java
View File

@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.LexException;
import net.sourceforge.pmd.lang.document.FileLocation;
import net.sourceforge.pmd.lang.document.TextDocument;
@ -43,7 +43,7 @@ public interface TokenFactory extends AutoCloseable {
recordToken(image, location.getStartLine(), location.getStartColumn(), location.getEndLine(), location.getEndColumn());
}
TokenMgrError makeLexException(int line, int column, String message, @Nullable Throwable cause);
LexException makeLexException(int line, int column, String message, @Nullable Throwable cause);
/**
* Sets the image of an existing token entry.

6
pmd-core/src/main/java/net/sourceforge/pmd/cpd/Tokens.java
View File

@ -14,7 +14,7 @@ import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import net.sourceforge.pmd.annotation.InternalApi;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.LexException;
import net.sourceforge.pmd.lang.document.FileId;
import net.sourceforge.pmd.lang.document.TextDocument;
@ -117,8 +117,8 @@ public class Tokens {
}
@Override
public TokenMgrError makeLexException(int line, int column, String message, @Nullable Throwable cause) {
return new TokenMgrError(line, column, fileId, message, cause);
public LexException makeLexException(int line, int column, String message, @Nullable Throwable cause) {
return new LexException(line, column, fileId, message, cause);
}
@Override

2
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java
View File

@ -16,7 +16,7 @@ import net.sourceforge.pmd.lang.document.FileLocation;
/**
* An exception that occurs while processing a file. Subtypes include
* <ul>
* <li>{@link TokenMgrError}: lexical syntax errors
* <li>{@link LexException}: lexical syntax errors
* <li>{@link ParseException}: syntax errors
* <li>{@link SemanticException}: exceptions occurring after the parsing
* phase, because the source code is semantically invalid

10
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java → pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/LexException.java
View File

@ -16,8 +16,10 @@ import net.sourceforge.pmd.util.StringUtil;
/**
* An error thrown during lexical analysis of a file.
*
* <p>Note: This exception has been called TokenMgrError in PMD 6.</p>
*/
public final class TokenMgrError extends FileAnalysisException {
public final class LexException extends FileAnalysisException {
private final int line;
private final int column;
@ -31,7 +33,7 @@ public final class TokenMgrError extends FileAnalysisException {
* @param message Message of the error
* @param cause Cause of the error, if any
*/
public TokenMgrError(int line, int column, @Nullable FileId filename, String message, @Nullable Throwable cause) {
public LexException(int line, int column, @Nullable FileId filename, String message, @Nullable Throwable cause) {
super(message, cause);
this.line = max(line, 1);
this.column = max(column, 1);
@ -44,7 +46,7 @@ public final class TokenMgrError extends FileAnalysisException {
* Constructor called by JavaCC.
*/
@InternalApi
public TokenMgrError(boolean eofSeen, String lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar) {
public LexException(boolean eofSeen, String lexStateName, int errorLine, int errorColumn, String errorAfter, char curChar) {
super(makeReason(eofSeen, lexStateName, errorAfter, curChar));
line = max(errorLine, 1);
column = max(errorColumn, 1);
@ -76,7 +78,7 @@ public final class TokenMgrError extends FileAnalysisException {
* @return A new exception
*/
@Override
public TokenMgrError setFileId(FileId fileId) {
public LexException setFileId(FileId fileId) {
super.setFileId(fileId);
return this;
}

4
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/antlr4/AntlrTokenManager.java
View File

@ -10,7 +10,7 @@ import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.LexException;
import net.sourceforge.pmd.lang.document.TextDocument;
/**
@ -63,7 +63,7 @@ public class AntlrTokenManager implements TokenManager<AntlrToken> {
final int charPositionInLine,
final String msg,
final RecognitionException ex) {
throw new TokenMgrError(line, charPositionInLine, textDoc.getFileId(), msg, ex);
throw new LexException(line, charPositionInLine, textDoc.getFileId(), msg, ex);
}
}

10
pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/TokenMgrErrorTest.java → pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/LexExceptionTest.java
View File

@ -8,30 +8,30 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.Test;
class TokenMgrErrorTest {
class LexExceptionTest {
@Test
void invalidLocation() {
TokenMgrError error = new TokenMgrError(2, 0, null, "test", null);
LexException error = new LexException(2, 0, null, "test", null);
// this shouldn't throw a IllegalArgumentException
assertEquals("line 2, column 1", error.location().startPosToString());
}
@Test
void invalidLocationJavaCC() {
TokenMgrError error = new TokenMgrError(false, "DEFAULT", 2, 0, "}", '\n');
LexException error = new LexException(false, "DEFAULT", 2, 0, "}", '\n');
// this shouldn't throw a IllegalArgumentException
assertEquals("line 2, column 1", error.location().startPosToString());
}
@Test
void validLocation() {
TokenMgrError error = new TokenMgrError(1, 1, null, "test", null);
LexException error = new LexException(1, 1, null, "test", null);
assertEquals("line 1, column 1", error.location().startPosToString());
}
@Test
void validLocationJavaCC() {
TokenMgrError error = new TokenMgrError(false, "DEFAULT", 1, 1, "}", '\n');
LexException error = new LexException(false, "DEFAULT", 1, 1, "}", '\n');
assertEquals("line 1, column 1", error.location().startPosToString());
}
}

3
pmd-cpp/etc/grammar/Cpp.jj
View File

@ -32,9 +32,6 @@ options {
PARSER_BEGIN(CppParserImpl)
package net.sourceforge.pmd.lang.cpp.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public final class CppParserImpl {
}

4
pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/cpd/CPPTokenizerTest.java
View File

@ -71,7 +71,7 @@ class CPPTokenizerTest extends CpdTextComparisonTest {
@Test
void testWrongUnicodeInIdentifier() {
expectTokenMgrError(" void main() { int ⚜ = __; }");
expectLexException(" void main() { int ⚜ = __; }");
}
@Test
@ -107,7 +107,7 @@ class CPPTokenizerTest extends CpdTextComparisonTest {
@Test
void testLexicalErrorFilename() {
expectTokenMgrError(sourceText("issue-1559"), dontSkipBlocks());
expectLexException(sourceText("issue-1559"), dontSkipBlocks());
}

4
pmd-cs/src/test/java/net/sourceforge/pmd/lang/cs/cpd/CsTokenizerTest.java
View File

@ -12,7 +12,7 @@ import org.junit.jupiter.api.Test;
import net.sourceforge.pmd.cpd.CpdLanguageProperties;
import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.LexException;
class CsTokenizerTest extends CpdTextComparisonTest {
@ -37,7 +37,7 @@ class CsTokenizerTest extends CpdTextComparisonTest {
@Test
void testOpenString() {
assertThrows(TokenMgrError.class, () -> doTest("unlexable_string"));
assertThrows(LexException.class, () -> doTest("unlexable_string"));
}
@Test

4
pmd-groovy/src/main/java/net/sourceforge/pmd/lang/groovy/ast/impl/antlr4/GroovyTokenManager.java
View File

@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.groovy.ast.impl.antlr4;
import org.apache.groovy.parser.antlr4.GroovyLexer;
import net.sourceforge.pmd.lang.TokenManager;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.LexException;
import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager;
import net.sourceforge.pmd.lang.document.TextDocument;
@ -81,7 +81,7 @@ public class GroovyTokenManager implements TokenManager<GroovyToken> {
final int charPositionInLine,
final String msg,
final RecognitionException ex) {
throw new TokenMgrError(line, charPositionInLine, textDoc.getFileId(), msg, ex);
throw new LexException(line, charPositionInLine, textDoc.getFileId(), msg, ex);
}
}

4
pmd-java/etc/grammar/Java.jjt
View File

@ -301,12 +301,8 @@ import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.Map;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.GenericToken;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
import net.sourceforge.pmd.lang.ast.Node;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.Node;
import net.sourceforge.pmd.lang.java.ast.JavaParserImplTokenManager.TokenContext;
import net.sourceforge.pmd.lang.java.types.JPrimitiveType.PrimitiveTypeKind;

4
pmd-java/src/test/java/net/sourceforge/pmd/lang/java/cpd/JavaTokenizerTest.java
View File

@ -16,7 +16,7 @@ import net.sourceforge.pmd.cpd.Tokenizer;
import net.sourceforge.pmd.cpd.Tokens;
import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
import net.sourceforge.pmd.cpd.test.LanguagePropertyConfig;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.LexException;
import net.sourceforge.pmd.lang.document.FileId;
import net.sourceforge.pmd.lang.document.TextDocument;
import net.sourceforge.pmd.lang.java.JavaLanguageModule;
@ -42,7 +42,7 @@ class JavaTokenizerTest extends CpdTextComparisonTest {
void testLexExceptionLocation() {
Tokenizer tokenizer = newTokenizer(defaultProperties());
Tokens tokens = new Tokens();
TokenMgrError lexException = assertThrows(TokenMgrError.class, () ->
LexException lexException = assertThrows(LexException.class, () ->
Tokenizer.tokenize(tokenizer,
// note: the source deliberately contains an unbalanced quote, unterminated string literal
TextDocument.readOnlyString("class F {\n String s=\"abc\";\"\n}\n", FileId.UNKNOWN, getLanguage().getDefaultVersion()),

2
pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/KotlinTestingDsl.kt
View File

@ -270,7 +270,7 @@ open class ParserTestCtx(testScope: TestScope,
Pair(true, null)
} catch (e: ParseException) {
Pair(false, e)
} catch (e: TokenMgrError) {
} catch (e: LexException) {
Pair(false, e)
}

3
pmd-javascript/etc/grammar/Ecmascript5.jj
View File

@ -15,9 +15,6 @@ options {
PARSER_BEGIN(Ecmascript5ParserImpl)
package net.sourceforge.pmd.lang.ecmascript5.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class Ecmascript5ParserImpl {
}

4
pmd-jsp/etc/grammar/Jsp.jjt
View File

@ -30,10 +30,6 @@ options {
PARSER_BEGIN(JspParserImpl)
package net.sourceforge.pmd.lang.jsp.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;
/**
* JSP Parser for PMD.
* @author Pieter, Application Engineers NV/SA, http://www.ae.be

15
pmd-lang-test/src/main/kotlin/net/sourceforge/pmd/cpd/test/CpdTextComparisonTest.kt
View File

@ -6,16 +6,13 @@ package net.sourceforge.pmd.cpd.test
import io.kotest.assertions.throwables.shouldThrow
import net.sourceforge.pmd.cpd.*
import net.sourceforge.pmd.lang.Language
import net.sourceforge.pmd.lang.LanguagePropertyBundle
import net.sourceforge.pmd.lang.LanguageRegistry
import net.sourceforge.pmd.lang.ast.TokenMgrError
import net.sourceforge.pmd.lang.ast.LexException
import net.sourceforge.pmd.lang.document.TextDocument
import net.sourceforge.pmd.lang.document.TextFile
import net.sourceforge.pmd.lang.document.FileId
import net.sourceforge.pmd.test.BaseTextComparisonTest
import org.apache.commons.lang3.StringUtils
import java.util.*
/**
* CPD test comparing a dump of a file against a saved baseline.
@ -73,18 +70,18 @@ abstract class CpdTextComparisonTest(
}
@JvmOverloads
fun expectTokenMgrError(
fun expectLexException(
source: String,
fileName: FileId = FileId.UNKNOWN,
properties: LanguagePropertyConfig = defaultProperties()
): TokenMgrError =
expectTokenMgrError(FileData(fileName, source), properties)
): LexException =
expectLexException(FileData(fileName, source), properties)
@JvmOverloads
fun expectTokenMgrError(
fun expectLexException(
fileData: FileData,
config: LanguagePropertyConfig = defaultProperties()
): TokenMgrError =
): LexException =
shouldThrow {
tokenize(newTokenizer(config), fileData)
}

3
pmd-matlab/etc/grammar/Matlab.jj
View File

@ -21,9 +21,6 @@ options {
PARSER_BEGIN(MatlabParserImpl)
package net.sourceforge.pmd.lang.matlab.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class MatlabParserImpl {
}

3
pmd-modelica/etc/grammar/Modelica.jjt
View File

@ -49,9 +49,6 @@ options {
PARSER_BEGIN(ModelicaParserImpl)
package net.sourceforge.pmd.lang.modelica.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
class ModelicaParserImpl {
}

6
pmd-objectivec/etc/grammar/ObjectiveC.jj
View File

@ -18,12 +18,6 @@ options {
PARSER_BEGIN(ObjectiveCParserImpl)
package net.sourceforge.pmd.lang.objectivec.ast;
import java.io.*;
import java.util.*;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
/**
* Grammar to parse ObjectiveC 2.0
* @author Michael Hall - Based on Parnel SableCC ObjectiveC grammar and javacc C grammar

8
pmd-plsql/etc/grammar/PLSQL.jjt
View File

@ -157,13 +157,11 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
package net.sourceforge.pmd.lang.plsql.ast;
import java.util.List;
import java.util.ArrayList;
import net.sourceforge.pmd.lang.ast.Node;
import net.sourceforge.pmd.lang.document.Chars;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.plsql.ast.internal.ParsingExclusion;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import java.util.ArrayList;
import java.util.List;
public class PLSQLParserImpl {

3
pmd-python/etc/grammar/Python.jj
View File

@ -17,9 +17,6 @@ PARSER_BEGIN(PythonParserImpl)
package net.sourceforge.pmd.lang.python.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class PythonParserImpl {
}

4
pmd-scala-modules/pmd-scala-common/src/test/java/net/sourceforge/pmd/cpd/ScalaTokenizerTest.java
View File

@ -9,7 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test;
import net.sourceforge.pmd.cpd.test.CpdTextComparisonTest;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.LexException;
import net.sourceforge.pmd.lang.scala.ScalaLanguageModule;
class ScalaTokenizerTest extends CpdTextComparisonTest {
@ -35,7 +35,7 @@ class ScalaTokenizerTest extends CpdTextComparisonTest {
@Test
void tokenizeFailTest() {
assertThrows(TokenMgrError.class, () -> doTest("unlexable_sample"));
assertThrows(LexException.class, () -> doTest("unlexable_sample"));
}
@Test

3
pmd-visualforce/etc/grammar/Vf.jjt
View File

@ -13,9 +13,6 @@ options {
PARSER_BEGIN(VfParserImpl)
package net.sourceforge.pmd.lang.vf.ast;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
public class VfParserImpl {

5
pmd-vm/etc/grammar/Vm.jjt
View File

@ -37,16 +37,11 @@ PARSER_BEGIN(VmParserImpl)
package net.sourceforge.pmd.lang.vm.ast;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import net.sourceforge.pmd.lang.ast.TokenMgrError;
import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream;
import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken;