phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix backup

Browse Source
This commit is contained in:
Clément Fournier
2020-04-18 12:31:11 +02:00
parent 052103ff0c
commit d3de455ede
3 changed files with 88 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java
View File

@ -4,6 +4,8 @@
package net.sourceforge.pmd.lang.ast.impl.io;
import static java.lang.Integer.max;
import java.io.EOFException;
import net.sourceforge.pmd.util.document.Chars;
@ -27,11 +29,10 @@ class EscapeTracker {
*
* Eg for "a\u00a0b" (translates as "a b"), the buffer looks like
* [a u00a0b]
* ^ this char has been replaced with the translated value of the escape
* ^^^^^ these characters are only present in the input, we jump over them when reading
* ^ off
* ^ invalid
* ^ off + len
* ^ (off) this char has been replaced with the translated value of the escape
* ^^^^^ (off + len - invalid) these characters are only present in the input, we jump over them when reading
* ^ (invalid)
* ^ (off + len)
*
* The escape record is (1,6,2)
*
@ -66,17 +67,24 @@ class EscapeTracker {
}
private int inOff(int idx) {
assert idx < nextFreeIdx;
return escapeRecords[idx];
}
private int inLen(int idx) {
assert idx < nextFreeIdx;
return escapeRecords[idx + 1];
}
private int invalidIdx(int idx) {
assert idx < nextFreeIdx;
return escapeRecords[idx + 2];
}
private int indexAfter(int idx) {
return inOff(idx) + inLen(idx);
}
/**
* Convert an offset in the translated file into an offset in
* the untranslated input.
@ -157,8 +165,9 @@ class EscapeTracker {
char c;
if (nextEscape < maxEscape() && pos == invalidIdx(nextEscape)) {
pos += inLen(nextEscape); // add escape length
int pos = indexAfter(nextEscape); // jump past escape
c = buf.charAt(pos);
this.pos = pos + 1;
this.nextEscape += RECORD_SIZE;
} else {
c = buf.charAt(pos);
@ -168,6 +177,7 @@ class EscapeTracker {
return c;
}
void backup(int numChars) {
ensureMarked();
if (numChars > markLength()) {
@ -180,20 +190,30 @@ class EscapeTracker {
if (nextEscape <= 0) {
pos -= numChars; // then there were no escapes before the 'pos'
} else {
int inoff = pos;
for (int i = maxEscape() - RECORD_SIZE; i >= 0 && numChars > 0; i -= RECORD_SIZE) {
int esc = inOff(i);
if (esc == inoff) {
inoff -= inLen(i);
} else if (esc > inoff) {
// then the current escape was before what we're looking at
int newOff = pos;
for (int i = nextEscape - RECORD_SIZE; i >= 0 && numChars > 0; i -= RECORD_SIZE) {
// aa __|||bb
// ^ invalid
// ^^^ jumped
// ^ inOff
// ^^ translated
int nc = numChars;
numChars -= newOff - indexAfter(i);
newOff = max(indexAfter(i), newOff - nc);
if (numChars <= 0) { // skip "bb", ie everything after the escape
break;
} else {
inoff--;
}
newOff = invalidIdx(i) - 1; // jump back over the escape |||
numChars--;
nextEscape = i;
if (numChars <= 0) {
break;
}
}
pos = inoff - numChars;
pos = newOff - numChars; // numChars is the remainder
}
}
@ -218,11 +238,11 @@ class EscapeTracker {
int esc = markEscape;
while (cur < pos && esc < nextEscape) {
sb.append(buf, cur, invalidIdx(esc));
cur = inOff(esc) + inLen(esc);
cur = indexAfter(esc);
esc += RECORD_SIZE;
}
// no more escape in the range, append everything until the pos
sb.append(buf, cur, pos + 1);
sb.append(buf, cur, pos);
assert sb.length() - prevLength == markLength() : sb + " should have length " + markLength();
}
}

2
pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java
View File

@ -37,7 +37,7 @@ public abstract class JjtreeParserAdapter<R extends RootNode> implements Parser
CharStream charStream = NewCharStream.open(doc);
return parseImpl(charStream, task);
} catch (IOException e) {
throw new TokenMgrError(-1, -1, fileName, "IO error", e);
throw new TokenMgrError(-1, -1, task.getFileDisplayName(), "IO error", e);
} catch (TokenMgrError tme) {
throw tme.setFileName(task.getFileDisplayName());
}

57
pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java
View File

@ -15,7 +15,10 @@ import org.junit.Test;
import org.junit.rules.ExpectedException;
import net.sourceforge.pmd.lang.ast.CharStream;
import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader;
import net.sourceforge.pmd.lang.ast.impl.io.JavaInputReader;
import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream;
import net.sourceforge.pmd.util.document.Chars;
import net.sourceforge.pmd.util.document.TextDocument;
public class CharStreamImplTest {
@ -26,7 +29,7 @@ public class CharStreamImplTest {
@Test
public void testReadZeroChars() throws IOException {
CharStream stream = getCharStream("");
CharStream stream = simpleCharStream("");
expect.expect(EOFException.class);
@ -42,7 +45,7 @@ public class CharStreamImplTest {
@Test
public void testReadEofChars() throws IOException {
CharStream stream = getCharStream("");
CharStream stream = simpleCharStream("");
expect.expect(EOFException.class);
@ -58,7 +61,7 @@ public class CharStreamImplTest {
@Test
public void testMultipleEofReads() throws IOException {
CharStream stream = getCharStream("");
CharStream stream = simpleCharStream("");
for (int i = 0; i < 3; i++) {
try {
@ -74,7 +77,7 @@ public class CharStreamImplTest {
@Test
public void testReadStuff() throws IOException {
CharStream stream = getCharStream("abcd");
CharStream stream = simpleCharStream("abcd");
assertEquals('a', stream.readChar());
assertEquals('b', stream.readChar());
@ -88,7 +91,7 @@ public class CharStreamImplTest {
@Test
public void testReadBacktrack() throws IOException {
CharStream stream = getCharStream("abcd");
CharStream stream = simpleCharStream("abcd");
assertEquals('a', stream.BeginToken());
assertEquals('b', stream.readChar());
@ -106,14 +109,52 @@ public class CharStreamImplTest {
stream.readChar();
}
public CharStream getCharStream(String abcd) throws IOException {
@Test
public void testReadBacktrackWithEscapes() throws IOException {
CharStream stream = javaCharStream("__\\u00a0_\\u00a0_");
assertEquals('_', stream.BeginToken());
assertEquals('_', stream.readChar());
assertEquals('\u00a0', stream.readChar());
assertEquals('_', stream.readChar());
assertEquals("__\u00a0_", stream.GetImage());
stream.backup(2);
assertEquals('\u00a0', stream.readChar());
assertEquals('_', stream.readChar());
assertEquals('\u00a0', stream.readChar());
assertEquals("__\u00a0_\u00a0", stream.GetImage());
assertEquals('_', stream.readChar());
stream.backup(2);
assertEquals('\u00a0', stream.BeginToken());
assertEquals('_', stream.readChar());
assertEquals("\u00a0_", stream.GetImage());
expect.expect(EOFException.class);
stream.readChar();
}
public static CharStream simpleCharStream(String abcd) throws IOException {
return NewCharStream.open(new JavaccTokenDocument(TextDocument.readOnlyString(abcd)));
}
public static CharStream javaCharStream(String abcd) throws IOException {
return NewCharStream.open(new JavaccTokenDocument(TextDocument.readOnlyString(abcd)) {
@Override
public EscapeAwareReader newReader(Chars text) {
return new JavaInputReader(text);
}
});
}
@Test
public void testBacktrackTooMuch() throws IOException {
CharStream stream = getCharStream("abcd");
CharStream stream = simpleCharStream("abcd");
assertEquals('a', stream.readChar());
assertEquals('b', stream.readChar());
@ -127,7 +168,7 @@ public class CharStreamImplTest {
@Test
public void testBacktrackTooMuch2() throws IOException {
CharStream stream = getCharStream("abcd");
CharStream stream = simpleCharStream("abcd");
assertEquals('a', stream.BeginToken());
assertEquals('b', stream.readChar());