public _updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
if (!this.tokenizationSupport) {
this._invalidLineStartIndex = buffer.getLineCount();
return;
}
const linesLength = buffer.getLineCount();
const endLineIndex = lineNumber - 1;
// Validate all states up to and including endLineIndex
for (let lineIndex = this._invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) {
const endStateIndex = lineIndex + 1;
let r: TokenizationResult2 = null;
const text = buffer.getLineContent(lineIndex + 1);
try {
// Tokenize only the first X characters
let freshState = this._getState(lineIndex).clone();
r = this.tokenizationSupport.tokenize2(text, freshState, 0);
} catch (e) {
onUnexpectedError(e);
}
if (!r) {
r = nullTokenize2(this.languageIdentifier.id, text, this._getState(lineIndex), 0);
}
this._setTokens(this.languageIdentifier.id, lineIndex, text.length, r.tokens);
eventBuilder.registerChangedTokens(lineIndex + 1);
this._setIsInvalid(lineIndex, false);
if (endStateIndex < linesLength) {
if (this._getState(endStateIndex) !== null && r.endState.equals(this._getState(endStateIndex))) {
// The end state of this line remains the same
let nextInvalidLineIndex = lineIndex + 1;
while (nextInvalidLineIndex < linesLength) {
if (this._isInvalid(nextInvalidLineIndex)) {
break;
}
if (nextInvalidLineIndex + 1 < linesLength) {
if (this._getState(nextInvalidLineIndex + 1) === null) {
break;
}
} else {
if (this._lastState === null) {
break;
}
}
nextInvalidLineIndex++;
}
this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, nextInvalidLineIndex);
lineIndex = nextInvalidLineIndex - 1; // -1 because the outer loop increments it
} else {
this._setState(endStateIndex, r.endState);
}
} else {
this._lastState = r.endState;
}
}
this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, endLineIndex + 1);
}
public _tokenizeOneLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number {
if (!this.hasLinesToTokenize(buffer)) {
return buffer.getLineCount() + 1;
}
const lineNumber = this._invalidLineStartIndex + 1;
this._updateTokensUntilLine(buffer, eventBuilder, lineNumber);
return lineNumber;
}
// #region TextBuffer
public equals(other: ITextBuffer): boolean {
if (!(other instanceof PieceTreeTextBuffer)) {
return false;
}
if (this._BOM !== other._BOM) {
return false;
}
if (this.getEOL() !== other.getEOL()) {
return false;
}
return this._pieceTree.equal(other._pieceTree);
}
public hasLinesToTokenize(buffer: ITextBuffer): boolean {
return (this._invalidLineStartIndex < buffer.getLineCount());
}
export function guessIndentation(source: ITextBuffer, defaultTabSize: number, defaultInsertSpaces: boolean): IGuessedIndentation {
// Look at most at the first 10k lines
const linesCount = Math.min(source.getLineCount(), 10000);
let linesIndentedWithTabsCount = 0; // number of lines that contain at least one tab in indentation
let linesIndentedWithSpacesCount = 0; // number of lines that contain only spaces in indentation
let previousLineText = ''; // content of latest line that contained non-whitespace chars
let previousLineIndentation = 0; // index at which latest line contained the first non-whitespace char
const ALLOWED_TAB_SIZE_GUESSES = [2, 4, 6, 8, 3, 5, 7]; // prefer even guesses for `tabSize`, limit to [2, 8].
const MAX_ALLOWED_TAB_SIZE_GUESS = 8; // max(ALLOWED_TAB_SIZE_GUESSES) = 8
let spacesDiffCount = [0, 0, 0, 0, 0, 0, 0, 0, 0]; // `tabSize` scores
let tmp = new SpacesDiffResult();
for (let lineNumber = 1; lineNumber <= linesCount; lineNumber++) {
let currentLineLength = source.getLineLength(lineNumber);
let currentLineText = source.getLineContent(lineNumber);
// if the text buffer is chunk based, so long lines are cons-string, v8 will flattern the string when we check charCode.
// checking charCode on chunks directly is cheaper.
const useCurrentLineText = (currentLineLength <= 65536);
let currentLineHasContent = false; // does `currentLineText` contain non-whitespace chars
let currentLineIndentation = 0; // index at which `currentLineText` contains the first non-whitespace char
let currentLineSpacesCount = 0; // count of spaces found in `currentLineText` indentation
let currentLineTabsCount = 0; // count of tabs found in `currentLineText` indentation
for (let j = 0, lenJ = currentLineLength; j < lenJ; j++) {
let charCode = (useCurrentLineText ? currentLineText.charCodeAt(j) : source.getLineCharCode(lineNumber, j));
if (charCode === CharCode.Tab) {
currentLineTabsCount++;
} else if (charCode === CharCode.Space) {
currentLineSpacesCount++;
} else {
// Hit non whitespace character on this line
currentLineHasContent = true;
currentLineIndentation = j;
break;
}
}
// Ignore empty or only whitespace lines
if (!currentLineHasContent) {
continue;
}
if (currentLineTabsCount > 0) {
linesIndentedWithTabsCount++;
} else if (currentLineSpacesCount > 1) {
linesIndentedWithSpacesCount++;
}
spacesDiff(previousLineText, previousLineIndentation, currentLineText, currentLineIndentation, tmp);
if (tmp.looksLikeAlignment) {
// skip this line entirely
continue;
}
let currentSpacesDiff = tmp.spacesDiff;
if (currentSpacesDiff <= MAX_ALLOWED_TAB_SIZE_GUESS) {
spacesDiffCount[currentSpacesDiff]++;
}
previousLineText = currentLineText;
previousLineIndentation = currentLineIndentation;
}
let insertSpaces = defaultInsertSpaces;
if (linesIndentedWithTabsCount !== linesIndentedWithSpacesCount) {
insertSpaces = (linesIndentedWithTabsCount < linesIndentedWithSpacesCount);
}
let tabSize = defaultTabSize;
let tabSizeScore = (insertSpaces ? 0 : 0.1 * linesCount);
// console.log("score threshold: " + tabSizeScore);
ALLOWED_TAB_SIZE_GUESSES.forEach((possibleTabSize) => {
let possibleTabSizeScore = spacesDiffCount[possibleTabSize];
if (possibleTabSizeScore > tabSizeScore) {
tabSizeScore = possibleTabSizeScore;
tabSize = possibleTabSize;
}
});
// Let a tabSize of 2 win even if it is not the maximum
// (only in case 4 was guessed)
if (tabSize === 4 && spacesDiffCount[4] > 0 && spacesDiffCount[2] > 0 && spacesDiffCount[2] >= spacesDiffCount[4] / 2) {
tabSize = 2;
}
// console.log('--------------------------');
// console.log('linesIndentedWithTabsCount: ' + linesIndentedWithTabsCount + ', linesIndentedWithSpacesCount: ' + linesIndentedWithSpacesCount);
// console.log('spacesDiffCount: ' + spacesDiffCount);
// console.log('tabSize: ' + tabSize + ', tabSizeScore: ' + tabSizeScore);
return {
insertSpaces: insertSpaces,
tabSize: tabSize
};
}
export function guessIndentation(source: ITextBuffer, defaultTabSize: number, defaultInsertSpaces: boolean): IGuessedIndentation {
// Look at most at the first 10k lines
const linesCount = Math.min(source.getLineCount(), 10000);
let linesIndentedWithTabsCount = 0; // number of lines that contain at least one tab in indentation
let linesIndentedWithSpacesCount = 0; // number of lines that contain only spaces in indentation
let previousLineText = ''; // content of latest line that contained non-whitespace chars
let previousLineIndentation = 0; // index at which latest line contained the first non-whitespace char
const ALLOWED_TAB_SIZE_GUESSES = [2, 4, 6, 8]; // limit guesses for `tabSize` to 2, 4, 6 or 8.
const MAX_ALLOWED_TAB_SIZE_GUESS = 8; // max(2,4,6,8) = 8
let spacesDiffCount = [0, 0, 0, 0, 0, 0, 0, 0, 0]; // `tabSize` scores
for (let lineNumber = 1; lineNumber <= linesCount; lineNumber++) {
let currentLineLength = source.getLineLength(lineNumber);
let currentLineText = source.getLineContent(lineNumber);
let charCodeAt: (offset: number) => number;
if (currentLineLength > 65536) {
// if the text buffer is chunk based, so long lines are cons-string, v8 will flattern the string when we check charCode.
// checking charCode on chunks directly is cheaper.
charCodeAt = (offset: number) => source.getLineCharCode(lineNumber, offset);
} else {
charCodeAt = (offset: number) => currentLineText.charCodeAt(offset);
}
let currentLineHasContent = false; // does `currentLineText` contain non-whitespace chars
let currentLineIndentation = 0; // index at which `currentLineText` contains the first non-whitespace char
let currentLineSpacesCount = 0; // count of spaces found in `currentLineText` indentation
let currentLineTabsCount = 0; // count of tabs found in `currentLineText` indentation
for (let j = 0, lenJ = currentLineLength; j < lenJ; j++) {
let charCode = charCodeAt(j);
if (charCode === CharCode.Tab) {
currentLineTabsCount++;
} else if (charCode === CharCode.Space) {
currentLineSpacesCount++;
} else {
// Hit non whitespace character on this line
currentLineHasContent = true;
currentLineIndentation = j;
break;
}
}
// Ignore empty or only whitespace lines
if (!currentLineHasContent) {
continue;
}
if (currentLineTabsCount > 0) {
linesIndentedWithTabsCount++;
} else if (currentLineSpacesCount > 1) {
linesIndentedWithSpacesCount++;
}
let currentSpacesDiff = spacesDiff(previousLineText, previousLineIndentation, currentLineText, currentLineIndentation);
if (currentSpacesDiff <= MAX_ALLOWED_TAB_SIZE_GUESS) {
spacesDiffCount[currentSpacesDiff]++;
}
previousLineText = currentLineText;
previousLineIndentation = currentLineIndentation;
}
// Take into account the last line as well
let deltaSpacesCount = spacesDiff(previousLineText, previousLineIndentation, '', 0);
if (deltaSpacesCount <= MAX_ALLOWED_TAB_SIZE_GUESS) {
spacesDiffCount[deltaSpacesCount]++;
}
let insertSpaces = defaultInsertSpaces;
if (linesIndentedWithTabsCount !== linesIndentedWithSpacesCount) {
insertSpaces = (linesIndentedWithTabsCount < linesIndentedWithSpacesCount);
}
let tabSize = defaultTabSize;
let tabSizeScore = (insertSpaces ? 0 : 0.1 * linesCount);
// console.log("score threshold: " + tabSizeScore);
ALLOWED_TAB_SIZE_GUESSES.forEach((possibleTabSize) => {
let possibleTabSizeScore = spacesDiffCount[possibleTabSize];
if (possibleTabSizeScore > tabSizeScore) {
tabSizeScore = possibleTabSizeScore;
tabSize = possibleTabSize;
}
});
// console.log('--------------------------');
// console.log('linesIndentedWithTabsCount: ' + linesIndentedWithTabsCount + ', linesIndentedWithSpacesCount: ' + linesIndentedWithSpacesCount);
// console.log('spacesDiffCount: ' + spacesDiffCount);
// console.log('tabSize: ' + tabSize + ', tabSizeScore: ' + tabSizeScore);
return {
insertSpaces: insertSpaces,
tabSize: tabSize
};
}
charCodeAt = (offset: number) => source.getLineCharCode(lineNumber, offset);