function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport): string {
let result = `<div class="monaco-tokenized-source">`;
let lines = text.split(/\r\n|\r|\n/);
let currentState = tokenizationSupport.getInitialState();
for (let i = 0, len = lines.length; i < len; i++) {
let line = lines[i];
if (i > 0) {
result += `<br/>`;
}
let tokenizationResult = tokenizationSupport.tokenize2(line, currentState, 0);
let lineTokens = new LineTokens(tokenizationResult.tokens, line);
let viewLineTokens = lineTokens.inflate();
let startOffset = 0;
for (let j = 0, lenJ = viewLineTokens.length; j < lenJ; j++) {
const viewLineToken = viewLineTokens[j];
result += `<span class="${viewLineToken.getType()}">${strings.escape(line.substring(startOffset, viewLineToken.endIndex))}</span>`;
startOffset = viewLineToken.endIndex;
}
currentState = tokenizationResult.endState;
}
result += `</div>`;
return result;
}
function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): string {
let html: string[] = [];
let state = tokenizationSupport.getInitialState();
for (let i = 0, length = lines.length; i < length; i++) {
let line = lines[i];
let tokenizeResult = tokenizationSupport.tokenize2(line, state, 0);
let lineTokens = new LineTokens(tokenizeResult.tokens, line);
let renderResult = renderViewLine(new RenderLineInput(
false,
line,
true/* check for RTL */,
0,
lineTokens.inflate(),
[],
tabSize,
0,
-1,
'none',
false,
false
));
html = html.concat(renderResult.html);
html.push('<br/>');
state = tokenizeResult.endState;
}
return html.join('');
}
function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): string {
let html: string[] = [];
let state = tokenizationSupport.getInitialState();
let colorMap = TokenizationRegistry.getColorMap();
for (let i = 0, length = lines.length; i < length; i++) {
let line = lines[i];
let tokenizeResult = tokenizationSupport.tokenize2(line, state, 0);
let lineTokens = new LineTokens(colorMap, tokenizeResult.tokens, line);
let renderResult = renderLine(new RenderLineInput(
line,
tabSize,
0,
-1,
'none',
false,
new LineParts(lineTokens.inflate(), line.length + 1)
));
html = html.concat(renderResult.output);
html.push('<br/>');
state = tokenizeResult.endState;
}
return html.join('');
}
function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): string {
let html: string[] = [];
let state = tokenizationSupport.getInitialState();
for (let i = 0, length = lines.length; i < length; i++) {
let line = lines[i];
let tokenizeResult = tokenizationSupport.tokenize2(line, state, 0);
LineTokens.convertToEndOffset(tokenizeResult.tokens, line.length);
let lineTokens = new LineTokens(tokenizeResult.tokens, line);
const isBasicASCII = ViewLineRenderingData.isBasicASCII(line, /* check for basic ASCII */true);
const containsRTL = ViewLineRenderingData.containsRTL(line, isBasicASCII, /* check for RTL */true);
let renderResult = renderViewLine(new RenderLineInput(
false,
line,
isBasicASCII,
containsRTL,
0,
lineTokens.inflate(),
[],
tabSize,
0,
-1,
'none',
false,
false
));
html = html.concat(renderResult.html);
html.push('<br/>');
state = tokenizeResult.endState;
}
return html.join('');
}
function assertLineTokens(_actual: LineTokens, _expected: TestToken[]): void {
let expected = ViewLineTokenFactory.inflateArr(TestToken.toTokens(_expected), _actual.getLineContent().length);
let actual = _actual.inflate();
let decode = (token: ViewLineToken) => {
return {
endIndex: token.endIndex,
type: token.getType()
};
};
assert.deepEqual(actual.map(decode), expected.map(decode));
}
_setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
let target: ModelLineTokens;
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
target = this._tokens[lineIndex];
} else {
target = new ModelLineTokens(null);
this._tokens[lineIndex] = target;
}
if (lineTextLength === 0) {
let hasDifferentLanguageId = false;
if (tokens && tokens.length > 1) {
hasDifferentLanguageId = (TokenMetadata.getLanguageId(tokens[1]) !== topLevelLanguageId);
}
if (!hasDifferentLanguageId) {
target._lineTokens = EMPTY_LINE_TOKENS;
return;
}
}
if (!tokens || tokens.length === 0) {
tokens = new Uint32Array(2);
tokens[0] = 0;
tokens[1] = getDefaultMetadata(topLevelLanguageId);
}
LineTokens.convertToEndOffset(tokens, lineTextLength);
target._lineTokens = tokens.buffer;
}
public static create(original:LineTokens, textLength:number): ViewLineTokens {
let inflatedTokens = original.inflate();
return new ViewLineTokens(
inflatedTokens,
0,
textLength
);
}
/**
* [startOffset; endOffset) (i.e. do not include endOffset)
*/
public static create(original:LineTokens, startOffset:number, endOffset:number, deltaStartIndex:number): ViewLineTokens {
let inflatedTokens = original.sliceAndInflate(startOffset, endOffset, deltaStartIndex);
return new ViewLineTokens(
inflatedTokens,
deltaStartIndex,
endOffset - startOffset + deltaStartIndex
);
}
public delete(fromChIndex: number, toChIndex: number): void {
if (this._lineTokens === null || this._lineTokens === EMPTY_LINE_TOKENS || fromChIndex === toChIndex) {
return;
}
const tokens = new Uint32Array(this._lineTokens);
const tokensCount = (tokens.length >>> 1);
// special case: deleting everything
if (fromChIndex === 0 && tokens[tokens.length - 2] === toChIndex) {
this._lineTokens = EMPTY_LINE_TOKENS;
return;
}
const fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, fromChIndex);
const fromTokenStartOffset = (fromTokenIndex > 0 ? tokens[(fromTokenIndex - 1) << 1] : 0);
const fromTokenEndOffset = tokens[fromTokenIndex << 1];
if (toChIndex < fromTokenEndOffset) {
// the delete range is inside a single token
const delta = (toChIndex - fromChIndex);
for (let i = fromTokenIndex; i < tokensCount; i++) {
tokens[i << 1] -= delta;
}
return;
}
let dest: number;
let lastEnd: number;
if (fromTokenStartOffset !== fromChIndex) {
tokens[fromTokenIndex << 1] = fromChIndex;
dest = ((fromTokenIndex + 1) << 1);
lastEnd = fromChIndex;
} else {
dest = (fromTokenIndex << 1);
lastEnd = fromTokenStartOffset;
}
const delta = (toChIndex - fromChIndex);
for (let tokenIndex = fromTokenIndex + 1; tokenIndex < tokensCount; tokenIndex++) {
const tokenEndOffset = tokens[tokenIndex << 1] - delta;
if (tokenEndOffset > lastEnd) {
tokens[dest++] = tokenEndOffset;
tokens[dest++] = tokens[(tokenIndex << 1) + 1];
lastEnd = tokenEndOffset;
}
}
if (dest === tokens.length) {
// nothing to trim
return;
}
let tmp = new Uint32Array(dest);
tmp.set(tokens.subarray(0, dest), 0);
this._lineTokens = tmp.buffer;
}
function assertLineTokens(actual: LineTokens, expected: TestToken[]): void {
let inflatedActual = actual.inflate();
assert.deepEqual(inflatedActual, expected.map((token) => {
return {
startIndex: token.startOffset,
type: 'mtk' + token.color
};
}), 'Line tokens are equal');
}
