var constants = require('../tokenizer/const');
|
var TYPE = constants.TYPE;
|
var NAME = constants.NAME;
|
|
var utils = require('../tokenizer/utils');
|
var cmpStr = utils.cmpStr;
|
|
var EOF = TYPE.EOF;
|
var WHITESPACE = TYPE.WhiteSpace;
|
var COMMENT = TYPE.Comment;
|
|
var OFFSET_MASK = 0x00FFFFFF;
|
var TYPE_SHIFT = 24;
|
|
var TokenStream = function() {
|
this.offsetAndType = null;
|
this.balance = null;
|
|
this.reset();
|
};
|
|
TokenStream.prototype = {
|
reset: function() {
|
this.eof = false;
|
this.tokenIndex = -1;
|
this.tokenType = 0;
|
this.tokenStart = this.firstCharOffset;
|
this.tokenEnd = this.firstCharOffset;
|
},
|
|
lookupType: function(offset) {
|
offset += this.tokenIndex;
|
|
if (offset < this.tokenCount) {
|
return this.offsetAndType[offset] >> TYPE_SHIFT;
|
}
|
|
return EOF;
|
},
|
lookupOffset: function(offset) {
|
offset += this.tokenIndex;
|
|
if (offset < this.tokenCount) {
|
return this.offsetAndType[offset - 1] & OFFSET_MASK;
|
}
|
|
return this.source.length;
|
},
|
lookupValue: function(offset, referenceStr) {
|
offset += this.tokenIndex;
|
|
if (offset < this.tokenCount) {
|
return cmpStr(
|
this.source,
|
this.offsetAndType[offset - 1] & OFFSET_MASK,
|
this.offsetAndType[offset] & OFFSET_MASK,
|
referenceStr
|
);
|
}
|
|
return false;
|
},
|
getTokenStart: function(tokenIndex) {
|
if (tokenIndex === this.tokenIndex) {
|
return this.tokenStart;
|
}
|
|
if (tokenIndex > 0) {
|
return tokenIndex < this.tokenCount
|
? this.offsetAndType[tokenIndex - 1] & OFFSET_MASK
|
: this.offsetAndType[this.tokenCount] & OFFSET_MASK;
|
}
|
|
return this.firstCharOffset;
|
},
|
|
// TODO: -> skipUntilBalanced
|
getRawLength: function(startToken, mode) {
|
var cursor = startToken;
|
var balanceEnd;
|
var offset = this.offsetAndType[Math.max(cursor - 1, 0)] & OFFSET_MASK;
|
var type;
|
|
loop:
|
for (; cursor < this.tokenCount; cursor++) {
|
balanceEnd = this.balance[cursor];
|
|
// stop scanning on balance edge that points to offset before start token
|
if (balanceEnd < startToken) {
|
break loop;
|
}
|
|
type = this.offsetAndType[cursor] >> TYPE_SHIFT;
|
|
// check token is stop type
|
switch (mode(type, this.source, offset)) {
|
case 1:
|
break loop;
|
|
case 2:
|
cursor++;
|
break loop;
|
|
default:
|
// fast forward to the end of balanced block
|
if (this.balance[balanceEnd] === cursor) {
|
cursor = balanceEnd;
|
}
|
|
offset = this.offsetAndType[cursor] & OFFSET_MASK;
|
}
|
}
|
|
return cursor - this.tokenIndex;
|
},
|
isBalanceEdge: function(pos) {
|
return this.balance[this.tokenIndex] < pos;
|
},
|
isDelim: function(code, offset) {
|
if (offset) {
|
return (
|
this.lookupType(offset) === TYPE.Delim &&
|
this.source.charCodeAt(this.lookupOffset(offset)) === code
|
);
|
}
|
|
return (
|
this.tokenType === TYPE.Delim &&
|
this.source.charCodeAt(this.tokenStart) === code
|
);
|
},
|
|
getTokenValue: function() {
|
return this.source.substring(this.tokenStart, this.tokenEnd);
|
},
|
getTokenLength: function() {
|
return this.tokenEnd - this.tokenStart;
|
},
|
substrToCursor: function(start) {
|
return this.source.substring(start, this.tokenStart);
|
},
|
|
skipWS: function() {
|
for (var i = this.tokenIndex, skipTokenCount = 0; i < this.tokenCount; i++, skipTokenCount++) {
|
if ((this.offsetAndType[i] >> TYPE_SHIFT) !== WHITESPACE) {
|
break;
|
}
|
}
|
|
if (skipTokenCount > 0) {
|
this.skip(skipTokenCount);
|
}
|
},
|
skipSC: function() {
|
while (this.tokenType === WHITESPACE || this.tokenType === COMMENT) {
|
this.next();
|
}
|
},
|
skip: function(tokenCount) {
|
var next = this.tokenIndex + tokenCount;
|
|
if (next < this.tokenCount) {
|
this.tokenIndex = next;
|
this.tokenStart = this.offsetAndType[next - 1] & OFFSET_MASK;
|
next = this.offsetAndType[next];
|
this.tokenType = next >> TYPE_SHIFT;
|
this.tokenEnd = next & OFFSET_MASK;
|
} else {
|
this.tokenIndex = this.tokenCount;
|
this.next();
|
}
|
},
|
next: function() {
|
var next = this.tokenIndex + 1;
|
|
if (next < this.tokenCount) {
|
this.tokenIndex = next;
|
this.tokenStart = this.tokenEnd;
|
next = this.offsetAndType[next];
|
this.tokenType = next >> TYPE_SHIFT;
|
this.tokenEnd = next & OFFSET_MASK;
|
} else {
|
this.tokenIndex = this.tokenCount;
|
this.eof = true;
|
this.tokenType = EOF;
|
this.tokenStart = this.tokenEnd = this.source.length;
|
}
|
},
|
|
forEachToken(fn) {
|
for (var i = 0, offset = this.firstCharOffset; i < this.tokenCount; i++) {
|
var start = offset;
|
var item = this.offsetAndType[i];
|
var end = item & OFFSET_MASK;
|
var type = item >> TYPE_SHIFT;
|
|
offset = end;
|
|
fn(type, start, end, i);
|
}
|
},
|
|
dump() {
|
var tokens = new Array(this.tokenCount);
|
|
this.forEachToken((type, start, end, index) => {
|
tokens[index] = {
|
idx: index,
|
type: NAME[type],
|
chunk: this.source.substring(start, end),
|
balance: this.balance[index]
|
};
|
});
|
|
return tokens;
|
}
|
};
|
|
module.exports = TokenStream;
|
