RightSquare = 93 // "]"
}
+const defaultDelimitersOpen = [123, 123] // "{{"
+const defaultDelimitersClose = [125, 125] // "}}"
+
/** All the states the tokenizer can be in. */
const enum State {
Text = 1,
+ Interpolation,
+
+ // Tags
BeforeTagName, // After <
InTagName,
InSelfClosingTag,
ontext(start: number, endIndex: number): void
ontextentity(codepoint: number, endIndex: number): void
+ oninterpolation(start: number, endIndex: number): void
+
onopentagname(start: number, endIndex: number): void
onopentagend(endIndex: number): void
onselfclosingtag(endIndex: number): void
/** Reocrd newline positions for fast line / column calculation */
private newlines: number[] = []
- private readonly decodeEntities: boolean
private readonly entityDecoder: EntityDecoder
- constructor(
- { decodeEntities = true }: { decodeEntities?: boolean },
- private readonly cbs: Callbacks
- ) {
- this.decodeEntities = decodeEntities
+ constructor(private readonly cbs: Callbacks) {
this.entityDecoder = new EntityDecoder(htmlDecodeTree, (cp, consumed) =>
this.emitCodePoint(cp, consumed)
)
this.baseState = State.Text
this.currentSequence = undefined!
this.newlines.length = 0
+ this.delimiterOpen = defaultDelimitersOpen
+ this.delimiterClose = defaultDelimitersClose
}
/**
}
private stateText(c: number): void {
- if (
- c === CharCodes.Lt ||
- (!this.decodeEntities && this.fastForwardTo(CharCodes.Lt))
- ) {
+ if (c === CharCodes.Lt) {
if (this.index > this.sectionStart) {
this.cbs.ontext(this.sectionStart, this.index)
}
this.state = State.BeforeTagName
this.sectionStart = this.index
- } else if (this.decodeEntities && c === CharCodes.Amp) {
+ } else if (c === CharCodes.Amp) {
this.startEntity()
+ } else if (this.matchDelimiter(c, this.delimiterOpen)) {
+ if (this.index > this.sectionStart) {
+ this.cbs.ontext(this.sectionStart, this.index)
+ }
+ this.state = State.Interpolation
+ this.sectionStart = this.index
+ this.index += this.delimiterOpen.length
+ }
+ }
+
+ public delimiterOpen: number[] = defaultDelimitersOpen
+ public delimiterClose: number[] = defaultDelimitersClose
+ private matchDelimiter(c: number, delimiter: number[]): boolean {
+ if (c === delimiter[0]) {
+ const l = delimiter.length
+ for (let i = 1; i < l; i++) {
+ if (this.buffer.charCodeAt(this.index + i) !== delimiter[i]) {
+ return false
+ }
+ }
+ return true
+ }
+ return false
+ }
+
+ private stateInterpolation(c: number): void {
+ if (this.matchDelimiter(c, this.delimiterClose)) {
+ this.index += this.delimiterClose.length
+ this.cbs.oninterpolation(this.sectionStart, this.index)
+ this.state = State.Text
+ this.sectionStart = this.index
}
}
} else if (this.sequenceIndex === 0) {
if (this.currentSequence === Sequences.TitleEnd) {
// We have to parse entities in <title> tags.
- if (this.decodeEntities && c === CharCodes.Amp) {
+ if (c === CharCodes.Amp) {
this.startEntity()
}
} else if (this.fastForwardTo(CharCodes.Lt)) {
}
}
private handleInAttributeValue(c: number, quote: number) {
- if (c === quote || (!this.decodeEntities && this.fastForwardTo(quote))) {
+ if (c === quote) {
this.cbs.onattribdata(this.sectionStart, this.index)
this.sectionStart = -1
this.cbs.onattribend(
this.index + 1
)
this.state = State.BeforeAttributeName
- } else if (this.decodeEntities && c === CharCodes.Amp) {
+ } else if (c === CharCodes.Amp) {
this.startEntity()
}
}
this.cbs.onattribend(QuoteType.Unquoted, this.index)
this.state = State.BeforeAttributeName
this.stateBeforeAttributeName(c)
- } else if (this.decodeEntities && c === CharCodes.Amp) {
+ } else if (c === CharCodes.Amp) {
this.startEntity()
}
}
this.stateText(c)
break
}
+ case State.Interpolation: {
+ this.stateInterpolation(c)
+ break
+ }
case State.SpecialStartSequence: {
this.stateSpecialStartSequence(c)
break
isVoidTag: NO,
isPreTag: NO,
isCustomElement: NO,
+ // TODO handle entities
decodeEntities: (rawText: string): string =>
rawText.replace(decodeRE, (_, p1) => decodeMap[p1]),
onError: defaultOnError,
// let inVPre = 0
const stack: ElementNode[] = []
-const tokenizer = new Tokenizer(
- // TODO handle entities
- { decodeEntities: true },
- {
- ontext(start, end) {
- onText(getSlice(start, end), start, end)
- },
-
- ontextentity(cp, end) {
- onText(fromCodePoint(cp), end - 1, end)
- },
-
- onopentagname(start, end) {
- emitOpenTag(getSlice(start, end), start)
- },
-
- onopentagend(end) {
- endOpenTag(end)
- },
-
- onclosetag(start, end) {
- const name = getSlice(start, end)
- if (!currentOptions.isVoidTag(name)) {
- const pos = stack.findIndex(e => e.tag === name)
- if (pos !== -1) {
- for (let index = 0; index <= pos; index++) {
- onCloseTag(stack.shift()!, end)
- }
- }
- }
- },
-
- onselfclosingtag(end) {
- closeCurrentTag(end)
- },
+const tokenizer = new Tokenizer({
+ ontext(start, end) {
+ onText(getSlice(start, end), start, end)
+ },
- onattribname(start, end) {
- // plain attribute
- currentProp = {
- type: NodeTypes.ATTRIBUTE,
- name: getSlice(start, end),
- value: undefined,
- loc: getLoc(start)
- }
- },
-
- ondirname(start, end) {
- const raw = getSlice(start, end)
- const name =
- raw === '.' || raw === ':'
- ? 'bind'
- : raw === '@'
- ? 'on'
- : raw === '#'
- ? 'slot'
- : raw.slice(2)
- currentProp = {
- type: NodeTypes.DIRECTIVE,
- name,
- exp: undefined,
- arg: undefined,
- modifiers: [],
- loc: getLoc(start)
- }
- },
+ ontextentity(cp, end) {
+ onText(fromCodePoint(cp), end - 1, end)
+ },
- ondirarg(start, end) {
- const arg = getSlice(start, end)
- const isStatic = arg[0] !== `[`
- ;(currentProp as DirectiveNode).arg = {
+ oninterpolation(start, end) {
+ let innerStart = start + tokenizer.delimiterOpen.length
+ let innerEnd = end - tokenizer.delimiterClose.length
+ while (isWhitespace(currentInput.charCodeAt(innerStart))) {
+ innerStart++
+ }
+ while (isWhitespace(currentInput.charCodeAt(innerEnd - 1))) {
+ innerEnd--
+ }
+ addNode({
+ type: NodeTypes.INTERPOLATION,
+ content: {
type: NodeTypes.SIMPLE_EXPRESSION,
- content: arg,
- isStatic,
- constType: isStatic
- ? ConstantTypes.CAN_STRINGIFY
- : ConstantTypes.NOT_CONSTANT,
- loc: getLoc(start, end)
- }
- },
- ondirmodifier(start, end) {
- ;(currentProp as DirectiveNode).modifiers.push(getSlice(start, end))
- },
-
- onattribdata(start, end) {
- currentAttrValue += getSlice(start, end)
- if (currentAttrStartIndex < 0) currentAttrStartIndex = start
- currentAttrEndIndex = end
- },
-
- onattribentity(codepoint) {
- currentAttrValue += fromCodePoint(codepoint)
- },
-
- onattribnameend(end) {
- // check duplicate attrs
- const start = currentProp!.loc.start.offset
- const name = getSlice(start, end)
- if (currentAttrs.has(name)) {
- currentProp = null
- // TODO emit error DUPLICATE_ATTRIBUTE
- throw new Error(`duplicate attr ${name}`)
- } else {
- currentAttrs.add(name)
+ isStatic: false,
+ // Set `isConstant` to false by default and will decide in transformExpression
+ constType: ConstantTypes.NOT_CONSTANT,
+ content: getSlice(innerStart, innerEnd),
+ loc: getLoc(innerStart, innerEnd)
+ },
+ loc: getLoc(start, end)
+ })
+ },
+
+ onopentagname(start, end) {
+ emitOpenTag(getSlice(start, end), start)
+ },
+
+ onopentagend(end) {
+ endOpenTag(end)
+ },
+
+ onclosetag(start, end) {
+ const name = getSlice(start, end)
+ if (!currentOptions.isVoidTag(name)) {
+ const pos = stack.findIndex(e => e.tag === name)
+ if (pos !== -1) {
+ for (let index = 0; index <= pos; index++) {
+ onCloseTag(stack.shift()!, end)
+ }
}
- },
-
- onattribend(quote, end) {
- if (currentElement && currentProp) {
- if (currentAttrValue) {
- if (currentProp.type === NodeTypes.ATTRIBUTE) {
- // assign value
- currentProp!.value = {
- type: NodeTypes.TEXT,
- content: currentAttrValue,
- loc:
- quote === QuoteType.Unquoted
- ? getLoc(currentAttrStartIndex, currentAttrEndIndex)
- : getLoc(currentAttrStartIndex - 1, currentAttrEndIndex + 1)
- }
- } else {
- // directive
- currentProp.exp = {
- type: NodeTypes.SIMPLE_EXPRESSION,
- content: currentAttrValue,
- isStatic: false,
- // Treat as non-constant by default. This can be potentially set
- // to other values by `transformExpression` to make it eligible
- // for hoisting.
- constType: ConstantTypes.NOT_CONSTANT,
- loc: getLoc(currentAttrStartIndex, currentAttrEndIndex)
- }
+ }
+ },
+
+ onselfclosingtag(end) {
+ closeCurrentTag(end)
+ },
+
+ onattribname(start, end) {
+ // plain attribute
+ currentProp = {
+ type: NodeTypes.ATTRIBUTE,
+ name: getSlice(start, end),
+ value: undefined,
+ loc: getLoc(start)
+ }
+ },
+
+ ondirname(start, end) {
+ const raw = getSlice(start, end)
+ const name =
+ raw === '.' || raw === ':'
+ ? 'bind'
+ : raw === '@'
+ ? 'on'
+ : raw === '#'
+ ? 'slot'
+ : raw.slice(2)
+ currentProp = {
+ type: NodeTypes.DIRECTIVE,
+ name,
+ exp: undefined,
+ arg: undefined,
+ modifiers: [],
+ loc: getLoc(start)
+ }
+ },
+
+ ondirarg(start, end) {
+ const arg = getSlice(start, end)
+ const isStatic = arg[0] !== `[`
+ ;(currentProp as DirectiveNode).arg = {
+ type: NodeTypes.SIMPLE_EXPRESSION,
+ content: arg,
+ isStatic,
+ constType: isStatic
+ ? ConstantTypes.CAN_STRINGIFY
+ : ConstantTypes.NOT_CONSTANT,
+ loc: getLoc(start, end)
+ }
+ },
+
+ ondirmodifier(start, end) {
+ ;(currentProp as DirectiveNode).modifiers.push(getSlice(start, end))
+ },
+
+ onattribdata(start, end) {
+ currentAttrValue += getSlice(start, end)
+ if (currentAttrStartIndex < 0) currentAttrStartIndex = start
+ currentAttrEndIndex = end
+ },
+
+ onattribentity(codepoint) {
+ currentAttrValue += fromCodePoint(codepoint)
+ },
+
+ onattribnameend(end) {
+ // check duplicate attrs
+ const start = currentProp!.loc.start.offset
+ const name = getSlice(start, end)
+ if (currentAttrs.has(name)) {
+ currentProp = null
+ // TODO emit error DUPLICATE_ATTRIBUTE
+ throw new Error(`duplicate attr ${name}`)
+ } else {
+ currentAttrs.add(name)
+ }
+ },
+
+ onattribend(quote, end) {
+ if (currentElement && currentProp) {
+ if (currentAttrValue) {
+ if (currentProp.type === NodeTypes.ATTRIBUTE) {
+ // assign value
+ currentProp!.value = {
+ type: NodeTypes.TEXT,
+ content: currentAttrValue,
+ loc:
+ quote === QuoteType.Unquoted
+ ? getLoc(currentAttrStartIndex, currentAttrEndIndex)
+ : getLoc(currentAttrStartIndex - 1, currentAttrEndIndex + 1)
+ }
+ } else {
+ // directive
+ currentProp.exp = {
+ type: NodeTypes.SIMPLE_EXPRESSION,
+ content: currentAttrValue,
+ isStatic: false,
+ // Treat as non-constant by default. This can be potentially set
+ // to other values by `transformExpression` to make it eligible
+ // for hoisting.
+ constType: ConstantTypes.NOT_CONSTANT,
+ loc: getLoc(currentAttrStartIndex, currentAttrEndIndex)
}
}
- currentProp.loc.end = tokenizer.getPos(end)
- currentElement.props.push(currentProp!)
- }
- currentAttrValue = ''
- currentAttrStartIndex = currentAttrEndIndex = -1
- },
-
- oncomment(start, end, offset) {
- // TODO oncomment
- },
-
- onend() {
- const end = currentInput.length - 1
- for (let index = 0; index < stack.length; index++) {
- onCloseTag(stack[index], end)
}
- },
-
- oncdata(start, end, offset) {
- // TODO throw error
+ currentProp.loc.end = tokenizer.getPos(end)
+ currentElement.props.push(currentProp!)
}
+ currentAttrValue = ''
+ currentAttrStartIndex = currentAttrEndIndex = -1
+ },
+
+ oncomment(start, end, offset) {
+ // TODO oncomment
+ },
+
+ onend() {
+ const end = currentInput.length - 1
+ for (let index = 0; index < stack.length; index++) {
+ onCloseTag(stack[index], end)
+ }
+ },
+
+ oncdata(start, end, offset) {
+ // TODO throw error
}
-)
+})
function getSlice(start: number, end: number) {
return currentInput.slice(start, end)
loc: {
start: tokenizer.getPos(start),
end: tokenizer.getPos(end),
- source: content
+ source: ''
}
})
}
stack.length = 0
}
+function toCharCodes(str: string): number[] {
+ return str.split('').map(c => c.charCodeAt(0))
+}
+
export function baseParse(input: string, options?: ParserOptions): RootNode {
reset()
+ const delimiters = options?.delimiters
+ if (delimiters) {
+ tokenizer.delimiterOpen = toCharCodes(delimiters[0])
+ tokenizer.delimiterClose = toCharCodes(delimiters[1])
+ }
currentInput = input
currentOptions = extend({}, defaultParserOptions, options)
const root = (currentRoot = createRoot([]))
projects / thirdparty / vuejs / core.git / commitdiff
