From: Evan You Date: Wed, 15 Nov 2023 15:33:57 +0000 (+0800) Subject: wip: parse interpolation X-Git-Tag: v3.4.0-alpha.2~55 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5762288bdf036e8c56214ea4691811d9b505a9d9;p=thirdparty%2Fvuejs%2Fcore.git wip: parse interpolation --- diff --git a/packages/compiler-core/src/parser/Tokenizer.ts b/packages/compiler-core/src/parser/Tokenizer.ts index 3ec0d0c8ca..2262aef54e 100644 --- a/packages/compiler-core/src/parser/Tokenizer.ts +++ b/packages/compiler-core/src/parser/Tokenizer.ts @@ -65,9 +65,15 @@ export const enum CharCodes { RightSquare = 93 // "]" } +const defaultDelimitersOpen = [123, 123] // "{{" +const defaultDelimitersClose = [125, 125] // "}}" + /** All the states the tokenizer can be in. */ const enum State { Text = 1, + Interpolation, + + // Tags BeforeTagName, // After < InTagName, InSelfClosingTag, @@ -134,6 +140,8 @@ export interface Callbacks { ontext(start: number, endIndex: number): void ontextentity(codepoint: number, endIndex: number): void + oninterpolation(start: number, endIndex: number): void + onopentagname(start: number, endIndex: number): void onopentagend(endIndex: number): void onselfclosingtag(endIndex: number): void @@ -190,14 +198,9 @@ export default class Tokenizer { /** Reocrd newline positions for fast line / column calculation */ private newlines: number[] = [] - private readonly decodeEntities: boolean private readonly entityDecoder: EntityDecoder - constructor( - { decodeEntities = true }: { decodeEntities?: boolean }, - private readonly cbs: Callbacks - ) { - this.decodeEntities = decodeEntities + constructor(private readonly cbs: Callbacks) { this.entityDecoder = new EntityDecoder(htmlDecodeTree, (cp, consumed) => this.emitCodePoint(cp, consumed) ) @@ -211,6 +214,8 @@ export default class Tokenizer { this.baseState = State.Text this.currentSequence = undefined! this.newlines.length = 0 + this.delimiterOpen = defaultDelimitersOpen + this.delimiterClose = defaultDelimitersClose } /** @@ -238,17 +243,45 @@ export default class Tokenizer { } private stateText(c: number): void { - if ( - c === CharCodes.Lt || - (!this.decodeEntities && this.fastForwardTo(CharCodes.Lt)) - ) { + if (c === CharCodes.Lt) { if (this.index > this.sectionStart) { this.cbs.ontext(this.sectionStart, this.index) } this.state = State.BeforeTagName this.sectionStart = this.index - } else if (this.decodeEntities && c === CharCodes.Amp) { + } else if (c === CharCodes.Amp) { this.startEntity() + } else if (this.matchDelimiter(c, this.delimiterOpen)) { + if (this.index > this.sectionStart) { + this.cbs.ontext(this.sectionStart, this.index) + } + this.state = State.Interpolation + this.sectionStart = this.index + this.index += this.delimiterOpen.length + } + } + + public delimiterOpen: number[] = defaultDelimitersOpen + public delimiterClose: number[] = defaultDelimitersClose + private matchDelimiter(c: number, delimiter: number[]): boolean { + if (c === delimiter[0]) { + const l = delimiter.length + for (let i = 1; i < l; i++) { + if (this.buffer.charCodeAt(this.index + i) !== delimiter[i]) { + return false + } + } + return true + } + return false + } + + private stateInterpolation(c: number): void { + if (this.matchDelimiter(c, this.delimiterClose)) { + this.index += this.delimiterClose.length + this.cbs.oninterpolation(this.sectionStart, this.index) + this.state = State.Text + this.sectionStart = this.index } } @@ -302,7 +335,7 @@ export default class Tokenizer { } else if (this.sequenceIndex === 0) { if (this.currentSequence === Sequences.TitleEnd) { // We have to parse entities in tags. - if (this.decodeEntities && c === CharCodes.Amp) { + if (c === CharCodes.Amp) { this.startEntity() } } else if (this.fastForwardTo(CharCodes.Lt)) { @@ -592,7 +625,7 @@ export default class Tokenizer { } } private handleInAttributeValue(c: number, quote: number) { - if (c === quote || (!this.decodeEntities && this.fastForwardTo(quote))) { + if (c === quote) { this.cbs.onattribdata(this.sectionStart, this.index) this.sectionStart = -1 this.cbs.onattribend( @@ -600,7 +633,7 @@ export default class Tokenizer { this.index + 1 ) this.state = State.BeforeAttributeName - } else if (this.decodeEntities && c === CharCodes.Amp) { + } else if (c === CharCodes.Amp) { this.startEntity() } } @@ -617,7 +650,7 @@ export default class Tokenizer { this.cbs.onattribend(QuoteType.Unquoted, this.index) this.state = State.BeforeAttributeName this.stateBeforeAttributeName(c) - } else if (this.decodeEntities && c === CharCodes.Amp) { + } else if (c === CharCodes.Amp) { this.startEntity() } } @@ -715,6 +748,10 @@ export default class Tokenizer { this.stateText(c) break } + case State.Interpolation: { + this.stateInterpolation(c) + break + } case State.SpecialStartSequence: { this.stateSpecialStartSequence(c) break diff --git a/packages/compiler-core/src/parser/index.ts b/packages/compiler-core/src/parser/index.ts index 926ba94f3c..7ced0f4301 100644 --- a/packages/compiler-core/src/parser/index.ts +++ b/packages/compiler-core/src/parser/index.ts @@ -47,6 +47,7 @@ export const defaultParserOptions: MergedParserOptions = { isVoidTag: NO, isPreTag: NO, isCustomElement: NO, + // TODO handle entities decodeEntities: (rawText: string): string => rawText.replace(decodeRE, (_, p1) => decodeMap[p1]), onError: defaultOnError, @@ -69,162 +70,182 @@ let inPre = 0 // let inVPre = 0 const stack: ElementNode[] = [] -const tokenizer = new Tokenizer( - // TODO handle entities - { decodeEntities: true }, - { - ontext(start, end) { - onText(getSlice(start, end), start, end) - }, - - ontextentity(cp, end) { - onText(fromCodePoint(cp), end - 1, end) - }, - - onopentagname(start, end) { - emitOpenTag(getSlice(start, end), start) - }, - - onopentagend(end) { - endOpenTag(end) - }, - - onclosetag(start, end) { - const name = getSlice(start, end) - if (!currentOptions.isVoidTag(name)) { - const pos = stack.findIndex(e => e.tag === name) - if (pos !== -1) { - for (let index = 0; index <= pos; index++) { - onCloseTag(stack.shift()!, end) - } - } - } - }, - - onselfclosingtag(end) { - closeCurrentTag(end) - }, +const tokenizer = new Tokenizer({ + ontext(start, end) { + onText(getSlice(start, end), start, end) + }, - onattribname(start, end) { - // plain attribute - currentProp = { - type: NodeTypes.ATTRIBUTE, - name: getSlice(start, end), - value: undefined, - loc: getLoc(start) - } - }, - - ondirname(start, end) { - const raw = getSlice(start, end) - const name = - raw === '.' || raw === ':' - ? 'bind' - : raw === '@' - ? 'on' - : raw === '#' - ? 'slot' - : raw.slice(2) - currentProp = { - type: NodeTypes.DIRECTIVE, - name, - exp: undefined, - arg: undefined, - modifiers: [], - loc: getLoc(start) - } - }, + ontextentity(cp, end) { + onText(fromCodePoint(cp), end - 1, end) + }, - ondirarg(start, end) { - const arg = getSlice(start, end) - const isStatic = arg[0] !== `[` - ;(currentProp as DirectiveNode).arg = { + oninterpolation(start, end) { + let innerStart = start + tokenizer.delimiterOpen.length + let innerEnd = end - tokenizer.delimiterClose.length + while (isWhitespace(currentInput.charCodeAt(innerStart))) { + innerStart++ + } + while (isWhitespace(currentInput.charCodeAt(innerEnd - 1))) { + innerEnd-- + } + addNode({ + type: NodeTypes.INTERPOLATION, + content: { type: NodeTypes.SIMPLE_EXPRESSION, - content: arg, - isStatic, - constType: isStatic - ? ConstantTypes.CAN_STRINGIFY - : ConstantTypes.NOT_CONSTANT, - loc: getLoc(start, end) - } - }, - ondirmodifier(start, end) { - ;(currentProp as DirectiveNode).modifiers.push(getSlice(start, end)) - }, - - onattribdata(start, end) { - currentAttrValue += getSlice(start, end) - if (currentAttrStartIndex < 0) currentAttrStartIndex = start - currentAttrEndIndex = end - }, - - onattribentity(codepoint) { - currentAttrValue += fromCodePoint(codepoint) - }, - - onattribnameend(end) { - // check duplicate attrs - const start = currentProp!.loc.start.offset - const name = getSlice(start, end) - if (currentAttrs.has(name)) { - currentProp = null - // TODO emit error DUPLICATE_ATTRIBUTE - throw new Error(`duplicate attr ${name}`) - } else { - currentAttrs.add(name) + isStatic: false, + // Set `isConstant` to false by default and will decide in transformExpression + constType: ConstantTypes.NOT_CONSTANT, + content: getSlice(innerStart, innerEnd), + loc: getLoc(innerStart, innerEnd) + }, + loc: getLoc(start, end) + }) + }, + + onopentagname(start, end) { + emitOpenTag(getSlice(start, end), start) + }, + + onopentagend(end) { + endOpenTag(end) + }, + + onclosetag(start, end) { + const name = getSlice(start, end) + if (!currentOptions.isVoidTag(name)) { + const pos = stack.findIndex(e => e.tag === name) + if (pos !== -1) { + for (let index = 0; index <= pos; index++) { + onCloseTag(stack.shift()!, end) + } } - }, - - onattribend(quote, end) { - if (currentElement && currentProp) { - if (currentAttrValue) { - if (currentProp.type === NodeTypes.ATTRIBUTE) { - // assign value - currentProp!.value = { - type: NodeTypes.TEXT, - content: currentAttrValue, - loc: - quote === QuoteType.Unquoted - ? getLoc(currentAttrStartIndex, currentAttrEndIndex) - : getLoc(currentAttrStartIndex - 1, currentAttrEndIndex + 1) - } - } else { - // directive - currentProp.exp = { - type: NodeTypes.SIMPLE_EXPRESSION, - content: currentAttrValue, - isStatic: false, - // Treat as non-constant by default. This can be potentially set - // to other values by `transformExpression` to make it eligible - // for hoisting. - constType: ConstantTypes.NOT_CONSTANT, - loc: getLoc(currentAttrStartIndex, currentAttrEndIndex) - } + } + }, + + onselfclosingtag(end) { + closeCurrentTag(end) + }, + + onattribname(start, end) { + // plain attribute + currentProp = { + type: NodeTypes.ATTRIBUTE, + name: getSlice(start, end), + value: undefined, + loc: getLoc(start) + } + }, + + ondirname(start, end) { + const raw = getSlice(start, end) + const name = + raw === '.' || raw === ':' + ? 'bind' + : raw === '@' + ? 'on' + : raw === '#' + ? 'slot' + : raw.slice(2) + currentProp = { + type: NodeTypes.DIRECTIVE, + name, + exp: undefined, + arg: undefined, + modifiers: [], + loc: getLoc(start) + } + }, + + ondirarg(start, end) { + const arg = getSlice(start, end) + const isStatic = arg[0] !== `[` + ;(currentProp as DirectiveNode).arg = { + type: NodeTypes.SIMPLE_EXPRESSION, + content: arg, + isStatic, + constType: isStatic + ? ConstantTypes.CAN_STRINGIFY + : ConstantTypes.NOT_CONSTANT, + loc: getLoc(start, end) + } + }, + + ondirmodifier(start, end) { + ;(currentProp as DirectiveNode).modifiers.push(getSlice(start, end)) + }, + + onattribdata(start, end) { + currentAttrValue += getSlice(start, end) + if (currentAttrStartIndex < 0) currentAttrStartIndex = start + currentAttrEndIndex = end + }, + + onattribentity(codepoint) { + currentAttrValue += fromCodePoint(codepoint) + }, + + onattribnameend(end) { + // check duplicate attrs + const start = currentProp!.loc.start.offset + const name = getSlice(start, end) + if (currentAttrs.has(name)) { + currentProp = null + // TODO emit error DUPLICATE_ATTRIBUTE + throw new Error(`duplicate attr ${name}`) + } else { + currentAttrs.add(name) + } + }, + + onattribend(quote, end) { + if (currentElement && currentProp) { + if (currentAttrValue) { + if (currentProp.type === NodeTypes.ATTRIBUTE) { + // assign value + currentProp!.value = { + type: NodeTypes.TEXT, + content: currentAttrValue, + loc: + quote === QuoteType.Unquoted + ? getLoc(currentAttrStartIndex, currentAttrEndIndex) + : getLoc(currentAttrStartIndex - 1, currentAttrEndIndex + 1) + } + } else { + // directive + currentProp.exp = { + type: NodeTypes.SIMPLE_EXPRESSION, + content: currentAttrValue, + isStatic: false, + // Treat as non-constant by default. This can be potentially set + // to other values by `transformExpression` to make it eligible + // for hoisting. + constType: ConstantTypes.NOT_CONSTANT, + loc: getLoc(currentAttrStartIndex, currentAttrEndIndex) } } - currentProp.loc.end = tokenizer.getPos(end) - currentElement.props.push(currentProp!) - } - currentAttrValue = '' - currentAttrStartIndex = currentAttrEndIndex = -1 - }, - - oncomment(start, end, offset) { - // TODO oncomment - }, - - onend() { - const end = currentInput.length - 1 - for (let index = 0; index < stack.length; index++) { - onCloseTag(stack[index], end) } - }, - - oncdata(start, end, offset) { - // TODO throw error + currentProp.loc.end = tokenizer.getPos(end) + currentElement.props.push(currentProp!) } + currentAttrValue = '' + currentAttrStartIndex = currentAttrEndIndex = -1 + }, + + oncomment(start, end, offset) { + // TODO oncomment + }, + + onend() { + const end = currentInput.length - 1 + for (let index = 0; index < stack.length; index++) { + onCloseTag(stack[index], end) + } + }, + + oncdata(start, end, offset) { + // TODO throw error } -) +}) function getSlice(start: number, end: number) { return currentInput.slice(start, end) @@ -283,7 +304,7 @@ function onText(content: string, start: number, end: number) { loc: { start: tokenizer.getPos(start), end: tokenizer.getPos(end), - source: content + source: '' } }) } @@ -413,8 +434,17 @@ function reset() { stack.length = 0 } +function toCharCodes(str: string): number[] { + return str.split('').map(c => c.charCodeAt(0)) +} + export function baseParse(input: string, options?: ParserOptions): RootNode { reset() + const delimiters = options?.delimiters + if (delimiters) { + tokenizer.delimiterOpen = toCharCodes(delimiters[0]) + tokenizer.delimiterClose = toCharCodes(delimiters[1]) + } currentInput = input currentOptions = extend({}, defaultParserOptions, options) const root = (currentRoot = createRoot([]))