refactor: extract inner text parsing and add syntax highlighting

playground/src/SyntaxStyler/DocNodeSyntaxStyler.ts

@@ -155,8 +155,10 @@ export class DocNodeSyntaxStyler {
  break;
  }
 
+ case 'XmlElement_Name':
  case 'XmlEndTag_Name':
  case 'XmlStartTag_Name': {
+ console.log('test');
  DocNodeSyntaxStyler._addTokenStyles(styles, docNode.content, {
  theme,
  styleTokens: [...styleTokens, 'element', 'name']

tsdoc-config/src/__tests__/assets/p10/base1/tsdoc-base1.json

@@ -1,5 +1,5 @@
 {
  "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
  "supportedXmlElements": ["span", "p"],
- "reportUnsupportedHtmlElements": false
+ "reportUnsupportedXmlElements": false
 }

tsdoc-config/src/__tests__/assets/p10/tsdoc.json

@@ -1,5 +1,5 @@
 {
  "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
  "extends": ["./base1/tsdoc-base1.json"],
- "reportUnsupportedHtmlElements": true
+ "reportUnsupportedXmlElements": true
 }

tsdoc-config/src/__tests__/assets/p11/base1/tsdoc-base1.json

@@ -1,4 +1,4 @@
 {
  "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
- "reportUnsupportedHtmlElements": false
+ "reportUnsupportedXmlElements": false
 }

tsdoc-config/src/__tests__/assets/p11/tsdoc.json

@@ -1,5 +1,5 @@
 {
  "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
  "extends": ["./base1/tsdoc-base1.json"],
- "reportUnsupportedHtmlElements": true
+ "reportUnsupportedXmlElements": true
 }

tsdoc-config/src/__tests__/assets/p12/tsdoc.json

@@ -1,5 +1,5 @@
 {
  "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
  "supportedXmlElements": [],
- "reportUnsupportedHtmlElements": false
+ "reportUnsupportedXmlElements": false
 }

tsdoc-config/src/__tests__/assets/p9/base1/tsdoc-base1.json

@@ -1,5 +1,5 @@
 {
  "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
  "supportedXmlElements": ["span", "p"],
- "reportUnsupportedHtmlElements": true
+ "reportUnsupportedXmlElements": true
 }

tsdoc-config/src/__tests__/assets/p9/tsdoc.json

@@ -1,5 +1,5 @@
 {
  "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
  "extends": ["./base1/tsdoc-base1.json"],
- "reportUnsupportedHtmlElements": false
+ "reportUnsupportedXmlElements": false
 }

tsdoc/etc/tsdoc.api.md

@@ -416,6 +416,8 @@ export class DocXmlElement extends DocNodeContainer {
  // (undocumented)
  get kind(): string;
  get name(): string;
+ // @override (undocumented)
+ protected onGetChildNodes(): ReadonlyArray<DocNode | undefined>;
  get selfClosingTag(): boolean;
  get spacingAfterEndTag(): string | undefined;
  get spacingAfterName(): string | undefined;
@@ -509,6 +511,8 @@ export enum ExcerptKind {
  // (undocumented)
  XmlAttribute_Value = "XmlAttribute_Value",
  // (undocumented)
+ XmlElement_Name = "XmlElement_Name",
+ // (undocumented)
  XmlEndTag_ClosingDelimiter = "XmlEndTag_ClosingDelimiter",
  // (undocumented)
  XmlEndTag_Name = "XmlEndTag_Name",
@@ -962,6 +966,8 @@ export interface IDocXmlElementParsedParameters extends IDocNodeParsedParameters
  // (undocumented)
  endTagClosingExcerpt?: TokenSequence;
  // (undocumented)
+ endTagNameExcerpt?: TokenSequence;
+ // (undocumented)
  endTagOpeningDelimiterExcerpt?: TokenSequence;
  // (undocumented)
  endTagOpeningExcerpt?: TokenSequence;
@@ -982,6 +988,8 @@ export interface IDocXmlElementParsedParameters extends IDocNodeParsedParameters
  // (undocumented)
  startTagClosingDelimiterExcerpt: TokenSequence;
  // (undocumented)
+ startTagNameExcerpt?: TokenSequence;
+ // (undocumented)
  startTagOpeningDelimiterExcerpt: TokenSequence;
  // (undocumented)
  xmlAttributes: DocXmlAttribute[];
@@ -1335,6 +1343,7 @@ export enum TSDocMessageId {
  UnnecessaryBackslash = "tsdoc-unnecessary-backslash",
  UnsupportedTag = "tsdoc-unsupported-tag",
  UnsupportedXmlElementName = "tsdoc-unsupported-xml-name",
+ UnterminatedXmlElement = "tsdoc-unterminated-xml-element",
  XmlStringMissingQuote = "tsdoc-xml-string-missing-quote",
  XmlTagMissingEquals = "tsdoc-xml-tag-missing-equals",
  XmlTagMissingGreaterThan = "tsdoc-xml-tag-missing-greater-than",

tsdoc/src/nodes/DocExcerpt.ts

@@ -44,6 +44,8 @@ export enum ExcerptKind {
  XmlAttribute_Equals = 'XmlAttribute_Equals',
  XmlAttribute_Value = 'XmlAttribute_Value',
 
+ XmlElement_Name = 'XmlElement_Name',
+
  XmlEndTag_OpeningDelimiter = 'XmlEndTag_OpeningDelimiter',
  XmlEndTag_Name = 'XmlEndTag_Name',
  XmlEndTag_ClosingDelimiter = 'XmlEndTag_ClosingDelimiter',

tsdoc/src/nodes/DocXMLElement.ts

@@ -29,6 +29,9 @@ export interface IDocXmlElementParsedParameters extends IDocNodeParsedParameters
  nameExcerpt: TokenSequence;
  spacingAfterNameExcerpt?: TokenSequence;
 
+ startTagNameExcerpt?: TokenSequence;
+ endTagNameExcerpt?: TokenSequence;
+
  xmlAttributes: DocXmlAttribute[];
  selfClosingTag?: boolean;
 
@@ -54,6 +57,9 @@ export class DocXmlElement extends DocNodeContainer {
  private _name: string | undefined;
  private readonly _nameExcerpt: DocExcerpt | undefined;
 
+ private readonly _startTagNameExcerpt: DocExcerpt | undefined;
+ private readonly _endTagNameExcerpt: DocExcerpt | undefined;
+
  private _startTagOpeningDelimiter: string | undefined;
  private readonly _startTagOpeningDelimiterExcerpt: DocExcerpt | undefined;
 
@@ -88,9 +94,23 @@ export class DocXmlElement extends DocNodeContainer {
  this.appendNodes(parameters.childNodes);
  this._nameExcerpt = new DocExcerpt({
  configuration: this.configuration,
- excerptKind: ExcerptKind.XmlStartTag_Name,
+ excerptKind: ExcerptKind.XmlElement_Name,
  content: parameters.nameExcerpt
  });
+ if (parameters.startTagNameExcerpt) {
+ this._startTagNameExcerpt = new DocExcerpt({
+ configuration: this.configuration,
+ excerptKind: ExcerptKind.XmlStartTag_Name,
+ content: parameters.startTagNameExcerpt
+ });
+ }
+ if (parameters.endTagNameExcerpt) {
+ this._endTagNameExcerpt = new DocExcerpt({
+ configuration: this.configuration,
+ excerptKind: ExcerptKind.XmlEndTag_Name,
+ content: parameters.endTagNameExcerpt
+ });
+ }
  if (parameters.spacingAfterNameExcerpt) {
  this._spacingAfterNameExcerpt = new DocExcerpt({
  configuration: this.configuration,
@@ -274,10 +294,25 @@ export class DocXmlElement extends DocNodeContainer {
  }
 
  public emitAsXml(): string {
- // NOTE: Here we're assuming that the TSDoc representation for a tag is also a valid HTML expression.
+ // NOTE: Here we're assuming that the TSDoc representation for a tag is also a valid XML expression.
  const stringBuilder: StringBuilder = new StringBuilder();
  const emitter: TSDocEmitter = new TSDocEmitter();
  emitter.renderXmlElement(stringBuilder, this);
  return stringBuilder.toString();
  }
+
+ /** @override */
+ protected onGetChildNodes(): ReadonlyArray<DocNode | undefined> {
+ return [
+ this._startTagOpeningDelimiterExcerpt,
+ this._nameExcerpt,
+ this._spacingAfterNameExcerpt,
+ this._startTagClosingDelimiterExcerpt,
+ ...this._xmlAttributes,
+ ...this.nodes,
+ this._endTagOpeningDelimiterExcerpt,
+ this._endTagNameExcerpt,
+ this._endTagClosingDelimiterExcerpt
+ ];
+ }
 }

tsdoc/src/nodes/index.ts

@@ -12,6 +12,7 @@ export * from './DocInheritDocTag';
 export * from './DocInlineTag';
 export * from './DocInlineTagBase';
 export * from './DocLinkTag';
+export * from './DocXmlElement';
 export * from './DocMemberIdentifier';
 export * from './DocMemberReference';
 export * from './DocMemberSelector';
@@ -24,4 +25,3 @@ export * from './DocParamCollection';
 export * from './DocPlainText';
 export * from './DocSection';
 export * from './DocSoftBreak';
-export * from './DocXmlElement';

tsdoc/src/parser/NodeParser.ts

@@ -1905,6 +1905,7 @@ export class NodeParser {
  startTagOpeningDelimiterExcerpt,
  startTagClosingDelimiterExcerpt,
  spacingAfterElementExcerpt: this._tryReadSpacingAndNewlines(tokenReader),
+ startTagNameExcerpt,
  xmlAttributes,
  selfClosingTag,
  childNodes: []
@@ -1943,41 +1944,19 @@ export class NodeParser {
 
  // We've hit an XML text node.
 
- while (tokenReader.peekTokenKind() !== TokenKind.LessThan) {
- // Technically any non-conflicting tokens are valid as XML text nodes.
- // This means we could end up consuming all available input if we're erroneously
- // given *only* a start tag (ie <tag> ...) .
+ const childNode: ResultOrFailure<DocNode> = this._parseXmlInnerText(tokenReader);
 
- // In other words, an unterminated XML element has a good chance of hitting EOI;
- // Let's handle that case here.
-
- if (tokenReader.peekTokenAfterKind() === TokenKind.EndOfInput) {
- return this._backtrackAndCreateErrorRange(
- tokenReader,
- startMarker,
- startTagClosingDelimiterMarker,
- TSDocMessageId.UnterminatedXmlElement,
- `The XML element "${startTagNameExcerpt.toString()}" is unterminated.`
- );
- }
-
- // Read out all of the text until we hit the next XML tag.
- tokenReader.readToken();
- }
-
- const plainText: TokenSequence | undefined = tokenReader.tryExtractAccumulatedSequence();
-
- if (!plainText) {
- continue;
+ if (isFailure(childNode)) {
+ return this._backtrackAndCreateErrorRangeForFailure(
+ tokenReader,
+ startMarker,
+ startTagClosingDelimiterMarker,
+ 'Invalid XML element: ',
+ childNode
+ );
  }
 
- childNodes.push(
- new DocPlainText({
- parsed: true,
- configuration: this._configuration,
- textExcerpt: plainText
- })
- );
+ childNodes.push(childNode);
  }
 
  const endMarker: number = tokenReader.createMarker();
@@ -1986,11 +1965,7 @@ export class NodeParser {
  if (endTagLessThanToken.kind !== TokenKind.LessThan) {
  return this._backtrackAndCreateError(
  tokenReader,
- // Note there's an edge case here that can cause the parser to fail
- // The input `<a><b/>` causes the next token to be EOI since it considers
- // `<b/>` to be a valid child element. After the child is parsed we will then
- // find out the the next token isn't a "<" and and try to backtrack to and read an EOI token.
- endTagLessThanToken.kind === TokenKind.EndOfInput ? endMarker - 1 : endMarker,
+ endMarker,
  TSDocMessageId.MissingXmlEndTag,
  'Expecting a closing tag starting with "</"'
  );
@@ -2031,7 +2006,7 @@ export class NodeParser {
  endTagNameStartMarker,
  endTagnameEndMarker,
  TSDocMessageId.XmlTagNameMismatch,
- `Expecting closing tag name to match opening tag name, got "${endTagNameExcerpt.toString()}" but expected "${startTagNameExcerpt.toString()}"`
+ `Expecting closing tag name "${endTagNameExcerpt.toString()}" to match opening tag name "${startTagNameExcerpt.toString()}"`
  );
  }
 
@@ -2062,6 +2037,8 @@ export class NodeParser {
  spacingAfterEndTagExcerpt,
  endTagOpeningDelimiterExcerpt,
  endTagClosingDelimiterExcerpt,
+ startTagNameExcerpt,
+ endTagNameExcerpt,
  nameExcerpt: startTagNameExcerpt,
  xmlAttributes: xmlAttributes,
  selfClosingTag,
@@ -2071,6 +2048,36 @@ export class NodeParser {
  return element;
  }
 
+ private _parseXmlInnerText(tokenReader: TokenReader): ResultOrFailure<DocNode> {
+ while (tokenReader.peekTokenKind() !== TokenKind.LessThan) {
+ // Technically any non-conflicting tokens are valid as XML text nodes.
+ // This means we could end up consuming all available input if we're erroneously
+ // given *only* a start tag (ie <tag> ...) .
+
+ // In other words, an unterminated XML element has a good chance of hitting EOI;
+ // Let's handle that case here.
+
+ if (tokenReader.peekTokenAfterKind() === TokenKind.EndOfInput) {
+ return this._createFailureForToken(
+ tokenReader,
+ TSDocMessageId.UnterminatedXmlElement,
+ `The XML element is unterminated.`
+ );
+ }
+
+ // Read out all of the text until we hit the next XML tag.
+ tokenReader.readToken();
+ }
+
+ const plainText: TokenSequence | undefined = tokenReader.extractAccumulatedSequence();
+
+ return new DocPlainText({
+ parsed: true,
+ configuration: this._configuration,
+ textExcerpt: plainText
+ });
+ }
+
  private _parseXmlAttribute(tokenReader: TokenReader): ResultOrFailure<DocXmlAttribute> {
  tokenReader.assertAccumulatedSequenceIsEmpty();