Function tokenizeWithSpans

tokenizeWithSpans(input: string): HighlightToken[]

Splits a raw query into highlightable tokens that carry source offsets and, unlike the parser's tokenize, preserve whitespace and the quote characters so the rendered markup round-trips exactly to the input. Bare words matching OPERATOR_WORDS are tagged as keyword; parens carry a nesting depth for rainbow coloring; unmatched parens and unterminated quotes are flagged with error.

Parameters

input: string
The raw query string.

Returns HighlightToken[]

Ordered tokens covering every character of input.

Example

tokenizeWithSpans("a AND (b)");
// term "a", ws, keyword "AND", ws, paren "(" depth 0, term "b", paren ")" depth 0

Source

export function tokenizeWithSpans(input: string): HighlightToken[] {
  const tokens: HighlightToken[] = [];
  const openParenTokens: number[] = [];
  let depth = 0;
  let i = 0;

  while (i < input.length) {
    const char = input[i];

    if (isWhitespace(char)) {
      const start = i;
      while (i < input.length && isWhitespace(input[i])) i++;
      tokens.push({ kind: 'whitespace', text: input.slice(start, i), start, end: i });
      continue;
    }

    if (char === '(') {
      tokens.push({
        kind: 'paren',
        text: '(',
        start: i,
        end: i + 1,
        depth: depth % PAREN_PALETTE_SIZE,
      });
      openParenTokens.push(tokens.length - 1);
      depth++;
      i++;
      continue;
    }

    if (char === ')') {
      if (openParenTokens.length > 0) {
        openParenTokens.pop();
        depth--;
        tokens.push({
          kind: 'paren',
          text: ')',
          start: i,
          end: i + 1,
          depth: depth % PAREN_PALETTE_SIZE,
        });
      } else {
        tokens.push({ kind: 'paren', text: ')', start: i, end: i + 1, error: true });
      }
      i++;
      continue;
    }

    if (char === '"') {
      const start = i;
      i++;
      while (i < input.length && input[i] !== '"') i++;
      const terminated = i < input.length;
      if (terminated) i++; // consume the closing quote
      tokens.push({
        kind: 'quoted',
        text: input.slice(start, i),
        start,
        end: i,
        error: !terminated,
      });
      continue;
    }

    // Bare word: runs until whitespace, a paren, or a quote.
    const start = i;
    while (
      i < input.length &&
      !isWhitespace(input[i]) &&
      input[i] !== '(' &&
      input[i] !== ')' &&
      input[i] !== '"'
    ) {
      i++;
    }
    const text = input.slice(start, i);
    const kind: HighlightTokenKind = OPERATOR_WORDS.has(text.toUpperCase()) ? 'keyword' : 'term';
    tokens.push({ kind, text, start, end: i });
  }

  // Any parens still open at the end are unbalanced.
  for (const idx of openParenTokens) {
    tokens[idx].error = true;
  }

  return tokens;
}