torch.tx.TokenType

export type TokenType =
  | 'number'
  | 'string'
  | 'tensor'
  | 'identifier'
  | 'operator'
  | 'lparen'
  | 'rparen'
  | 'lbracket'
  | 'rbracket'
  | 'comma'
  | 'question'
  | 'colon'
  | 'arrow' // -> for dimension label output specification
  | 'eof';

Values

'number'
'string'
'tensor'
'identifier'
'operator'
'lparen'
'rparen'
'lbracket'
'rbracket'
'comma'
'question'
'colon'
'arrow'
'eof'

Tokenizer for tensor expressions.

Converts expression strings with tensor placeholders into a stream of tokens. Handles operators, numbers, identifiers (function names), and tensor references.