diff --git a/CHANGELOG.md b/CHANGELOG.md index aeea8dd511..d780b7beaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed +- Fixed the parser rejecting formulas that carry the function-name prefixes Excel writes into `.xlsx` files (`_xlfn.`, `_xlfn._xlws.`, `_xlws.`, `_xlpm.`, `_xludf.`), which made every function added after Excel 2007 (e.g. `IFS`, `XLOOKUP`, `TEXTJOIN`, `FILTER`, `SORT`) fail to parse when a workbook was imported with a library that passes the prefixes through. The prefixes are now ignored, and `getCellFormula()` returns the formula without them. [#1771](https://github.com/handsontable/hyperformula/pull/1771) - Fixed the `AVERAGEIF` function returning a division-by-zero error when the calculated average was `0`. [#1733](https://github.com/handsontable/hyperformula/pull/1733) - Fixed the localized names of `VSTACK` and `HSTACK` in 14 language packs to match Microsoft Excel. [#1748](https://github.com/handsontable/hyperformula/pull/1748) - Fixed the MAXPOOL and MEDIANPOOL functions throwing an uncaught `TypeError` instead of returning the `#VALUE!` error when the range dimensions are not a whole multiple of the window size and the stride. [#1718](https://github.com/handsontable/hyperformula/pull/1718) diff --git a/docs/guide/file-import.md b/docs/guide/file-import.md index 470383da9e..9dbf290b52 100644 --- a/docs/guide/file-import.md +++ b/docs/guide/file-import.md @@ -23,6 +23,27 @@ To import CSV files, use a third-party [CSV parser](https://www.npmjs.com/search To import XLSX files, use a third-party [XLSX parser](https://www.npmjs.com/search?q=xlsx) (e.g., [ExcelJS](https://www.npmjs.com/package/exceljs) or [xlsx](https://www.npmjs.com/package/xlsx)). Then pass the result to HyperFormula as a JavaScript array. +### Excel's internal function prefixes + +Excel stores some function names with an internal prefix. It marks every function added after Excel 2007 this way, so a file can contain `_xlfn.IFS(...)` where the user typed `IFS(...)`. The prefixes are an artifact of how Excel saves the file, not part of the function name. + +Which prefixes reach your code depends on the parser you use. ExcelJS passes them through. SheetJS removes `_xlfn.` but keeps `_xlws.`. + +HyperFormula ignores these prefixes, so you can pass the formula straight to the engine: + +| Prefix | Example | +| --- | --- | +| `_xlfn.` | `=_xlfn.IFS(A1>B1,"Pass","Fail")` | +| `_xlfn._xlws.` | `=_xlfn._xlws.FILTER(A1:A9,B1:B9>1)` | +| `_xlws.` | `=_xlws.SORT(A1:A9)` | +| `_xludf.` | `=_xludf.MY_FUNCTION()` | + +The `_xlpm.` prefix marks a `LAMBDA`/`LET` parameter name. HyperFormula does not support `LAMBDA` or `LET`, so a formula containing one returns `#NAME?` whether or not the prefix is present. + +[`getCellFormula()`](../api/classes/hyperformula.md#getcellformula) returns the formula without the prefix, so `=_xlfn.IFS(A1>B1,"Pass","Fail")` reads back as `=IFS(A1>B1,"Pass","Fail")`. + +A prefix does not add a function. If HyperFormula does not support the function itself, the cell holds a `#NAME?` error. See the [list of supported functions](built-in-functions.md). + ### Example: Import XLSX files in Node This example uses [ExcelJS](https://www.npmjs.com/package/exceljs) to import XLSX files into HyperFormula. diff --git a/src/parser/FormulaParser.ts b/src/parser/FormulaParser.ts index 82c70e1c1d..9783b8c5bd 100644 --- a/src/parser/FormulaParser.ts +++ b/src/parser/FormulaParser.ts @@ -69,6 +69,7 @@ import { ArrayLParen, ArrayRParen, BooleanOp, + canonicalProcedureNameFromToken, CellReference, ColumnRange, ConcatenateOp, @@ -157,8 +158,7 @@ export class FormulaParser extends EmbeddedActionsParser { */ private procedureExpression: AstRule = this.RULE('procedureExpression', () => { const procedureNameToken = this.CONSUME(ProcedureName) as ExtendedToken - const procedureName = procedureNameToken.image.toUpperCase().slice(0, -1) - const canonicalProcedureName = this.lexerConfig.functionMapping[procedureName] ?? procedureName + const canonicalProcedureName = canonicalProcedureNameFromToken(procedureNameToken.image, this.lexerConfig.functionMapping) const args: Ast[] = [] let argument = this.SUBRULE(this.booleanExpressionOrEmpty) diff --git a/src/parser/LexerConfig.ts b/src/parser/LexerConfig.ts index 72fc4bb3b6..c0f602c7e7 100644 --- a/src/parser/LexerConfig.ts +++ b/src/parser/LexerConfig.ts @@ -9,6 +9,7 @@ import {ParserConfig} from './ParserConfig' import { ALL_WHITESPACE_PATTERN, COLUMN_REFERENCE_PATTERN, + EXCEL_INTERNAL_FUNCTION_PREFIX_PATTERN, NON_RESERVED_CHARACTER_PATTERN, ODFF_WHITESPACE_PATTERN, RANGE_OPERATOR, @@ -47,7 +48,58 @@ export const RangeSeparator = createToken({ name: 'RangeSeparator', pattern: new export const ColumnRange = createToken({ name: 'ColumnRange', pattern: new RegExp(`${COLUMN_REFERENCE_PATTERN}${RANGE_OPERATOR}${COLUMN_REFERENCE_PATTERN}`) }) export const RowRange = createToken({ name: 'RowRange', pattern: new RegExp(`${ROW_REFERENCE_PATTERN}${RANGE_OPERATOR}${ROW_REFERENCE_PATTERN}`) }) -export const ProcedureName = createToken({ name: 'ProcedureName', pattern: new RegExp(`([${UNICODE_LETTER_PATTERN}][${NON_RESERVED_CHARACTER_PATTERN}]*)\\(`) }) +export const ProcedureName = createToken({ name: 'ProcedureName', pattern: new RegExp(`(?:${EXCEL_INTERNAL_FUNCTION_PREFIX_PATTERN})?([${UNICODE_LETTER_PATTERN}][${NON_RESERVED_CHARACTER_PATTERN}]*)\\(`) }) + +const excelInternalFunctionPrefixRegexp = new RegExp(`^(?:${EXCEL_INTERNAL_FUNCTION_PREFIX_PATTERN})`) +const UNDERSCORE_CHAR_CODE = '_'.charCodeAt(0) + +/** + * Strips one of Excel's internal function-name prefixes (see EXCEL_INTERNAL_FUNCTION_PREFIX_PATTERN) + * off the front of a token image, if present. + * + * Every prefix starts with `_`, and this function is called on every ProcedureName and + * OffsetProcedureName token in every parse, so the common case — a name with no prefix at all — takes + * a plain character check instead of always paying for the regex match-and-fail. + * + * @param {string} nameWithoutTrailingParen - a token image with any trailing `(` already removed + */ +function stripExcelInternalFunctionPrefix(nameWithoutTrailingParen: string): string { + if (nameWithoutTrailingParen.charCodeAt(0) !== UNDERSCORE_CHAR_CODE) { + return nameWithoutTrailingParen + } + return nameWithoutTrailingParen.replace(excelInternalFunctionPrefixRegexp, '') +} + +/** + * Reads the canonical function name out of a ProcedureName token. + * + * The token image spans the whole match, so it carries the trailing opening parenthesis and, for a + * formula imported from an .xlsx file, one of the prefixes Excel prepends when it serializes a + * workbook. Dropping both here is what lets `_xlfn.IFS(A1)` resolve to the same function as `IFS(A1)`. + * + * The prefix is removed before the name is upper-cased, because the prefixes are matched in lower + * case only. + * + * @param {string} image - image of the ProcedureName token, for example `_xlfn.IFS(` + * @param {Record} functionMapping - maps a translated function name to its canonical English name + */ +export function canonicalProcedureNameFromToken(image: string, functionMapping: Record): string { + const procedureName = stripExcelInternalFunctionPrefix(image.slice(0, -1)).toUpperCase() + return functionMapping[procedureName] ?? procedureName +} + +/** + * Strips an Excel internal function-name prefix off an OffsetProcedureName token's image. + * + * OffsetProcedureName has no trailing parenthesis to remove (unlike ProcedureName — OFFSET's grammar + * rule consumes `(` separately) and no translation lookup (the token pattern already embeds the + * localized OFFSET name), so it needs only the prefix stripped, not the full canonicalization above. + * + * @param {string} image - image of the OffsetProcedureName token, for example `_xlfn.OFFSET` + */ +export function canonicalOffsetProcedureNameFromToken(image: string): string { + return stripExcelInternalFunctionPrefix(image) +} const cellReferenceMatcher = new CellReferenceMatcher() export const CellReference = createToken({ @@ -93,7 +145,15 @@ export const buildLexerConfig = (config: ParserConfig): LexerConfig => { const ArrayRowSeparator = createToken({name: 'ArrayRowSep', pattern: config.arrayRowSeparator}) const ArrayColSeparator = createToken({name: 'ArrayColSep', pattern: config.arrayColumnSeparator}) const NumberLiteral = createToken({ name: 'NumberLiteral', pattern: new RegExp(`(([${config.decimalSeparator}]\\d+)|(\\d+([${config.decimalSeparator}]\\d*)?))(e[+-]?\\d+)?`) }) - const OffsetProcedureName = createToken({ name: 'OffsetProcedureName', pattern: new RegExp(offsetProcedureNameLiteral, 'i') }) + // OFFSET has its own token because it has its own grammar rule, so it needs the prefix handling of + // ProcedureName repeated here. The 'i' flag exists for the translated OFFSET name (pre-dates this + // prefix support) and applies to the whole pattern, so unlike ProcedureName it also makes the prefix + // case-insensitive: `_XLFN.OFFSET(...)` is accepted here where `_XLFN.SUM(...)` is a parsing error. + // Left as-is rather than given a dedicated case-sensitive-prefix matcher: OFFSET predates the OOXML + // cutoff this whole prefix scheme exists for, so Excel can never actually emit a prefixed OFFSET call + // in any case — the asymmetry has no reachable real input, only hand-typed formulas (see the + // "an upper-cased prefix is accepted on OFFSET, unlike everywhere else" test). + const OffsetProcedureName = createToken({ name: 'OffsetProcedureName', pattern: new RegExp(`(?:${EXCEL_INTERNAL_FUNCTION_PREFIX_PATTERN})?${offsetProcedureNameLiteral}`, 'i') }) let ArgSeparator: TokenType let inject: TokenType[] diff --git a/src/parser/ParserWithCaching.ts b/src/parser/ParserWithCaching.ts index a116aa8e56..e8f8eaa157 100644 --- a/src/parser/ParserWithCaching.ts +++ b/src/parser/ParserWithCaching.ts @@ -19,6 +19,8 @@ import {Cache} from './Cache' import {FormulaLexer, FormulaParser, ExtendedToken} from './FormulaParser' import { buildLexerConfig, + canonicalOffsetProcedureNameFromToken, + canonicalProcedureNameFromToken, CellReference, ColumnRange, LexerConfig, @@ -239,9 +241,10 @@ export class ParserWithCaching { hash = hash.concat(cellAddress.hash(true)) } } else if (tokenMatcher(token, ProcedureName)) { - const procedureName = token.image.toUpperCase().slice(0, -1) - const canonicalProcedureName = this.lexerConfig.functionMapping[procedureName] ?? procedureName + const canonicalProcedureName = canonicalProcedureNameFromToken(token.image, this.lexerConfig.functionMapping) hash = hash.concat(canonicalProcedureName, '(') + } else if (tokenMatcher(token, this.lexerConfig.OffsetProcedureName)) { + hash = hash.concat(canonicalOffsetProcedureNameFromToken(token.image)) } else if (tokenMatcher(token, ColumnRange)) { const [start, end] = token.image.split(':') const startAddress = columnAddressFromString(start, baseAddress, this.resolveSheetReference) diff --git a/src/parser/parser-consts.ts b/src/parser/parser-consts.ts index 2572a8f551..6b0d05104c 100644 --- a/src/parser/parser-consts.ts +++ b/src/parser/parser-consts.ts @@ -22,6 +22,31 @@ export const ROW_REFERENCE_PATTERN = `(${SHEET_NAME_PATTERN})?\\${ABSOLUTE_OPERA export const R1C1_CELL_REFERENCE_PATTERN = '[rR][0-9]*[cC][0-9]*' export const CELL_REFERENCE_WITH_NEXT_CHARACTER_PATTERN = `(${CELL_REFERENCE_PATTERN})[^${NON_RESERVED_CHARACTER_PATTERN}]` +/** + * Prefixes that Excel prepends to a function name when it serializes a workbook. + * + * Excel marks every function added after the original OOXML specification (~Excel 2007) with one + * of these prefixes in the stored XML, so a file may contain `_xlfn.IFS(...)` where the user typed + * `IFS(...)`. They are serialization artifacts rather than a part of the function name, and other + * spreadsheet engines drop them on import, so HyperFormula accepts and ignores them too. + * + * | Prefix | Meaning | + * |----------------|----------------------------------------------------------| + * | `_xlfn.` | function newer than the OOXML specification | + * | `_xlfn._xlws.` | as above, and callable only in a worksheet | + * | `_xlws.` | callable only in a worksheet | + * | `_xlpm.` | `LAMBDA`/`LET` parameter name | + * | `_xludf.` | user-defined function (a `LAMBDA` stored in Name Manager) | + * + * The alternatives are ordered longest-first so that `_xlfn._xlws.` is consumed whole instead of + * leaving `_xlws.` behind. Excel always writes them in lower case, so wherever this pattern is used + * to recognize a *function call* (the `ProcedureName` token in LexerConfig.ts) it is matched in lower + * case only: an upper-cased spelling there is not something Excel can produce. The one exception is + * the `OffsetProcedureName` token, which reuses this pattern inside an already case-insensitive regex + * (for the translated OFFSET name) and so matches an upper-cased prefix too — see its comment. + */ +export const EXCEL_INTERNAL_FUNCTION_PREFIX_PATTERN = '_xlfn\\._xlws\\.|_xlfn\\.|_xlws\\.|_xlpm\\.|_xludf\\.' + export const NAMED_EXPRESSION_PATTERN = `[${UNICODE_LETTER_PATTERN}_][${NON_RESERVED_CHARACTER_PATTERN}]*` export const ALL_DIGITS_ARRAY = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']