diff --git a/.gitignore b/.gitignore index 20c4b562..453c8bef 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,7 @@ env.bak/ venv.bak/ docs/* !docs/ai_configuration.md +!docs/spreadsheet_recipes.md !docs/ai_answers.md !docs/extract_ai_user_guide.md !docs/github-app-deployment.md diff --git a/docs/spreadsheet_recipes.md b/docs/spreadsheet_recipes.md new file mode 100644 index 00000000..00c777a4 --- /dev/null +++ b/docs/spreadsheet_recipes.md @@ -0,0 +1,96 @@ +# Spreadsheet recipe inputs and writes + +`excel.selected_data` reads the selected data supplied by WranglesXL for the +current recipe batch. It wraps `grid.selected_data`, which provides the same +options and behavior for grid hosts. Both support the usual read filters such +as `columns`, `not_columns`, and `where`. + +```yaml +read: + - join: + how: left + left_on: ID + right_on: ID + sources: + - excel.selected_data: {} + - file: + name: products.csv +``` + +Excel batches only the selected rows. A batch size of 2 over five rows sends +2, 2, and 1 rows, with the matching selection headings, batch number, batch +total, and row count. External-only reads execute once, independently of the +selection's size. A mixed recipe executes its authored external reads for each +selected-data batch; scope ID-based queries to that batch's IDs when necessary. +An unfiltered external source can repeat rows across union batches. + +Selected data must be present and nonempty. A missing or empty selected-data +operand in a join, union, or concatenate raises an error, including when its +read filters remove every row. Legacy `input` remains supported; a standalone +generic Python `input` can still represent an empty dataframe. Within a nested +recipe wrangle, these reads use the current parent dataframe, including earlier +transformations. They do not reread the original UI selection. The `read: +recipe` connector does not inherit that dataframe. + +## Spreadsheet write modes + +| Mode | Connector | Behavior | +| --- | --- | --- | +| `columns` | `excel.columns` | Insert new/changed output columns alongside the selected rows. | +| `sheet` | `excel.sheet` | Write all requested columns to the configured sheet/range. | + +The complete calculation dataframe remains available. In columns mode, XL +compares the returned values with that batch's input and omits unchanged input +columns from the write. If everything is unchanged, nothing is inserted. The +number of output rows must equal the number of selected input rows; use sheet +mode for row-expanding or row-reducing results. Validation happens before +columns or headers are inserted for the batch. + +```yaml +write: + - excel.columns: + columns: [Description, Category] + - excel.sheet: + name: Complete result + action: overwrite +``` + +Each write respects its own column selection. Sheet mode retains unchanged +input columns and the existing `name`, `cell`, action, and formatting options. +Later API batches append to the same resolved target. `matrix` repeats its +configured child writes: a child `excel.sheet` still uses sheet mode. Matrix +is a separate connector and is not another spreadsheet write mode. + +Columns appearing in later batches are appended in first-seen order, with +values aligned by column name. Their final order cannot be known in advance. +If an input column first changes in a later batch, XL fills its earlier output +cells from the original selected values; it also retains subsequent unchanged +values in that output column. It stores range coordinates rather than buffering +the complete selection. + +## Compatibility and execution + +- `dataframe` keeps its Python return-value and column-selection semantics. + In XL columns mode, the runner translates a top-level legacy `dataframe` + write to `excel.columns`, including when combined with sheet writes. +- Recipes using selected data default to columns mode. Existing external-only + top-level reads retain their implicit sheet output. Legacy `output: range` + and `output: sheet` select the default mode; explicit `excel.columns` and + `excel.sheet` payloads select their own modes. +- An explicit columns write from an external-only read uses the selection only + as an output target and checks row alignment. The external read executes once. +- Nested recipe wrangles suppress side-effecting writes, including both Excel + writers. Use `dataframe` to shape a nested recipe's returned dataframe. +- Normal list runs use the Production version when present, otherwise latest. + Development Editor runs use the loaded/current editor definition, including + historical versions. The same snapshot controls planning and every batch. + +The shared UI planning and dataframe projection helpers live in WranglesJS's +`recipe-execution` entry point. XL requests a full response with +`drop_unmodified_cols=false` and performs unchanged-column suppression at its +write boundary. Other API callers retain their existing default behavior. + +The Python package containing these connectors must be promoted to the recipe +Lambda used by XL. A Python merge alone does not update that runtime. Likewise, +XL requires a WranglesJS package containing `recipe-execution`; publish/install +that companion package before integrating the XL change. diff --git a/schema/generate_recipe_schema.py b/schema/generate_recipe_schema.py index ad929f09..2aefee8d 100644 --- a/schema/generate_recipe_schema.py +++ b/schema/generate_recipe_schema.py @@ -3,7 +3,6 @@ import json import logging import yaml -import requests import jsonschema # Get the parent directory of the current file @@ -125,7 +124,10 @@ def getConnectorDocs(schema_wrangles, obj, path): schema['write']['dataframe'] = yaml.safe_load( """ type: object - description: Define the dataframe that is returned from the recipe.run() function + description: >- + Define the dataframe returned from recipe.run(). Retains its generic + Python return-value behavior. In Excel this is also legacy columns-mode + syntax; prefer excel.columns for an explicit spreadsheet write mode. properties: {} """ ) @@ -234,7 +236,7 @@ def getMethodDocs(schema_wrangles, obj, path): recipe_schema['$defs']['wrangles']['items']['properties'] = schema['wrangles'] # Validate the generated schema -jsonschema.validate(recipe_schema, requests.get('http://json-schema.org/draft-07/schema#').json()) +jsonschema.Draft7Validator.check_schema(recipe_schema) # Write final schema with open('schema.json', 'w') as f: diff --git a/tests/connectors/test_excel.py b/tests/connectors/test_excel.py index 9419a3f6..2a8c599d 100644 --- a/tests/connectors/test_excel.py +++ b/tests/connectors/test_excel.py @@ -35,6 +35,52 @@ def test_default_write(): ) +def test_columns_and_sheet_writes_keep_complete_logical_dataframe(): + memory.clear() + try: + original = pd.DataFrame({'ID': [1, 2], 'Value': ['a', 'b']}) + result = wrangles.recipe.run( + {'write': [ + {'excel.columns': {'columns': ['Value']}}, + {'excel.sheet': {'name': 'Complete result'}}, + ]}, dataframe=original, + ) + pd.testing.assert_frame_equal(result, original) + payloads = list(memory.dataframes.values()) + assert payloads[0]['connector'] == 'excel.columns.write' + assert payloads[0]['columns'] == ['Value'] + assert payloads[0]['data'] == [['a'], ['b']] + assert payloads[1]['connector'] == 'excel.sheet.write' + assert payloads[1]['columns'] == ['ID', 'Value'] + assert payloads[1]['data'] == [[1, 'a'], [2, 'b']] + finally: + memory.clear() + + +def test_nested_excel_columns_does_not_emit_side_effecting_output(): + memory.clear() + try: + result = wrangles.recipe.run( + {'wrangles': [{'recipe': { + 'write': [{'excel.columns': {'columns': ['Value']}}] + }}]}, dataframe=pd.DataFrame({'ID': [1], 'Value': ['a']}), + ) + assert result.columns.tolist() == ['ID', 'Value'] + assert not memory.dataframes + finally: + memory.clear() + + +def test_generic_dataframe_still_shapes_the_python_return_value(): + memory.clear() + result = wrangles.recipe.run( + {'write': {'dataframe': {'columns': ['Value']}}}, + dataframe=pd.DataFrame({'ID': [1], 'Value': ['a']}), + ) + assert result.to_dict('list') == {'Value': ['a']} + assert not memory.dataframes + + def test_recipe_wrangle_in_batch_writes_all_rows_to_excel_sheet(): """ Test the WranglesXL output connector path when a recipe wrangle is used diff --git a/tests/connectors/test_input.py b/tests/connectors/test_input.py index a5fbfeb8..480cd94e 100644 --- a/tests/connectors/test_input.py +++ b/tests/connectors/test_input.py @@ -1,5 +1,6 @@ import wrangles import pandas as pd +import pytest def test_read(): """ @@ -79,3 +80,120 @@ def test_read_union(): df["header1"].values.tolist() == ["a", "b", "c", "d"] and len(df) == 4 ) + + +@pytest.mark.parametrize('connector', ['grid.selected_data', 'excel.selected_data']) +def test_selected_data_filters_current_batch_without_mutating_input(connector): + batch = pd.DataFrame({'ID': [3, 4], 'Value': ['c', 'd']}) + result = wrangles.recipe.run( + {'read': {connector: {'columns': ['Value'], 'where': 'ID = 4'}}}, + dataframe=batch, + ) + assert result.to_dict('list') == {'Value': ['d']} + assert batch.to_dict('list') == {'ID': [3, 4], 'Value': ['c', 'd']} + + +@pytest.mark.parametrize('connector', ['input', 'grid.selected_data', 'excel.selected_data']) +@pytest.mark.parametrize('composition', ['union', 'join', 'concatenate']) +@pytest.mark.parametrize('data', [None, pd.DataFrame(columns=['ID'])]) +def test_composed_selected_data_rejects_missing_and_empty(connector, composition, data): + params = {'sources': [{connector: {}}, {'test': {'rows': 1, 'values': {'ID': 1}}}]} + if composition == 'join': + params['on'] = 'ID' + with pytest.raises(ValueError, match='missing or empty'): + wrangles.recipe.run({'read': {composition: params}}, dataframe=data) + + +@pytest.mark.parametrize('connector', ['input', 'grid.selected_data', 'excel.selected_data']) +def test_composed_selected_data_rejects_filter_removing_all_rows(connector): + with pytest.raises(ValueError, match='empty after filtering'): + wrangles.recipe.run( + {'read': {'union': {'sources': [ + {connector: {'where': 'ID > 10'}}, + {'test': {'rows': 1, 'values': {'ID': 2}}}, + ]}}}, dataframe=pd.DataFrame({'ID': [1]}), + ) + + +def test_generic_input_still_allows_an_empty_dataframe_outside_composition(): + result = wrangles.recipe.run({'read': {'input': {}}}, dataframe=pd.DataFrame(columns=['ID'])) + assert result.empty and result.columns.tolist() == ['ID'] + + +@pytest.mark.parametrize('connector', ['grid.selected_data', 'excel.selected_data']) +@pytest.mark.parametrize('saved', [False, True]) +def test_nested_selected_data_reads_transformed_parent_dataframe(connector, saved, monkeypatch): + child = {'read': {connector: {}}} + if saved: + monkeypatch.setattr(wrangles.data, 'model', lambda *_: { + 'purpose': 'recipe', 'name': 'Synthetic child', + }) + monkeypatch.setattr(wrangles.data, 'model_content', lambda *_: { + 'recipe': f'read:\n - {connector}: {{}}\n', + }) + child = {'name': '12345678-abcd-abcd'} + result = wrangles.recipe.run( + {'wrangles': [ + {'convert.case': {'input': 'Value', 'case': 'upper'}}, + {'recipe': child}, + ]}, dataframe=pd.DataFrame({'ID': [3], 'Value': ['batch three']}), + ) + assert result.to_dict('list') == {'ID': [3], 'Value': ['BATCH THREE']} + + +def test_empty_external_result_is_not_an_empty_selected_data_error(): + result = wrangles.recipe.run({'read': {'test': {'rows': 0, 'values': {'ID': 1}}}}) + assert result.empty + + +@pytest.mark.parametrize('connector', ['input', 'grid.selected_data', 'excel.selected_data']) +def test_mixed_read_uses_only_current_batch_ids(connector): + # Synthetic lookup: the recipe author scopes the external query to the + # current invocation's IDs. Excel does not batch or deduplicate its rows. + catalog = pd.DataFrame({'ID': range(1, 6), 'Product': list('abcde')}) + requested_ids = [] + outputs = [] + for ids in ([1, 2], [3, 4], [5]): + def query_products(ids): + requested_ids.append(ids) + return catalog[catalog.ID.isin(ids)].copy() + + outputs.append(wrangles.recipe.run( + {'read': {'join': {'on': 'ID', 'how': 'left', 'sources': [ + {connector: {}}, + {'custom.query_products': {'ids': '${batch_ids}'}}, + ]}}}, + dataframe=pd.DataFrame({'ID': ids}), + variables={'batch_ids': ids}, + functions=query_products, + )) + assert requested_ids == [[1, 2], [3, 4], [5]] + assert pd.concat(outputs).to_dict('list') == catalog.to_dict('list') + + +def test_composed_read_allows_empty_external_source(): + result = wrangles.recipe.run( + {'read': {'union': {'sources': [ + {'input': {}}, {'test': {'rows': 0, 'values': {'ID': 1}}}, + ]}}}, dataframe=pd.DataFrame({'ID': [7]}), + ) + assert result.to_dict('list') == {'ID': [7]} + + +def test_generated_schema_exposes_shared_and_excel_selected_data(tmp_path, monkeypatch): + import json + import runpy + from pathlib import Path + import jsonschema + + root = Path(__file__).resolve().parents[2] + (tmp_path / 'recipe_base_schema.json').write_text( + (root / 'schema/recipe_base_schema.json').read_text(encoding='utf-8'), + encoding='utf-8', + ) + monkeypatch.chdir(tmp_path) + runpy.run_path(str(root / 'schema/generate_recipe_schema.py')) + schema = json.loads((tmp_path / 'schema.json').read_text(encoding='utf-8')) + for connector in ['input', 'grid.selected_data', 'excel.selected_data']: + jsonschema.validate({'read': [{connector: {'columns': ['ID']}}]}, schema) + jsonschema.validate({'write': [{'excel.columns': {'columns': ['Result']}}]}, schema) diff --git a/wrangles/connectors/__init__.py b/wrangles/connectors/__init__.py index 1c50c4c0..a3cdbb48 100644 --- a/wrangles/connectors/__init__.py +++ b/wrangles/connectors/__init__.py @@ -8,6 +8,7 @@ from . import concurrent from . import duckdb from . import excel +from . import grid from . import file from . import http from . import memory diff --git a/wrangles/connectors/excel.py b/wrangles/connectors/excel.py index 11226df0..b79d1a63 100644 --- a/wrangles/connectors/excel.py +++ b/wrangles/connectors/excel.py @@ -3,6 +3,7 @@ """ import pandas as _pd from . import memory as _memory +from . import grid as _grid import logging as _logging @@ -119,6 +120,37 @@ def _append_rows_by_column(saved: dict, df: _pd.DataFrame) -> bool: return True +class selected_data: + """Excel-facing wrapper for the shared grid selection contract.""" + + def read(dataframe: _pd.DataFrame = None): + return _grid.selected_data.read(dataframe) + + _schema = {"read": _grid.selected_data._schema["read"].replace( + "Read the selected data", "Read the selected Excel data" + )} + + +class columns: + """Spreadsheet write mode: columns alongside the selected rows.""" + + def write(df: _pd.DataFrame): + # Keep every requested column here. The grid writer compares the full + # result with its batch input and suppresses unchanged columns only + # in the columns-mode projection. + _memory.write(df, connector="excel.columns.write", orient="split") + + _schema = {"write": """ +type: object +description: >- + Spreadsheet write mode: columns. Insert new or changed output columns + alongside the selected rows in Excel. The output must retain row alignment + with the selection. Unchanged input columns are not duplicated. +additionalProperties: false +properties: {} +"""} + + class sheet(): _schema = {} @@ -179,7 +211,9 @@ def write( _schema["write"] = """ type: object - description: Write to an excel sheet + description: >- + Spreadsheet write mode: sheet. Write all requested columns to an + Excel sheet/range, including unchanged input columns. additionalProperties: false properties: name: diff --git a/wrangles/connectors/grid.py b/wrangles/connectors/grid.py new file mode 100644 index 00000000..cc40108f --- /dev/null +++ b/wrangles/connectors/grid.py @@ -0,0 +1,25 @@ +"""Data supplied by a grid application to the current recipe invocation.""" +import pandas as _pd + + +class selected_data: + """Shared selected-data read, independent of the grid application's API.""" + + def read(dataframe: _pd.DataFrame = None): + if dataframe is None or dataframe.empty: + raise ValueError( + "Selected data is missing or empty. Select data rows before " + "running this recipe." + ) + return dataframe.copy() + + _schema = {"read": """ +type: object +description: >- + Read the selected data supplied to this recipe invocation. Grid applications + supply the current batch of selected rows. Within a recipe wrangle, this is + the current parent dataframe, including any preceding transformations. + Missing or empty selected data raises an error. +additionalProperties: false +properties: {} +"""} diff --git a/wrangles/recipe.py b/wrangles/recipe.py index 1c2e8afd..bea3cdb8 100644 --- a/wrangles/recipe.py +++ b/wrangles/recipe.py @@ -448,7 +448,8 @@ def _read_data( recipe: _Union[dict, list], functions: dict = None, variables: dict = None, - input_dataframe: _pandas.DataFrame = None + input_dataframe: _pandas.DataFrame = None, + _composite_source: bool = False ) -> _pandas.DataFrame: """ Import data from requested datasources as defined by the recipe @@ -466,6 +467,8 @@ def _read_data( recipe = [recipe] results = [] + # A list of reads is also combined by the recipe runner. + _composite_source = _composite_source or len(recipe) > 1 for read in recipe: if not isinstance(read, dict): if isinstance(read, str): @@ -499,12 +502,23 @@ def _read_data( # Reference the recipe execution input dataframe if read_type == "input": df = input_dataframe + if _composite_source and (df is None or df.empty): + raise ValueError( + "Selected input data is missing or empty in a composed read. " + "Supply data rows for every declared input source." + ) + elif read_type in ('grid.selected_data', 'excel.selected_data'): + func = _get_nested_function(read_type, _connectors, functions, 'read') + df = func(input_dataframe, **params_specific) # Allow blended imports elif read_type in ['join', 'concatenate', 'union']: dfs = [] # Recursively call sub-reads for source in params_specific['sources']: - result = _read_data(source, functions, variables, input_dataframe) + result = _read_data( + source, functions, variables, input_dataframe, + _composite_source=True + ) if result is None: # Skip if None returned # e.g. in the case of a false if condition @@ -544,7 +558,17 @@ def _read_data( if isinstance(df, _pandas.DataFrame): # Response is a single dataframe, filter appropriately - results.append(_filter_dataframe(df, **params_general)) + filtered = _filter_dataframe(df, **params_general) + if ( + filtered.empty and + (read_type in ('grid.selected_data', 'excel.selected_data') or + (read_type == 'input' and _composite_source)) + ): + raise ValueError( + "Selected input data is empty after filtering. " + "Supply data rows for every declared input source." + ) + results.append(filtered) elif isinstance(df, list) and all([isinstance(x, _pandas.DataFrame) for x in df]): # Response is a list of dataframes, filter each appropriately results.extend([_filter_dataframe(x, **params_general) for x in df])