From ca14289c28b28bb08861ee4497085eddb0e9a7ca Mon Sep 17 00:00:00 2001 From: zackdotcat Date: Mon, 10 Nov 2025 22:37:24 -0500 Subject: [PATCH] work :n progress --- db/records.py | 8 + db/sql/00_msar_all_objects_table.sql | 20 +- db/sql/05_msar.sql | 1020 +++++++++++++++-- mathesar/rpc/records.py | 44 +- mathesar_ui/src/api/rpc/records.ts | 30 +- .../components/cell-fabric/CellFabric.svelte | 21 + .../CountLinkedRecordCell.svelte | 455 ++++++++ .../MultiLinkedRecordCell.svelte | 518 +++++++++ .../linked-record/linkedRecordDmlUtils.ts | 195 ++++ .../data-types/components/typeDefinitions.ts | 66 ++ .../src/components/cell-fabric/utils.ts | 16 + mathesar_ui/src/i18n/languages/en/dict.json | 5 +- mathesar_ui/src/stores/table-data/index.ts | 8 + .../src/stores/table-data/joinPathUtils.ts | 55 + mathesar_ui/src/stores/table-data/meta.ts | 31 +- .../stores/table-data/recordSummariesCache.ts | 144 +++ mathesar_ui/src/stores/table-data/records.ts | 187 +++ .../table-data/relatedColumnDefinitions.ts | 540 +++++++++ .../src/stores/table-data/relatedColumns.ts | 123 ++ mathesar_ui/src/stores/table-data/sorting.ts | 7 +- .../src/stores/table-data/tabularData.ts | 124 +- .../AttachableRowSeekerController.ts | 5 +- .../src/systems/row-seeker/RowSeeker.svelte | 58 +- .../systems/row-seeker/RowSeekerController.ts | 53 +- .../systems/row-seeker/RowSeekerOption.svelte | 9 +- .../src/systems/table-view/TableView.svelte | 23 +- .../actions-pane/ActionsPane.svelte | 4 +- .../actions-pane/MiniActionsPane.svelte | 4 +- .../RelatedColumnsContent.svelte | 114 ++ .../RelatedColumnsDropdown.svelte | 41 + .../related-columns/RelatedColumnsTree.svelte | 226 ++++ .../related-columns/RelatedTableNode.svelte | 333 ++++++ .../related-columns/types.ts | 13 + .../systems/table-view/header/Header.svelte | 72 +- .../RelatedColumnHeaderCell.svelte | 122 ++ .../src/systems/table-view/row/Row.svelte | 16 +- .../src/systems/table-view/row/RowCell.svelte | 276 ++++- .../table-inspector/TableInspector.svelte | 36 +- .../src/systems/table-view/tableViewUtils.ts | 11 +- 39 files changed, 4794 insertions(+), 239 deletions(-) create mode 100644 mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/CountLinkedRecordCell.svelte create mode 100644 mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/MultiLinkedRecordCell.svelte create mode 100644 mathesar_ui/src/components/cell-fabric/data-types/components/linked-record/linkedRecordDmlUtils.ts create mode 100644 mathesar_ui/src/stores/table-data/joinPathUtils.ts create mode 100644 mathesar_ui/src/stores/table-data/recordSummariesCache.ts create mode 100644 mathesar_ui/src/stores/table-data/relatedColumnDefinitions.ts create mode 100644 mathesar_ui/src/stores/table-data/relatedColumns.ts create mode 100644 mathesar_ui/src/systems/table-view/actions-pane/record-operations/related-columns/RelatedColumnsContent.svelte create mode 100644 mathesar_ui/src/systems/table-view/actions-pane/record-operations/related-columns/RelatedColumnsDropdown.svelte create mode 100644 mathesar_ui/src/systems/table-view/actions-pane/record-operations/related-columns/RelatedColumnsTree.svelte create mode 100644 mathesar_ui/src/systems/table-view/actions-pane/record-operations/related-columns/RelatedTableNode.svelte create mode 100644 mathesar_ui/src/systems/table-view/actions-pane/record-operations/related-columns/types.ts create mode 100644 mathesar_ui/src/systems/table-view/header/header-cell/RelatedColumnHeaderCell.svelte diff --git a/db/records.py b/db/records.py index 8b61c017ac..fb14cbea1d 100644 --- a/db/records.py +++ b/db/records.py @@ -17,6 +17,7 @@ def list_records_from_table( group=None, return_record_summaries=False, table_record_summary_templates=None, + related_columns=None, ): """ Get records from a table. @@ -36,6 +37,8 @@ def list_records_from_table( group: An array of group definition objects. return_record_summaries: Whether to return self record summaries. table_record_summary_templates: A dict of record summary templates, per table. + related_columns: Optional list of related column requests to fetch + aggregated values from related tables. """ result = db_conn.exec_msar_func( conn, @@ -48,6 +51,7 @@ def list_records_from_table( _json_or_none(group), return_record_summaries, _json_or_none(table_record_summary_templates), + _json_or_none(related_columns), ).fetchone()[0] return result @@ -89,6 +93,7 @@ def search_records_from_table( offset=0, return_record_summaries=False, table_record_summary_templates=None, + related_columns=None, ): """ Get records from a table, according to a search specification @@ -101,6 +106,8 @@ def search_records_from_table( limit: The maximum number of rows we'll return. return_record_summaries: Whether to return self record summaries. table_record_summary_templates: A dict of record summary templates, per table. + related_columns: Optional list of related column requests to fetch + aggregated values from related tables. The search definition objects should have the form {"attnum": , "literal": } @@ -115,6 +122,7 @@ def search_records_from_table( offset, return_record_summaries, _json_or_none(table_record_summary_templates), + _json_or_none(related_columns), ).fetchone()[0] return result diff --git a/db/sql/00_msar_all_objects_table.sql b/db/sql/00_msar_all_objects_table.sql index 72f26ece93..5670499acd 100644 --- a/db/sql/00_msar_all_objects_table.sql +++ b/db/sql/00_msar_all_objects_table.sql @@ -83,6 +83,7 @@ INSERT INTO msar.all_mathesar_objects VALUES ('__msar', '__msar.rename_schema(text,text)', 'FUNCTION', NULL), ('__msar', '__msar.rename_table(text,text)', 'FUNCTION', NULL), ('__msar', '__msar.set_not_nulls(text,__msar.not_null_def[])', 'FUNCTION', NULL), + ('__msar', '__msar.split_ctes(text)', 'FUNCTION', NULL), ('__msar', '__msar.update_pk_sequence_to_latest(text,text)', 'FUNCTION', NULL), ('mathesar_types', 'mathesar_types.cast_to__double_quote_char_double_quote_("char")', 'FUNCTION', NULL), ('mathesar_types', 'mathesar_types.cast_to__double_quote_char_double_quote_(bigint)', 'FUNCTION', NULL), @@ -1220,12 +1221,18 @@ INSERT INTO msar.all_mathesar_objects VALUES ('msar', 'msar.uri_parts(text)', 'FUNCTION', NULL), ('msar', 'msar.uri_path(text)', 'FUNCTION', NULL), ('msar', 'msar.uri_query(text)', 'FUNCTION', NULL), - ('msar', 'msar.uri_scheme(text)', 'FUNCTION', NULL); - - --- --- Name: all_mathesar_objects all_mathesar_objects_obj_schema_obj_name_obj_kind_key; Type: CONSTRAINT; Schema: msar; Owner: - --- + ('msar', 'msar.uri_scheme(text)', 'FUNCTION', NULL), + ('msar', 'msar.build_related_column_id(jsonb,integer)', 'FUNCTION', NULL), + ('msar', 'msar.path_hash(jsonb)', 'FUNCTION', NULL), + ('msar', 'msar.related_alias(text)', 'FUNCTION', NULL), + ('msar', 'msar.value_col_name(integer,text)', 'FUNCTION', NULL), + ('msar', 'msar.build_related_column_query(oid,jsonb,integer,text,text)', 'FUNCTION', NULL), + ('msar', 'msar.build_related_columns_aggregation_query(jsonb,text)', 'FUNCTION', NULL), + ('msar', 'msar.build_related_columns_ctes(oid,jsonb,text)', 'FUNCTION', NULL), + ('msar', 'msar.build_related_columns_join_expr(oid,text,jsonb)', 'FUNCTION', NULL), + ('msar', 'msar.list_records_from_table(oid,integer,integer,jsonb,jsonb,jsonb,boolean,jsonb,jsonb)', 'FUNCTION', NULL), + ('msar', 'msar.search_records_from_table(oid,jsonb,integer,integer,boolean,jsonb,jsonb)', 'FUNCTION', NULL), + ('msar', 'msar.split_ctes(text)', 'FUNCTION', NULL); ALTER TABLE ONLY msar.all_mathesar_objects ADD CONSTRAINT all_mathesar_objects_obj_schema_obj_name_obj_kind_key UNIQUE (obj_schema, obj_name, obj_kind); @@ -1234,4 +1241,3 @@ ALTER TABLE ONLY msar.all_mathesar_objects -- -- PostgreSQL database dump complete -- - diff --git a/db/sql/05_msar.sql b/db/sql/05_msar.sql index c379a602c5..d198ebc0f3 100644 --- a/db/sql/05_msar.sql +++ b/db/sql/05_msar.sql @@ -954,7 +954,7 @@ CREATE OR REPLACE FUNCTION msar.column_info_table(tab_id regclass) RETURNS TABLE type text, -- The type of the column for the table. type_options jsonb, -- type_options for the column(if any). nullable boolean, -- is the column nullable. - primary_key boolean, -- whether the column has primary key constraint. + primary_key boolean, -- whether the column has primary key constraint. "default" jsonb, -- the default for the column(if any). has_dependents boolean, -- is the column referenced by others. description text, -- The description of the column on the database. @@ -2488,7 +2488,7 @@ __msar.process_pk_col_def( -- The below tuple(s) defines a default 'id' column for Mathesar. It can have a given name, type -- integer or uuid, it's not null, it uses the 'identity' or 'gen_random_uuid()' functionality to -- generate default values, has a default comment. - SELECT CASE pkey_type + SELECT CASE pkey_type WHEN 'IDENTITY' THEN ARRAY[ (col_name, 'integer', true, null, pkey_type, 'Mathesar default integer ID column') @@ -3260,7 +3260,7 @@ BEGIN END IF; IF jsonb_path_exists(col_defs, '$[*] ? (@.name == "id")') THEN - -- rename 'id' + -- rename 'id' SELECT array_agg(col_def->>'name') INTO existing_col_names FROM jsonb_array_elements(col_defs) col_def; id_col_name := msar.get_unique_local_identifier(existing_col_names, 'id'); @@ -3875,15 +3875,15 @@ BEGIN -- set new default PERFORM msar.set_col_default(tab_id, col.attnum, col.new_default #>> '{}'); ELSEIF (col.new_default IS NULL OR jsonb_typeof(col.new_default)<>'null') AND col.new_type IS NOT NULL THEN - -- preserve old default + -- preserve old default -- when a new_default is absent and col is retyped with a new_type. -- Note: We don't want to preserve old default for jsonb_typeof(col.new_default)='null' - -- as we consider it as an intent to drop the default. + -- as we consider it as an intent to drop the default. PERFORM msar.set_old_col_default(tab_id, col.attnum, col.old_default, col.new_type, is_default_dynamic, col.cast_options); END IF; -- PG13 doesn't allow concat b/w integer[] and smallint need to typecast - return_attnum_arr := return_attnum_arr || col.attnum::integer; + return_attnum_arr := return_attnum_arr || col.attnum::integer; END LOOP; RETURN return_attnum_arr; -- do we really need this?? END; @@ -4554,16 +4554,23 @@ Build a deterministic order expression for the given table and order JSON. Args: tab_id: The OID of the table whose columns we'll order by. order_: A JSONB array defining any desired ordering of columns. + Note: This function only handles regular columns (numeric attnums). + For Related column sorting, use build_related_column_order_expr. */ SELECT string_agg(format('%I %s', attnum, msar.sanitize_direction(direction)), ', ') -FROM jsonb_to_recordset( +FROM ( + SELECT + (entry->>'attnum')::smallint AS attnum, + entry->>'direction' AS direction + FROM jsonb_array_elements( COALESCE( COALESCE(order_, '[]'::jsonb) || msar.get_pkey_order(tab_id), COALESCE(order_, '[]'::jsonb) || msar.get_total_order(tab_id) ) -) - AS x(attnum smallint, direction text) -WHERE has_column_privilege(tab_id, attnum, 'SELECT'); + ) AS entry + WHERE jsonb_typeof(entry->'attnum') = 'number' +) AS x +WHERE has_column_privilege(tab_id, x.attnum, 'SELECT'); $$ LANGUAGE SQL STABLE; @@ -4576,6 +4583,9 @@ with `msar.build_selectable_column_expr`. It will only use the columns to which Finally, this function will append either a primary key, or all columns to the produced ORDER BY so the resulting ordering is totally defined (i.e., deterministic). +Note: This function only handles regular columns (numeric attnums). +For Related column sorting, the ORDER BY must be built separately and combined. + Args: tab_id: The OID of the table whose columns we'll order by. order_: A JSONB array defining any desired ordering of columns. @@ -4719,7 +4729,7 @@ CREATE OR REPLACE FUNCTION msar.build_groups_cte_expr(tab_id oid, eq_cte_name text, ranked_cte_name text, group_ jsonb) RETURNS TEXT AS $$/* */ SELECT format( - $gj$ + $gj$SELECT %1$I.__mathesar_gid AS id, __mathesar_gcount AS count, to_jsonb(%1$I) - '__mathesar_gid' AS results_eq, @@ -4771,7 +4781,7 @@ this function will return a jsonb as follows: Args: tab_id: The OID of the table containing the columns to select. */ -SELECT coalesce(jsonb_object_agg(attnum, attname), '{}'::jsonb) +SELECT coalesce(jsonb_object_agg(attnum, attname ORDER BY attnum), '{}'::jsonb) FROM pg_catalog.pg_attribute WHERE attrelid = tab_id @@ -4792,7 +4802,7 @@ Args: { "2": , "4": } */ -SELECT string_agg(format('msar.format_data(%I) AS %I', sel_column.value, sel_column.key), ', ') +SELECT string_agg(format('msar.format_data(%I) AS %I', sel_column.value, sel_column.key), ', ' ORDER BY sel_column.key::integer) FROM jsonb_each_text(columns) as sel_column; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; @@ -5115,9 +5125,7 @@ Args: */ WITH fkey_map_cte AS (SELECT * FROM msar.get_fkey_map_table(tab_id)) SELECT concat( - format(E'\nLEFT JOIN summary_cte_self ON %1$I.', cte_name) - || quote_ident(msar.get_selectable_pkey_attnum(tab_id)::text) - || ' = summary_cte_self.key' , + format(E'\nLEFT JOIN summary_cte_self ON %1$I.pk = summary_cte_self.key', cte_name), string_agg( format( $j$ @@ -5170,6 +5178,592 @@ END; $$ LANGUAGE SQL STABLE RETURNS NULL ON NULL INPUT; +CREATE OR REPLACE FUNCTION +msar.build_related_column_id(join_path jsonb, column_attnum integer) RETURNS text AS $$/* +Build a unique identifier for a related column based on its join path and column attnum. + +Uses MD5 hash of the join path to create a safe SQL identifier (avoids JSON characters in identifiers). +MD5 is used because it's built-in to PostgreSQL and doesn't require the pgcrypto extension. + +The join_path is normalized to compact JSON (no spaces) to match frontend hash generation. + +Args: + join_path: A JSONB array representing the join path: [[[table_oid, attnum], [table_oid, attnum]], ...] + column_attnum: The attnum of the target column in the final table. +*/ +SELECT format('related_%s_%s', md5(replace(join_path::text, ' ', '')), column_attnum); +$$ LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.path_hash(join_path jsonb) RETURNS text AS $$/* +Returns first 8 hex characters of MD5 hash of join_path for path grouping. + +This is used to group related columns that share the same join path, allowing +one CTE to serve multiple related columns with different aggregations. + +Uses MD5 (built-in, no extensions required) instead of SHA256. + +Args: + join_path: A JSONB array representing the join path: [[[table_oid, attnum], [table_oid, attnum]], ...] +*/ +SELECT substr(md5(replace(join_path::text, ' ', '')), 1, 8); +$$ LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.related_alias(h8 text) RETURNS text AS $$/* +Returns CTE alias for a related column path group. + +Args: + h8: First 8 hex characters of path hash. +*/ +SELECT 'rc_' || h8; +$$ LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.value_col_name(attnum integer, agg text) RETURNS text AS $$/* +Returns value column name for a related column aggregation. + +Args: + attnum: The column attnum. + agg: The aggregation type (e.g., 'sum', 'count', 'list'). +*/ +SELECT format('v_%s_%s', attnum, lower(coalesce(agg, ''))); +$$ LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT; + + +CREATE OR REPLACE FUNCTION +msar.build_related_column_query( + base_table_oid oid, + join_path jsonb, + target_column_attnum integer, + aggregation text, + ids_cte_name text DEFAULT NULL +) RETURNS text AS $$/* +Build a SQL query that retrieves related column values following a join path. + +Args: + base_table_oid: The OID of the base table to start from. + join_path: A JSONB array representing the join path: [[[table_oid, attnum], [table_oid, attnum]], ...] + target_column_attnum: The attnum of the target column in the final table. + aggregation: The aggregation type: 'list', 'join', 'sum', 'count', or NULL for single value. + ids_cte_name: Optional name of a CTE containing base_pk values to scope the query to (e.g., 'page_ids'). +*/ +DECLARE + query_parts text[]; + base_alias text := 'base'; + current_table_oid oid; + current_table_schema text; + current_table_name text; + join_step jsonb; + left_table_oid oid; + left_attnum integer; + left_attname text; + right_table_oid oid; + right_attnum integer; + right_attname text; + alias_counter integer := 0; + current_alias text; + next_alias text; + target_table_oid oid; + target_table_schema text; + target_table_name text; + target_column_name text; + target_pk_attnum integer; + target_pk_attname text; + base_pk_attnum integer; + base_pk_attname text; + ids_expr text; + values_expr text; + inner_query text; + outer_query text; +BEGIN + -- Get base table info + current_table_oid := base_table_oid; + SELECT msar.get_relation_schema_name(current_table_oid), + msar.get_relation_name(current_table_oid) + INTO current_table_schema, current_table_name; + + base_pk_attnum := msar.get_selectable_pkey_attnum(base_table_oid); + + -- Get base PK column name + SELECT attname INTO base_pk_attname + FROM pg_attribute + WHERE attrelid = base_table_oid + AND attnum = base_pk_attnum + AND NOT attisdropped; + + current_alias := base_alias; + + -- Build initial FROM clause with optional JOIN to ids_cte if provided + IF ids_cte_name IS NOT NULL THEN + query_parts := array_append(query_parts, format( + 'FROM %I.%I AS %I JOIN %I ON %I.base_pk = %I.%I', + current_table_schema, + current_table_name, + current_alias, + ids_cte_name, + ids_cte_name, + current_alias, + base_pk_attname + )); + ELSE + query_parts := array_append(query_parts, format( + 'FROM %I.%I AS %I', + current_table_schema, + current_table_name, + current_alias + )); + END IF; + + -- Process each join step + FOR join_step IN SELECT jsonb_array_elements(join_path) + LOOP + left_table_oid := (join_step -> 0 ->> 0)::bigint::oid; + left_attnum := (join_step -> 0 ->> 1)::integer; + right_table_oid := (join_step -> 1 ->> 0)::bigint::oid; + right_attnum := (join_step -> 1 ->> 1)::integer; + + -- Get actual column names for join conditions + SELECT attname INTO left_attname + FROM pg_attribute + WHERE attrelid = left_table_oid + AND attnum = left_attnum + AND NOT attisdropped; + + SELECT attname INTO right_attname + FROM pg_attribute + WHERE attrelid = right_table_oid + AND attnum = right_attnum + AND NOT attisdropped; + + -- Determine which table we're joining to + IF current_table_oid = left_table_oid THEN + -- Joining from left to right + current_table_oid := right_table_oid; + alias_counter := alias_counter + 1; + next_alias := format('t%s', alias_counter); + + SELECT msar.get_relation_schema_name(current_table_oid), + msar.get_relation_name(current_table_oid) + INTO current_table_schema, current_table_name; + + query_parts := array_append(query_parts, format( + 'LEFT JOIN %I.%I AS %I ON %I.%I = %I.%I', + current_table_schema, + current_table_name, + next_alias, + current_alias, + left_attname, + next_alias, + right_attname + )); + + current_alias := next_alias; + ELSIF current_table_oid = right_table_oid THEN + -- Joining from right to left (reverse direction) + current_table_oid := left_table_oid; + alias_counter := alias_counter + 1; + next_alias := format('t%s', alias_counter); + + SELECT msar.get_relation_schema_name(current_table_oid), + msar.get_relation_name(current_table_oid) + INTO current_table_schema, current_table_name; + + query_parts := array_append(query_parts, format( + 'LEFT JOIN %I.%I AS %I ON %I.%I = %I.%I', + current_table_schema, + current_table_name, + next_alias, + current_alias, + right_attname, + next_alias, + left_attname + )); + + current_alias := next_alias; + ELSE + -- This shouldn't happen if join_path is valid, but handle gracefully + RAISE EXCEPTION 'Invalid join path: current table % does not match join step', current_table_oid; + END IF; + END LOOP; + + -- Get target table info and column names + target_table_oid := current_table_oid; + SELECT msar.get_relation_schema_name(target_table_oid), + msar.get_relation_name(target_table_oid), + attname + INTO target_table_schema, target_table_name, target_column_name + FROM pg_attribute + WHERE attrelid = target_table_oid + AND attnum = target_column_attnum + AND NOT attisdropped; + + -- Get target table PK column name + target_pk_attnum := msar.get_selectable_pkey_attnum(target_table_oid); + SELECT attname INTO target_pk_attname + FROM pg_attribute + WHERE attrelid = target_table_oid + AND attnum = target_pk_attnum + AND NOT attisdropped; + + -- Build the inner query (FROM/JOINs) as a string + inner_query := array_to_string(query_parts, E'\n'); + + -- Validate aggregation type (only 'list' and 'count' are supported in this simplified version) + IF aggregation IS NOT NULL AND aggregation NOT IN ('list', 'count') THEN + RAISE EXCEPTION 'Unsupported aggregation type: %. Only ''list'' and ''count'' are supported.', aggregation; + END IF; + + -- Build aggregation expressions with DISTINCT moved outside aggregates + -- Each CTE must expose a typed 'value' column for sorting (numeric/date/text/etc.) + IF aggregation = 'list' THEN + -- For 'list', aggregate as array of {id, value} objects + -- Limit to 25 items per base_pk for performance + -- Add a typed value column (NULL for list aggregation - not sortable) + -- Use window function for limiting, then aggregate + outer_query := format( + $q$SELECT base_pk, + jsonb_agg(jsonb_build_object('id', id, 'value', val) ORDER BY id) AS items, + NULL::text AS value + FROM ( + SELECT base_pk, id, val + FROM ( + SELECT base_pk, + id, + val, + row_number() OVER (PARTITION BY base_pk ORDER BY id) AS rn + FROM ( + SELECT %I.%I AS base_pk, + %I.%I AS id, + %I.%I AS val + %s + WHERE %I.%I IS NOT NULL + ) distinct_rows + ) numbered + WHERE rn <= 25 + ) d + GROUP BY base_pk$q$, + base_alias, + base_pk_attname, + current_alias, target_pk_attname, + current_alias, target_column_name, + inner_query, + current_alias, target_pk_attname + ); + ELSIF aggregation = 'count' THEN + -- For 'count', return typed integer value for sorting + -- Also return JSONB structure for frontend display (via aggregation query) + outer_query := format( + $q$SELECT base_pk, + COUNT(*)::bigint AS value, + jsonb_build_object( + 'count', COUNT(*), + 'ids', jsonb_agg(id ORDER BY id) + ) AS value_jsonb + FROM ( + SELECT %I.%I AS base_pk, + %I.%I AS id + %s + WHERE %I.%I IS NOT NULL + ) d + GROUP BY base_pk$q$, + base_alias, + base_pk_attname, + current_alias, target_pk_attname, + inner_query, + current_alias, target_pk_attname + ); + ELSE + -- Single value (no aggregation) - return single value directly + outer_query := format( + $q$SELECT %I.%I AS base_pk, MIN(%I.%I) AS value + %s + GROUP BY %I.%I$q$, + base_alias, + base_pk_attname, + current_alias, target_column_name, + inner_query, + base_alias, + base_pk_attname + ); + END IF; + + RETURN outer_query; +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION +msar.build_related_columns_ctes( + base_table_oid oid, + related_columns jsonb, + ids_cte_name text DEFAULT NULL +) RETURNS text AS $$/* +Build SQL text defining CTEs for related column queries. + +Args: + base_table_oid: The OID of the base table. + related_columns: A JSONB array of related column requests, each with: + - join_path: [[[table_oid, attnum], [table_oid, attnum]], ...] + - column_attnum: integer + - aggregation: 'list' or 'count' (only these two are supported) + ids_cte_name: Optional name of a CTE containing base_pk values to scope queries to (e.g., 'page_ids'). +*/ +DECLARE + cte_parts text[]; + rel_col jsonb; + rel_col_id text; + rel_col_query text; +BEGIN + IF related_columns IS NULL OR jsonb_array_length(related_columns) = 0 THEN + RETURN ''; + END IF; + + FOR rel_col IN SELECT jsonb_array_elements(related_columns) + LOOP + rel_col_id := msar.build_related_column_id( + rel_col -> 'join_path', + (rel_col -> 'column_attnum')::integer + ); + + rel_col_query := msar.build_related_column_query( + base_table_oid, + rel_col -> 'join_path', + (rel_col -> 'column_attnum')::integer, + NULLIF(rel_col ->> 'aggregation', '')::text, + ids_cte_name + ); + + cte_parts := array_append(cte_parts, format('%I AS (%s)', rel_col_id, rel_col_query)); +END LOOP; + + RETURN ', ' || array_to_string(cte_parts, ', '); +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION +msar.build_related_columns_join_expr( + base_table_oid oid, + base_cte_name text, + related_columns jsonb +) RETURNS text AS $$/* +Build SQL expressions to LEFT JOIN related column CTEs to the main CTE. + +Args: + base_table_oid: The OID of the base table. + base_cte_name: The name of the main CTE to join to. + related_columns: A JSONB array of related column requests. +*/ +DECLARE + join_parts text[]; + rel_col jsonb; + rel_col_id text; + base_pk_attnum integer; + base_pk_attname text; +BEGIN + IF related_columns IS NULL OR jsonb_array_length(related_columns) = 0 THEN + RETURN ''; + END IF; + + base_pk_attnum := msar.get_selectable_pkey_attnum(base_table_oid); + + -- Get base PK column name + SELECT attname INTO base_pk_attname + FROM pg_attribute + WHERE attrelid = base_table_oid + AND attnum = base_pk_attnum + AND NOT attisdropped; + + FOR rel_col IN SELECT jsonb_array_elements(related_columns) + LOOP + rel_col_id := msar.build_related_column_id( + rel_col -> 'join_path', + (rel_col -> 'column_attnum')::integer + ); + + join_parts := array_append(join_parts, format(E' +LEFT JOIN %I ON %I.%I = %I.base_pk', rel_col_id, base_cte_name, base_pk_attname, rel_col_id)); + END LOOP; + + RETURN array_to_string(join_parts, ''); +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION +msar.build_related_columns_aggregation_query( + related_columns jsonb, + page_ids_cte_name text DEFAULT NULL +) RETURNS text AS $$/* +Build a SQL query that aggregates all related column CTEs into the final JSON structure. + +Args: + related_columns: A JSONB array of related column requests. + page_ids_cte_name: Optional name of a CTE containing base_pk values to ensure all are included + (e.g., 'page_ids'). If provided, LEFT JOINs with this CTE to include + base_pk values with 0 related rows. +*/ +DECLARE + union_parts text[]; + rel_col jsonb; + rel_col_id text; + cte_alias text; + join_expr text; +BEGIN + IF related_columns IS NULL OR jsonb_array_length(related_columns) = 0 THEN + RETURN 'SELECT NULL::text AS key, NULL::jsonb AS value WHERE false'; + END IF; + + FOR rel_col IN SELECT jsonb_array_elements(related_columns) + LOOP + rel_col_id := msar.build_related_column_id( + rel_col -> 'join_path', + (rel_col -> 'column_attnum')::integer + ); + cte_alias := rel_col_id; -- Use rel_col_id directly as the CTE alias + + -- If page_ids_cte_name is provided, LEFT JOIN with it to ensure all base_pk values are included + -- This ensures that base_pk values with 0 related rows still appear in the result with NULL/0 values + -- Note: page_ids.base_pk is text (formatted column value), so we need to cast for comparison + IF page_ids_cte_name IS NOT NULL THEN + join_expr := format(' FROM %I LEFT JOIN %I ON %I.base_pk::text = %I.base_pk::text', + page_ids_cte_name, cte_alias, page_ids_cte_name, cte_alias); + ELSE + join_expr := format(' FROM %I', cte_alias); + END IF; + + -- Check aggregation type to determine which column to use + -- For 'list' aggregation, use 'items' column (array of {id, value} objects) + -- For 'count', use 'value_jsonb' column (JSONB structure for frontend) + -- Only 'list' and 'count' are supported in this simplified version + IF (rel_col -> 'aggregation')::text = '"list"' THEN + IF page_ids_cte_name IS NOT NULL THEN + -- When joining with page_ids, use COALESCE to provide default empty array for NULL values + union_parts := array_append(union_parts, format( + $q$SELECT %L::text AS key, + jsonb_object_agg(%I.base_pk::text, COALESCE(%I.items, '[]'::jsonb)) AS value + %s$q$, + rel_col_id, + page_ids_cte_name, cte_alias, + join_expr + )); + ELSE + union_parts := array_append(union_parts, format( + $q$SELECT %L::text AS key, + jsonb_object_agg(base_pk::text, COALESCE(items, '[]'::jsonb)) AS value + %s$q$, + rel_col_id, + join_expr + )); + END IF; + ELSIF (rel_col -> 'aggregation')::text = '"count"' THEN + IF page_ids_cte_name IS NOT NULL THEN + -- When joining with page_ids, use COALESCE to provide default {count: 0, ids: []} for NULL values + union_parts := array_append(union_parts, format( + $q$SELECT %L::text AS key, + jsonb_object_agg(%I.base_pk::text, COALESCE(%I.value_jsonb, '{"count": 0, "ids": []}'::jsonb)) AS value + %s$q$, + rel_col_id, + page_ids_cte_name, cte_alias, + join_expr + )); + ELSE + union_parts := array_append(union_parts, format( + $q$SELECT %L::text AS key, + jsonb_object_agg(base_pk::text, value_jsonb) AS value + %s$q$, + rel_col_id, + join_expr + )); + END IF; + ELSE + IF page_ids_cte_name IS NOT NULL THEN + union_parts := array_append(union_parts, format( + $q$SELECT %L::text AS key, + jsonb_object_agg(%I.base_pk::text, %I.value) AS value + %s$q$, + rel_col_id, + page_ids_cte_name, cte_alias, + join_expr + )); + ELSE + union_parts := array_append(union_parts, format( + $q$SELECT %L::text AS key, + jsonb_object_agg(base_pk::text, value) AS value + %s$q$, + rel_col_id, + join_expr + )); + END IF; + END IF; + END LOOP; + + RETURN array_to_string(union_parts, ' UNION ALL '); +END; +$$ LANGUAGE plpgsql STABLE; + + +CREATE OR REPLACE FUNCTION msar.split_ctes(cte_block text) +RETURNS text[] LANGUAGE plpgsql IMMUTABLE AS $$/* +Splits a comma-separated string of CTE definitions, respecting parentheses depth. +Used to safely parse multi-CTE blocks returned by helper functions. + +Expected input format: + ', cte1 AS (...), cte2 AS (...), cte3 AS (...)' + or + 'cte1 AS (...), cte2 AS (...), cte3 AS (...)' + +The function handles: + - Leading comma (common in helper function outputs) + - Nested parentheses in CTE definitions + - Empty strings (returns empty array) + - Single CTE definitions + +Returns an array of CTE definition strings, each in the form 'cte_name AS (...)'. +*/ +DECLARE + result text[]; + start_ int := 1; + depth int := 0; + i int; + current char; + piece text; +BEGIN + cte_block := btrim(coalesce(cte_block, '')); + IF cte_block = '' THEN + RETURN ARRAY[]::text[]; + END IF; + + FOR i IN 1..length(cte_block) LOOP + current := substring(cte_block, i, 1); + IF current = '(' THEN + depth := depth + 1; + ELSIF current = ')' THEN + depth := depth - 1; + END IF; + + IF (current = ',' AND depth = 0) OR i = length(cte_block) THEN + -- When at the end, include the current character; when at comma, include everything up to (but not including) the comma + -- This means we want length = i - start_ (to include the character at position i-1, which is before the comma) + piece := trim(substring(cte_block, start_, i - start_ + (CASE WHEN i = length(cte_block) THEN 1 ELSE 0 END))); + IF piece <> '' THEN + result := array_append(result, piece); + END IF; + start_ := i + 1; + END IF; + END LOOP; + + RETURN result; +END; +$$; + + + + CREATE OR REPLACE FUNCTION msar.build_record_list_query_components_with_ctes( tab_id oid, limit_ integer, @@ -5205,6 +5799,9 @@ DECLARE BEGIN SELECT msar.get_selectable_columns(tab_id) INTO selectable_columns; + -- Note: filter_ should only contain base column filters (no related_* attnums) + -- Related column filtering is not supported in this simplified version + SELECT jsonb_build_object( 'relation_name', msar.get_relation_name(tab_id), 'relation_schema_name', msar.get_relation_schema_name(tab_id), @@ -5251,123 +5848,246 @@ msar.list_records_from_table( filter_ jsonb, group_ jsonb, return_record_summaries boolean DEFAULT false, - table_record_summary_templates jsonb DEFAULT NULL -) RETURNS jsonb AS $$/* + table_record_summary_templates jsonb DEFAULT NULL, + related_columns jsonb DEFAULT NULL +) RETURNS jsonb AS $$ +/* Get records from a table. Only columns to which the user has access are returned. Args: - tab_id: The OID of the table whose records we'll get - limit_: The maximum number of rows we'll return - offset_: The number of rows to skip before returning records from following rows. - order_: An array of ordering definition objects. - filter_: An array of filter definition objects. - group_: An array of group definition objects. - return_record_summaries : Whether to return a summary for each record listed. - table_record_summary_templates: (optional) A JSON object that maps table OIDs to record summary - templates. + tab_id: OID of the base table. + limit_: Maximum number of rows to return (NULL = no limit). + offset_: Number of rows to skip before returning. + order_: JSONB array of order objects: [{"attnum": , "direction": "asc"|"desc"}]. Base columns only. + filter_: JSONB array of filter objects (schema defined by msar.build_* utilities). Base columns only. + group_: JSONB array of group-by definitions (schema defined by msar.build_* utilities). + return_record_summaries: If TRUE, include a per-record self summary. + table_record_summary_templates: Optional JSON mapping { :