From c0fd31d866236b150addfae95cccae71d079a2c6 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 1 Jul 2026 17:55:02 -0700 Subject: [PATCH 1/2] Add semantic column-type inference + override UI to Data Overview Prototype: classify each column as continuous / categorical / identifier by dtype + cardinality (+ id-name hint) instead of raw dtype, so numeric codes (e.g. class labels) and identifiers (e.g. unique keys) aren't summarized as continuous measurements. - infer_column_roles(df, overrides) helper in web/routes/utils.py - /summary-statistics: describe() over continuous only; value-count bars for categoricals; identifiers listed and excluded from stats/plots - /column-roles endpoint persists per-session overrides; roles reset on new upload; overrides bust cached metric results - Data Overview 'Override Column Types' panel: per-column role dropdowns with search + 14-row cap for wide datasets, Apply recomputes - unit tests for the inference heuristic (incl. small-dataset guards) Web-only for now; CLI/Globus/library unaffected (see follow-ups). --- tests/unit/test_column_roles.py | 88 ++++++++ web/routes/core.py | 90 +++++--- web/routes/utils.py | 152 ++++++++++++- web/static/css/inspector.css | 11 + web/static/js/inspector.js | 367 ++++++++++++++++++++++---------- 5 files changed, 562 insertions(+), 146 deletions(-) create mode 100644 tests/unit/test_column_roles.py diff --git a/tests/unit/test_column_roles.py b/tests/unit/test_column_roles.py new file mode 100644 index 00000000..d2d9c0ed --- /dev/null +++ b/tests/unit/test_column_roles.py @@ -0,0 +1,88 @@ +"""Tests for semantic column-role inference (continuous / categorical / identifier).""" + +import numpy as np +import pandas as pd + +from web.routes.utils import infer_column_roles + + +def _df(n=1000): + rng = np.random.default_rng(42) + return pd.DataFrame( + { + # genuine continuous float measures + "pt": rng.normal(50, 10, n).astype("float32"), + "eta": rng.normal(0, 2, n).astype("float32"), + # low-cardinality integer code -> categorical + "flavor": rng.choice([-1, 1, 2, 5, 21], n).astype("int32"), + # near-unique integer -> identifier (by cardinality) + "jetIndex": np.arange(n, dtype="uint64"), + # moderate-cardinality integer with id-like name -> identifier (by name) + "eventIndex": rng.integers(0, n // 5, n).astype("uint64"), + # genuine string column -> categorical + "label": rng.choice(["a", "b", "c"], n), + } + ) + + +def test_continuous_floats(): + roles = infer_column_roles(_df()) + assert roles["pt"] == "continuous" + assert roles["eta"] == "continuous" + + +def test_low_cardinality_int_is_categorical(): + roles = infer_column_roles(_df()) + assert roles["flavor"] == "categorical" + + +def test_near_unique_int_is_identifier(): + roles = infer_column_roles(_df()) + assert roles["jetIndex"] == "identifier" + + +def test_idlike_name_with_moderate_cardinality_is_identifier(): + roles = infer_column_roles(_df()) + assert roles["eventIndex"] == "identifier" + + +def test_string_column_is_categorical(): + roles = infer_column_roles(_df()) + assert roles["label"] == "categorical" + + +def test_user_override_wins(): + roles = infer_column_roles(_df(), overrides={"flavor": "continuous", "pt": "categorical"}) + assert roles["flavor"] == "continuous" + assert roles["pt"] == "categorical" + + +def test_invalid_override_ignored(): + roles = infer_column_roles(_df(), overrides={"flavor": "bogus", "missing": "categorical"}) + assert roles["flavor"] == "categorical" # heuristic retained + assert "missing" not in roles + + +def test_bool_column_is_categorical(): + df = pd.DataFrame({"flag": [True, False, True, False] * 10}) + assert infer_column_roles(df)["flag"] == "categorical" + + +def test_empty_dataframe(): + assert infer_column_roles(pd.DataFrame()) == {} + + +def test_small_dataset_all_distinct_int_is_continuous(): + # In a tiny dataset every value is trivially distinct — a numeric column + # must not be mistaken for an identifier or (via low nunique) a category. + df = pd.DataFrame({"age": [25, 30, 35], "active": [True, False, True]}) + roles = infer_column_roles(df) + assert roles["age"] == "continuous" + assert roles["active"] == "categorical" + + +def test_identifier_requires_enough_distinct_values(): + # An id-like name with only a handful of distinct values in a small dataset + # should not be forced to "identifier". + df = pd.DataFrame({"user_id": [1, 2, 3, 4, 5]}) + assert infer_column_roles(df)["user_id"] != "identifier" diff --git a/web/routes/core.py b/web/routes/core.py index 7fb624b9..48632eae 100644 --- a/web/routes/core.py +++ b/web/routes/core.py @@ -18,9 +18,11 @@ from werkzeug.utils import secure_filename from aidrin.file_handling.file_parser import SUPPORTED_FILE_TYPES, READER_MAP from web.routes.utils import ( + categorical_bars, clear_all_user_cache, ensure_json_serializable, get_current_user_id, + infer_column_roles, load_dataframe, summary_histograms, ) @@ -56,6 +58,8 @@ def inspector(): session["uploaded_file_name"] = display_name session["uploaded_file_path"] = file_path session["uploaded_file_type"] = request.form.get("fileTypeSelector") + # Column-role overrides are per-dataset; drop any from a prior file. + session.pop("column_roles", None) # Track files this session created so /clear removes only these, # never files belonging to other concurrent sessions. @@ -349,11 +353,19 @@ def summary_statistics(): if load_error: return jsonify({"success": False, "message": load_error}), 200 - # Restrict to numeric columns: describe() would otherwise fall back to - # object-column stats (top/freq are strings) and the numeric formatter - # below would fail on them. Datasets with no numerical features simply - # get an empty numerical summary (issue #125). - numeric_df = df.select_dtypes(include="number") + # Classify columns by *semantic role* rather than raw dtype, so numeric + # codes (e.g. a class label) and identifiers (e.g. a near-unique index) + # are not summarized as if they were continuous measurements. User + # overrides stored in the session take precedence over the heuristic. + role_overrides = session.get("column_roles", {}) + column_roles = infer_column_roles(df, role_overrides) + continuous_columns = [c for c in df.columns if column_roles[c] == "continuous"] + categorical_columns = [c for c in df.columns if column_roles[c] == "categorical"] + identifier_columns = [c for c in df.columns if column_roles[c] == "identifier"] + + # describe() runs only on continuous columns. Datasets with no continuous + # features simply get an empty numerical summary (issue #125). + numeric_df = df[continuous_columns] if numeric_df.shape[1] == 0: summary_statistics = {} else: @@ -361,20 +373,10 @@ def summary_statistics(): lambda x: round(x, 2) if x == 0 or abs(x) >= 0.001 else f"{x:.2e}" ).to_dict() - histograms = summary_histograms(df) - - # Booleans are treated as categorical (they're excluded from describe() - # and select_dtypes("number") above, so they belong with the categorical - # summary, not the numerical one). - numerical_columns = [ - col for col, dtype in df.dtypes.items() - if pd.api.types.is_numeric_dtype(dtype) and not pd.api.types.is_bool_dtype(dtype) - ] - categorical_columns = [ - col for col, dtype in df.dtypes.items() - if pd.api.types.is_string_dtype(dtype) or pd.api.types.is_bool_dtype(dtype) - ] - all_features = numerical_columns + categorical_columns + # KDE curves for continuous columns; value-count bars for categorical + # ones (a KDE of a discrete code is meaningless). Identifiers get neither. + histograms = summary_histograms(df, columns=continuous_columns) + categorical_histograms = categorical_bars(df, categorical_columns) for v in summary_statistics.values(): for old_key in list(v.keys()): @@ -382,10 +384,9 @@ def summary_statistics(): new_key = old_key.replace("%", "th percentile") v[new_key] = v.pop(old_key) - # Per-column summary for categorical features (describe() above only - # covers numerical columns). Reports non-null count, distinct values, - # the most frequent value, its frequency, and that frequency as a - # percentage of the non-null values. + # Per-column summary for categorical features (numeric codes included). + # Reports non-null count, distinct values, the most frequent value, its + # frequency, and that frequency as a percentage of the non-null values. categorical_summary = {} for col in categorical_columns: counts = df[col].value_counts(dropna=True) @@ -399,17 +400,32 @@ def summary_statistics(): "freq_pct": round(freq / count * 100, 1) if count else 0.0, } + # Compact per-identifier note (excluded from stats/plots). + identifier_summary = { + str(col): { + "count": int(df[col].notna().sum()), + "unique": int(df[col].nunique(dropna=True)), + } + for col in identifier_columns + } + + all_features = continuous_columns + categorical_columns + response_data = ensure_json_serializable({ "success": True, "message": "File uploaded successfully", "records_count": len(df), "features_count": len(df.columns), "categorical_features": list(categorical_columns), - "numerical_features": list(numerical_columns), + "numerical_features": list(continuous_columns), + "identifier_features": list(identifier_columns), "all_features": all_features, + "column_roles": {str(c): r for c, r in column_roles.items()}, "summary_statistics": summary_statistics, "categorical_summary": categorical_summary, + "identifier_summary": identifier_summary, "histograms": histograms, + "categorical_histograms": categorical_histograms, }) return jsonify(response_data) except Exception as e: @@ -417,6 +433,32 @@ def summary_statistics(): return jsonify({"success": False, "message": "An internal error occurred"}) +@core_bp.route("/column-roles", methods=["POST"]) +def set_column_roles(): + """Persist user overrides of inferred column roles for the current session. + + Accepts a JSON body ``{"column_roles": {col: role}}`` (or a bare + ``{col: role}`` map). The full override set replaces any previous one, so + sending ``{}`` resets to the inferred defaults. The Data Overview re-fetches + ``/summary-statistics`` afterwards to recompute with the new roles. + """ + try: + data = request.get_json(silent=True) or {} + overrides = data.get("column_roles", data) + if not isinstance(overrides, dict): + return jsonify({"success": False, "message": "Invalid payload"}), 200 + valid_roles = {"continuous", "categorical", "identifier"} + cleaned = {str(c): r for c, r in overrides.items() if r in valid_roles} + session["column_roles"] = cleaned + # Overrides change which columns each metric treats as numeric/categorical, + # so any cached metric results are now stale. + clear_all_user_cache() + return jsonify({"success": True, "column_roles": cleaned}) + except Exception as e: + file_upload_time_log.error("Error setting column roles: %s", e, exc_info=True) + return jsonify({"success": False, "message": "An internal error occurred"}), 200 + + @core_bp.route("/feature-set", methods=["POST"]) def extract_features(): try: diff --git a/web/routes/utils.py b/web/routes/utils.py index b6f0e0fb..9d4c1dc0 100644 --- a/web/routes/utils.py +++ b/web/routes/utils.py @@ -3,6 +3,7 @@ import io import base64 import logging +import re import time import uuid @@ -16,6 +17,86 @@ logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Semantic column-role inference +# --------------------------------------------------------------------------- +# +# Metrics historically keyed off the pandas dtype (``select_dtypes``), which +# mislabels numeric columns that are really categorical codes (e.g. a class +# label ``flavor`` with values -1/1/2/5/21) or identifiers (e.g. a near-unique +# ``jetIndex``). Those get meaningless means, KDE plots, and correlations. +# +# ``infer_column_roles`` assigns each column a *semantic role* — one of +# ``continuous``, ``categorical``, or ``identifier`` — from dtype + cardinality +# (+ a light column-name hint for ids). The result is a suggestion the user can +# override; ``overrides`` takes precedence over the heuristic. + +VALID_ROLES = ("continuous", "categorical", "identifier") + +# Cardinality thresholds (fractions are of the row count). +_ID_UNIQUE_RATIO = 0.98 # near-unique integer column -> identifier +_ID_NAME_MIN_RATIO = 0.10 # id-like name needs at least this spread to be an id +_ID_MIN_UNIQUE = 50 # ... and enough distinct values that near-uniqueness + # is meaningful (tiny datasets are trivially unique) +_CATEGORICAL_MAX_UNIQUE = 20 # at most this many distinct values ... +_CATEGORICAL_MAX_RATIO = 0.50 # ... AND distinct/rows below this (guards tiny + # datasets where every value is trivially distinct) + +_ID_NAME_TOKENS = {"id", "idx", "index", "key", "uid", "guid", "uuid", "pk"} + + +def _looks_like_id_name(name) -> bool: + """True when a column name's leading/trailing token looks like an identifier.""" + # Split camelCase then snake/space/dash: "eventIndex" -> ["event", "index"]. + spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", str(name)) + parts = [p.lower() for p in re.split(r"[_\s\-]+", spaced) if p] + if not parts: + return False + return parts[-1] in _ID_NAME_TOKENS or parts[0] in _ID_NAME_TOKENS + + +def infer_column_roles(df, overrides=None): + """Return ``{column: role}`` where role is continuous/categorical/identifier. + + ``overrides`` (``{column: role}``) wins over the heuristic; unknown columns + or invalid role strings in ``overrides`` are ignored. + """ + roles = {} + n = len(df) + for col in df.columns: + series = df[col] + dtype = series.dtype + + if not pd.api.types.is_numeric_dtype(dtype) or pd.api.types.is_bool_dtype(dtype): + roles[col] = "categorical" + continue + + nunique = int(series.nunique(dropna=True)) + ratio = nunique / n if n else 0.0 + is_int = pd.api.types.is_integer_dtype(dtype) + + if is_int and nunique >= _ID_MIN_UNIQUE and ratio >= _ID_UNIQUE_RATIO: + roles[col] = "identifier" + elif ( + is_int + and nunique >= _ID_MIN_UNIQUE + and _looks_like_id_name(col) + and ratio >= _ID_NAME_MIN_RATIO + ): + roles[col] = "identifier" + elif nunique <= _CATEGORICAL_MAX_UNIQUE and ratio <= _CATEGORICAL_MAX_RATIO: + roles[col] = "categorical" + else: + roles[col] = "continuous" + + if overrides: + for col, role in overrides.items(): + if col in roles and role in VALID_ROLES: + roles[col] = role + + return roles + + # --------------------------------------------------------------------------- # File loading # --------------------------------------------------------------------------- @@ -247,13 +328,31 @@ def ensure_json_serializable(obj): return obj -def summary_histograms(df): - """Generate base64-encoded KDE distribution plots for all numeric columns.""" +def _fig_to_base64(fig): + img_buffer = io.BytesIO() + fig.savefig(img_buffer, format="png", dpi=150, transparent=True) + img_buffer.seek(0) + encoded = base64.b64encode(img_buffer.read()).decode("utf-8") + plt.close(fig) + img_buffer.close() + return encoded + + +def summary_histograms(df, columns=None): + """Generate base64-encoded KDE distribution plots. + + ``columns`` restricts the plots to the given columns (used to plot only + columns whose semantic role is *continuous*); when ``None`` it falls back + to every numeric column for backward compatibility. + """ text_color = "#6b7280" curve_color = "#4485F4" + if columns is None: + columns = list(df.select_dtypes(include="number").columns) + line_graphs = {} - for column in df.select_dtypes(include="number").columns: + for column in columns: fig, ax = plt.subplots(figsize=(4, 3)) fig.patch.set_alpha(0) ax.set_facecolor("none") @@ -267,14 +366,45 @@ def summary_histograms(df): spine.set_color(text_color) fig.tight_layout(pad=0.5) - img_buffer = io.BytesIO() - fig.savefig(img_buffer, format="png", dpi=150, transparent=True) - img_buffer.seek(0) - encoded_img = base64.b64encode(img_buffer.read()).decode("utf-8") - # Store as _light for backward compat with JS picker - line_graphs[f"{column}_light"] = encoded_img - plt.close(fig) - img_buffer.close() + line_graphs[f"{column}_light"] = _fig_to_base64(fig) return line_graphs + + +def categorical_bars(df, columns, max_categories=20): + """Generate base64-encoded bar charts of value counts for categorical columns. + + KDE curves are meaningless for discrete/coded columns, so categorical + columns (including numeric codes like ``flavor``) get a value-count bar + chart instead. High-cardinality columns show only the top ``max_categories``. + """ + text_color = "#6b7280" + bar_color = "#4485F4" + + bars = {} + for column in columns: + counts = df[column].value_counts(dropna=True).head(max_categories) + if counts.empty: + continue + + fig, ax = plt.subplots(figsize=(4, 3)) + fig.patch.set_alpha(0) + ax.set_facecolor("none") + + ax.bar([str(i) for i in counts.index], counts.values, color=bar_color) + + ax.set_xlabel("Category", fontsize=10, color=text_color) + ax.set_ylabel("Count", fontsize=10, color=text_color) + ax.tick_params(colors=text_color, labelsize=8) + if len(counts) > 6: + for label in ax.get_xticklabels(): + label.set_rotation(45) + label.set_ha("right") + for spine in ax.spines.values(): + spine.set_color(text_color) + fig.tight_layout(pad=0.5) + + bars[f"{column}_light"] = _fig_to_base64(fig) + + return bars diff --git a/web/static/css/inspector.css b/web/static/css/inspector.css index 0ee0d18e..fe6e276f 100644 --- a/web/static/css/inspector.css +++ b/web/static/css/inspector.css @@ -31,6 +31,17 @@ html.dark .sidebar-metric-item.active { max-width: none; } +/* Column-roles editor: match the compact AI-settings control height + (px-3 py-2 text-sm) instead of the taller global select padding. */ +#roles-role-filter, +#column-roles-grid select { + padding: 8px 32px 8px 12px; + font-size: 0.875rem; + line-height: 1.25rem; + border-radius: 8px; + max-width: none; +} + /* Tables: outer border + header separator */ .relative.overflow-x-auto { border: 1px solid #e5e7eb; diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index b93a92cc..239f82de 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2113,42 +2113,279 @@ function buildCategoricalSummaryTable(summary) { // ==================== Histograms ==================== /** - * Render histogram images in the data overview panel. - * @param {Object} histograms - Dict of {column_theme: base64_img} from /summary-statistics + * Build one distribution section (heading + image grid). Returns "" when the + * histogram dict is empty so callers can concatenate sections unconditionally. + * @param {Object} histograms - Dict of {column_theme: base64_img} + * @param {string} title - Section heading */ -function renderWorkspaceHistograms(histograms) { - const container = document.getElementById("workspace-histograms"); - if (!container) return; - +function buildHistogramSection(histograms, title) { + if (!histograms) return ""; // Always use the light variant — CSS filter handles dark mode const columns = {}; for (const [key, base64] of Object.entries(histograms)) { if (key.endsWith("_light")) { - const colName = key.slice(0, -"_light".length); - columns[colName] = base64; + columns[key.slice(0, -"_light".length)] = base64; } } + if (Object.keys(columns).length === 0) return ""; - if (Object.keys(columns).length === 0) { - container.innerHTML = ""; - return; - } - - let html = - '

Numerical Feature Distributions

'; - html += '
'; - + let html = `

${title}

`; + html += '
'; for (const [colName, base64] of Object.entries(columns)) { html += `
Distribution of ${colName} -
${colName}
+
${escapeHtml(colName)}
`; } + html += "
"; + return html; +} + +/** + * Render distribution plots in the data overview panel: KDE curves for + * continuous columns and value-count bars for categorical columns. + * @param {Object} histograms - continuous KDE plots (`histograms`) + * @param {Object} [categoricalHistograms] - categorical bar charts + */ +function renderWorkspaceHistograms(histograms, categoricalHistograms) { + const container = document.getElementById("workspace-histograms"); + if (!container) return; + container.innerHTML = + buildHistogramSection(histograms, "Numerical Feature Distributions") + + buildHistogramSection(categoricalHistograms, "Categorical Feature Distributions"); +} + +/** + * Build the editable column-roles panel. Each column shows its inferred role + * (continuous/categorical/identifier) in a dropdown the user can change; the + * Apply button persists overrides and recomputes the overview. + * @param {Object} roles - {column: role} from /summary-statistics + */ +function buildColumnRolesEditor(roles) { + if (!roles || Object.keys(roles).length === 0) return ""; + const roleOptions = ["continuous", "categorical", "identifier"]; + const cols = Object.keys(roles); + + // Centered section title, consistent with the other Data Overview headings. + let html = + '

Override Column Types

'; + // Search + Apply (app-standard control sizing; left-aligned). + html += '
'; + html += + ''; + html += + ''; html += "
"; + + // Capped grid (no scroll — the 14-row limit keeps it compact at any width) + html += + '
'; + cols.forEach((col) => { + html += ` +
+ ${escapeHtml(col)} + +
`; + }); + // "More available" tile — occupies the 15th slot when matches exceed the cap. + html += + ''; + html += "
"; + html += + ''; + return html; +} + +// Show at most this many matching rows; the rest collapse into a "more" tile. +const ROLES_VISIBLE_LIMIT = 14; + +/** Live-filter the column-role rows by name substring and/or selected role. */ +function filterColumnRoles() { + const q = (document.getElementById("roles-search")?.value || "").toLowerCase(); + let matched = 0; + document.querySelectorAll("#column-roles-grid .role-row").forEach((row) => { + const col = (row.getAttribute("data-col") || "").toLowerCase(); + const nameMatch = !q || col.includes(q); + // Show only the first N matches; matches beyond the cap stay hidden. + const show = nameMatch && matched < ROLES_VISIBLE_LIMIT; + row.style.display = show ? "" : "none"; + if (nameMatch) matched++; + }); + + const overflow = Math.max(0, matched - ROLES_VISIBLE_LIMIT); + const more = document.getElementById("roles-more"); + if (more) { + more.textContent = `+${overflow} more — narrow with search`; + more.style.display = overflow > 0 ? "" : "none"; + } + const empty = document.getElementById("roles-empty"); + if (empty) { + empty.style.display = matched === 0 ? "" : "none"; + } +} + +/** Gather role overrides, persist them, and recompute the overview. */ +function applyColumnRoles() { + const overrides = {}; + document.querySelectorAll("#column-roles-grid .role-select").forEach((s) => { + overrides[s.getAttribute("data-col")] = s.value; + }); + const btn = document.getElementById("apply-roles-btn"); + if (btn) { + btn.disabled = true; + btn.textContent = "Applying…"; + } + fetch("/column-roles", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ column_roles: overrides }), + }) + .then((r) => r.json()) + .then(() => refreshWorkspaceSummary()) + .catch((err) => { + if (btn) { + btn.disabled = false; + btn.textContent = "Apply"; + } + console.error("Error applying column roles:", err); + }); +} + +/** Re-fetch /summary-statistics and re-render the overview (after a role change). */ +function refreshWorkspaceSummary() { + const container = document.getElementById("workspace-summary"); + if (container) + container.innerHTML = + '
Recomputing summary…
'; + const hist = document.getElementById("workspace-histograms"); + if (hist) hist.innerHTML = ""; + _beginServerProcessing(); + return fetch("/summary-statistics") + .then((r) => r.json()) + .then((data) => renderWorkspaceSummary(data)) + .catch((err) => { + if (container) + container.innerHTML = `

Error loading summary: ${err.message}

`; + }) + .finally(() => _endServerProcessing()); +} + +/** Render the whole Data Overview panel from a /summary-statistics payload. */ +function renderWorkspaceSummary(data) { + const container = document.getElementById("workspace-summary"); + if (!container) return; + + if (!data.success) { + container.innerHTML = ` +
+ + ${data.message || "Unable to load summary."} +
`; + return; + } + + const idCount = (data.identifier_features || []).length; + let html = ` +
+
+
${data.records_count.toLocaleString()}
+
Records
+
+
+
${data.features_count}
+
Features
+
+
+
${data.numerical_features?.length || 0}
+
Continuous
+
+
+
${data.categorical_features?.length || 0}
+
Categorical
+
+
+
${idCount}
+
Identifiers
+
+
+ `; + + html += buildColumnRolesEditor(data.column_roles); + + // Numerical (continuous) statistics table — rows = features, columns = stats + if ( + data.summary_statistics && + Object.keys(data.summary_statistics).length > 0 + ) { + const features = Object.keys(data.summary_statistics); + const allStats = Object.keys(data.summary_statistics[features[0]]); + html += + '

Numerical Features

'; + const preferredOrder = [ + "count", + "min", + "25th percentile", + "50th percentile", + "mean", + "75th percentile", + "max", + "std", + ]; + const statKeys = preferredOrder + .filter((s) => allStats.includes(s)) + .concat(allStats.filter((s) => !preferredOrder.includes(s))); + + html += '
'; + html += ''; + html += ''; + html += ''; + statKeys.forEach((s) => { + html += ``; + }); + html += ""; + features.forEach((feat, i) => { + const stripe = + i % 2 === 0 + ? "bg-white dark:bg-gray-800" + : "bg-gray-50 dark:bg-gray-700/50"; + html += ``; + html += ``; + statKeys.forEach((s) => { + html += ``; + }); + html += ""; + }); + html += "
Feature${s}
${escapeHtml(feat)}${data.summary_statistics[feat][s] ?? "—"}
"; + } + + html += buildCategoricalSummaryTable(data.categorical_summary); + + // Identifiers: excluded from stats/plots, listed for transparency + if (data.identifier_summary && Object.keys(data.identifier_summary).length > 0) { + html += + '

Identifiers

'; + html += + '

Treated as identifiers and excluded from statistics, distributions, and correlations.

'; + html += '
'; + Object.keys(data.identifier_summary).forEach((col) => { + const info = data.identifier_summary[col] || {}; + html += `${escapeHtml(col)}${(info.unique ?? 0).toLocaleString()} distinct`; + }); + html += "
"; + } + container.innerHTML = html; + + // Apply the default "Needs review" filter so continuous columns start hidden. + filterColumnRoles(); + + renderWorkspaceHistograms(data.histograms, data.categorical_histograms); } // ==================== Workspace Init ==================== @@ -2175,99 +2412,7 @@ function initWorkspace() { fetch("/summary-statistics") .then((r) => r.json()) - .then((data) => { - const container = document.getElementById("workspace-summary"); - if (!container) return; - - if (data.success) { - let html = ` -
-
-
${data.records_count.toLocaleString()}
-
Records
-
-
-
${data.features_count}
-
Features
-
-
-
${data.numerical_features?.length || 0}
-
Numerical
-
-
-
${data.categorical_features?.length || 0}
-
Categorical
-
-
- `; - - // Summary statistics table — pivoted: rows = features, columns = stats - if ( - data.summary_statistics && - Object.keys(data.summary_statistics).length > 0 - ) { - const features = Object.keys(data.summary_statistics); - const allStats = Object.keys(data.summary_statistics[features[0]]); - html += - '

Numerical Features

'; - // Preferred order - const preferredOrder = [ - "count", - "min", - "25th percentile", - "50th percentile", - "mean", - "75th percentile", - "max", - "std", - ]; - const statKeys = preferredOrder - .filter((s) => allStats.includes(s)) - .concat(allStats.filter((s) => !preferredOrder.includes(s))); - - html += '
'; - html += - ''; - html += - ''; - html += ''; - statKeys.forEach((s) => { - html += ``; - }); - html += ""; - - features.forEach((feat, i) => { - const stripe = - i % 2 === 0 - ? "bg-white dark:bg-gray-800" - : "bg-gray-50 dark:bg-gray-700/50"; - html += ``; - html += ``; - statKeys.forEach((s) => { - html += ``; - }); - html += ""; - }); - - html += "
Feature${s}
${feat}${data.summary_statistics[feat][s] ?? "—"}
"; - } - - html += buildCategoricalSummaryTable(data.categorical_summary); - - container.innerHTML = html; - - // Render histograms in the data overview panel - if (data.histograms) { - renderWorkspaceHistograms(data.histograms); - } - } else { - container.innerHTML = ` -
- - ${data.message} -
`; - } - }) + .then((data) => renderWorkspaceSummary(data)) .catch((err) => { const container = document.getElementById("workspace-summary"); if (container) From 75c5204dfc7deb0055af14c519a745a611e193f7 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Wed, 1 Jul 2026 18:04:06 -0700 Subject: [PATCH 2/2] Fix lint/prettier: flake8 comment indentation, JS formatting, pre-existing E231 - utils.py: move multi-line threshold comments to own lines (E114/E116) - inspector.js: prettier --write formatting - retriever.py: add missing space after comma (E231, pre-existing on develop) --- aidrin/agentic/retriever.py | 2 +- web/routes/utils.py | 19 ++++++++++++------- web/static/js/inspector.js | 23 +++++++++++++++++------ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/aidrin/agentic/retriever.py b/aidrin/agentic/retriever.py index a6c9a54c..cd3ca8d6 100644 --- a/aidrin/agentic/retriever.py +++ b/aidrin/agentic/retriever.py @@ -304,7 +304,7 @@ def _compress_one(item: dict[str, Any]) -> str: context_texts = [item.get("full_text", "") for item in retrieved] prompt_context = "\n\n".join( - f"[Source: {item.get('source','unknown')}]\n{self._sanitize(text)}" + f"[Source: {item.get('source', 'unknown')}]\n{self._sanitize(text)}" for item, text in zip(retrieved, context_texts) ) diff --git a/web/routes/utils.py b/web/routes/utils.py index 9d4c1dc0..39ac59ee 100644 --- a/web/routes/utils.py +++ b/web/routes/utils.py @@ -34,13 +34,18 @@ VALID_ROLES = ("continuous", "categorical", "identifier") # Cardinality thresholds (fractions are of the row count). -_ID_UNIQUE_RATIO = 0.98 # near-unique integer column -> identifier -_ID_NAME_MIN_RATIO = 0.10 # id-like name needs at least this spread to be an id -_ID_MIN_UNIQUE = 50 # ... and enough distinct values that near-uniqueness - # is meaningful (tiny datasets are trivially unique) -_CATEGORICAL_MAX_UNIQUE = 20 # at most this many distinct values ... -_CATEGORICAL_MAX_RATIO = 0.50 # ... AND distinct/rows below this (guards tiny - # datasets where every value is trivially distinct) +# A near-unique integer column reads as an identifier. +_ID_UNIQUE_RATIO = 0.98 +# An id-like name needs at least this spread to be treated as an identifier. +_ID_NAME_MIN_RATIO = 0.10 +# ... and enough distinct values that near-uniqueness is meaningful +# (tiny datasets are trivially unique). +_ID_MIN_UNIQUE = 50 +# A column is categorical when it has at most this many distinct values ... +_CATEGORICAL_MAX_UNIQUE = 20 +# ... AND distinct/rows is below this (guards tiny datasets where every value +# is trivially distinct). +_CATEGORICAL_MAX_RATIO = 0.50 _ID_NAME_TOKENS = {"id", "idx", "index", "key", "uid", "guid", "uuid", "pk"} diff --git a/web/static/js/inspector.js b/web/static/js/inspector.js index 239f82de..65ab2ab6 100644 --- a/web/static/js/inspector.js +++ b/web/static/js/inspector.js @@ -2130,7 +2130,8 @@ function buildHistogramSection(histograms, title) { if (Object.keys(columns).length === 0) return ""; let html = `

${title}

`; - html += '
'; + html += + '
'; for (const [colName, base64] of Object.entries(columns)) { html += `
@@ -2154,7 +2155,10 @@ function renderWorkspaceHistograms(histograms, categoricalHistograms) { if (!container) return; container.innerHTML = buildHistogramSection(histograms, "Numerical Feature Distributions") + - buildHistogramSection(categoricalHistograms, "Categorical Feature Distributions"); + buildHistogramSection( + categoricalHistograms, + "Categorical Feature Distributions", + ); } /** @@ -2207,7 +2211,9 @@ const ROLES_VISIBLE_LIMIT = 14; /** Live-filter the column-role rows by name substring and/or selected role. */ function filterColumnRoles() { - const q = (document.getElementById("roles-search")?.value || "").toLowerCase(); + const q = ( + document.getElementById("roles-search")?.value || "" + ).toLowerCase(); let matched = 0; document.querySelectorAll("#column-roles-grid .role-row").forEach((row) => { const col = (row.getAttribute("data-col") || "").toLowerCase(); @@ -2342,8 +2348,10 @@ function renderWorkspaceSummary(data) { .concat(allStats.filter((s) => !preferredOrder.includes(s))); html += '
'; - html += ''; - html += ''; + html += + '
'; + html += + ''; html += ''; statKeys.forEach((s) => { html += ``; @@ -2367,7 +2375,10 @@ function renderWorkspaceSummary(data) { html += buildCategoricalSummaryTable(data.categorical_summary); // Identifiers: excluded from stats/plots, listed for transparency - if (data.identifier_summary && Object.keys(data.identifier_summary).length > 0) { + if ( + data.identifier_summary && + Object.keys(data.identifier_summary).length > 0 + ) { html += '

Identifiers

'; html +=
Feature${s}