Adds date / time data type

docs/TABLE_CHART_REACT.md

@@ -11,6 +11,10 @@ The React Table Chart provides:
 - Find & Replace functionality
 - Cross-chart highlighting/selection
 
+### Date columns (`is_date`)
+
+Numeric columns with `is_date: true` store **days since Unix epoch** as `double`. Table cells, Find, Replace, and cell edit use ISO `YYYY-MM-DD` via `getValue` / date-aware string helpers. Sorting uses the raw day numbers (chronological), not lexicographic string order. Row filtering comes from DataStore filters (e.g. Selection Dialog range filters on the same column).
+
 ## File Structure
 
 | File | Purpose |

docs/extradocs/datasource.md

@@ -44,7 +44,10 @@ integer/double columns. In the former the colors array is mapped to the values a
 * **editable** - specifies a column can be edited.
 
 * **is_url** - the column contains url links (unique and text columns)
-* **is_url** - the column contains url links (unique and text columns)
+
+* **is_date** - when `true` on an `integer`/`double`/`int32` column, values are calendar dates stored as **days since Unix epoch (UTC)**. Charts use the numeric values for continuous spacing and filters. Display formatting as `YYYY-MM-DD` is applied via `getValue`, continuous axis ticks, color-legend ticks, and Selection Dialog range inputs. Timezone-aware timestamps are converted to UTC at ingest (which can shift the calendar day); values with a time-of-day may be stored as fractional days (display rounds to the nearest day).
+
+* **date_unit** - for `is_date` columns; currently always `"days"`.
 
 * **minMax** - for integer and double columns, the minimum/maximum values in an array of length 2
 

docs/jsdocs/extradocs/datasource.md

@@ -44,7 +44,10 @@ integer/double columns. In the former the colors array is mapped to the values a
 * **editable** - specifies a column can be edited.
 
 * **is_url** - the column contains url links (unique and text columns)
-* **is_url** - the column contains url links (unique and text columns)
+
+* **is_date** - when `true` on an `integer`/`double`/`int32` column, values are calendar dates stored as **days since Unix epoch (UTC)**. Charts use the numeric values for continuous spacing and filters. Display formatting as `YYYY-MM-DD` is applied via `getValue`, continuous axis ticks, color-legend ticks, and Selection Dialog range inputs. Timezone-aware timestamps are converted to UTC at ingest (which can shift the calendar day); values with a time-of-day may be stored as fractional days (display rounds to the nearest day).
+
+* **date_unit** - for `is_date` columns; currently always `"days"`.
 
 * **minMax** - for integer and double columns, the minimum/maximum values in an array of length 2
 

python/mdvtools/mdvproject.py

@@ -21,7 +21,7 @@
 from werkzeug.utils import secure_filename
 from shutil import copytree, ignore_patterns, copyfile
 from typing import Optional, NewType, List, Union, Any, cast
-from pandas.api.types import is_bool_dtype
+from pandas.api.types import is_bool_dtype, is_datetime64_any_dtype
 import polars as pl
 # from mdvtools.charts.view import View 
 import time
@@ -93,6 +93,76 @@ def _agent_debug_log(
     # unique created in fly (depends on string length)
 }
 
+_UNIX_EPOCH_UTC = pandas.Timestamp("1970-01-01", tz="UTC")
+_SECONDS_PER_DAY = 86400.0
+
+
+def _pandas_series_to_day_doubles(series: pandas.Series) -> pandas.Series:
+    """Convert a datetime-like pandas Series to float days since Unix epoch (UTC)."""
+    ts = pandas.to_datetime(series, utc=True, errors="coerce")
+    days = (ts - _UNIX_EPOCH_UTC).dt.total_seconds() / _SECONDS_PER_DAY
+    return days.astype("float64")
+
+
+def _object_column_looks_like_dates(series: pandas.Series, sample_size: int = 50) -> bool:
+    """Heuristic: uniformly parseable ISO-like strings in a sample of non-null values."""
+    sample = series.dropna()
+    if sample.empty:
+        return False
+    sample = sample.head(sample_size)
+    parsed = pandas.to_datetime(sample, utc=True, errors="coerce", format="mixed")
+    return bool(parsed.notna().mean() >= 0.8)
+
+
+def convert_pandas_datetime_columns(
+    dataframe: pandas.DataFrame,
+    *,
+    parse_object_dates: bool = True,
+) -> tuple[pandas.DataFrame, set[str]]:
+    """
+    Convert datetime columns (and optionally date-like object columns) to float64
+    days since Unix epoch. Returns the (possibly copied) dataframe and the set of
+    field names that were converted.
+    """
+    date_fields: set[str] = set()
+    out = dataframe
+    for col_name in list(dataframe.columns):
+        series = dataframe[col_name]
+        converted: Optional[pandas.Series] = None
+        if is_datetime64_any_dtype(series):
+            converted = _pandas_series_to_day_doubles(series)
+        elif parse_object_dates and (
+            series.dtype == object or str(series.dtype) == "string"
+        ):
+            if _object_column_looks_like_dates(series):
+                converted = _pandas_series_to_day_doubles(series)
+        if converted is not None:
+            if out is dataframe:
+                out = dataframe.copy()
+            out[col_name] = converted
+            date_fields.add(col_name)
+    return out, date_fields
+
+
+def apply_date_column_metadata(columns: list[dict], date_fields: set[str]) -> list[dict]:
+    """Mark converted date fields as double + is_date metadata."""
+    for col in columns:
+        field = col.get("field") or col.get("name")
+        if field in date_fields:
+            col["datatype"] = "double"
+            col["is_date"] = True
+            col["date_unit"] = "days"
+    return columns
+
+
+def map_polars_datetime_to_day_doubles(series: "pl.Series") -> "pl.Series":
+    """Convert Polars Date/Datetime to float days since Unix epoch (UTC)."""
+    # Cast to datetime[ms, UTC] then to epoch ms, then divide.
+    as_dt = series.cast(pl.Datetime("ms", "UTC"), strict=False)
+    ms = as_dt.dt.epoch(time_unit="ms")
+    days = ms / (86400.0 * 1000.0)
+    return days.cast(pl.Float64)
+
 
 class MDVProject:
     def __init__(
@@ -1337,9 +1407,12 @@ def add_datasource(
         try:
             if isinstance(dataframe, str):
                 dataframe = pandas.read_csv(dataframe, sep=separator)
+
+            dataframe, date_fields = convert_pandas_datetime_columns(dataframe)
 
             # Get columns to add
             columns = get_column_info(columns, dataframe, supplied_columns_only)
+            columns = apply_date_column_metadata(columns, date_fields)
 
             # Check if the datasource already exists
             try:
@@ -1485,7 +1558,9 @@ def add_datasource_polars(
 
             # Get columns to add using polars-compatible function.
             # Pass num_rows to avoid recalculating it for LazyFrames.
+            date_fields = get_polars_date_fields(dataframe)
             columns = get_column_info_polars(columns, dataframe, supplied_columns_only, num_rows)
+            columns = apply_date_column_metadata(columns, date_fields)
 
             has_existing_datasources = len(self.datasources) > 0
 
@@ -1544,6 +1619,19 @@ def add_datasource_polars(
                     else:
                         polars_series = dataframe[col["field"]]
 
+                    # Only convert Date/Datetime dtypes. Already-numeric day doubles
+                    # (e.g. re-import with is_date metadata) must not be remapped —
+                    # casting floats as datetime epoch-ms would destroy values.
+                    if _is_polars_datetime_dtype(polars_series.dtype):
+                        polars_series = map_polars_datetime_to_day_doubles(polars_series)
+                        col["datatype"] = "double"
+                        col["is_date"] = True
+                        col["date_unit"] = "days"
+                    elif col.get("is_date"):
+                        col["datatype"] = "double"
+                        col["is_date"] = True
+                        col["date_unit"] = "days"
+
                     add_column_to_group_from_polars(col, polars_series, gr, num_rows, self.skip_column_clean)
                 except Exception as e:
                     print(f" ++++++ DODGY COLUMN: {col['field']}")
@@ -2781,10 +2869,25 @@ def map_polars_to_mdv_type(polars_dtype):
         return "text"
     elif polars_dtype in [pl.Utf8, pl.String]:
         return "text"  # Default to text, might change to unique based on cardinality
+    elif _is_polars_datetime_dtype(polars_dtype):
+        return "double"
     else:
         # Default to text for any other types
         return "text"
 
+
+def _is_polars_datetime_dtype(polars_dtype) -> bool:
+    dtype_name = str(polars_dtype)
+    return dtype_name == "Date" or dtype_name.startswith("Datetime")
+
+
+def get_polars_date_fields(dataframe: "pl.DataFrame | pl.LazyFrame") -> set[str]:
+    return {
+        name
+        for name, dtype in dataframe.collect_schema().items()
+        if _is_polars_datetime_dtype(dtype)
+    }
+
 def add_column_to_group_from_polars(col_info, polars_series, h5_group, num_rows, skip_column_clean):
     """Add a column to HDF5 group using Polars Series data"""
     import numpy as np

python/mdvtools/tests/test_date_columns.py

@@ -0,0 +1,106 @@
+"""Tests for datetime → days-since-epoch date column ingest."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+import polars as pl
+import pytest
+
+from mdvtools.mdvproject import (
+    MDVProject,
+    apply_date_column_metadata,
+    convert_pandas_datetime_columns,
+    map_polars_datetime_to_day_doubles,
+)
+
+
+def test_convert_pandas_datetime_columns_sets_day_doubles():
+    df = pd.DataFrame(
+        {
+            "when": pd.to_datetime(
+                ["1970-01-01", "2020-01-01", None],
+                utc=True,
+            ),
+            "label": ["a", "b", "c"],
+        }
+    )
+    out, date_fields = convert_pandas_datetime_columns(df)
+    assert date_fields == {"when"}
+    assert out["when"].dtype == np.float64
+    assert out["when"].iloc[0] == pytest.approx(0.0)
+    assert out["when"].iloc[1] == pytest.approx(18262.0)
+    assert np.isnan(out["when"].iloc[2])
+    assert list(out["label"]) == ["a", "b", "c"]
+
+
+def test_convert_pandas_object_iso_dates():
+    df = pd.DataFrame({"d": ["2024-06-01", "2024-06-15", "2024-07-01"]})
+    out, date_fields = convert_pandas_datetime_columns(df)
+    assert date_fields == {"d"}
+    assert out["d"].iloc[0] < out["d"].iloc[1] < out["d"].iloc[2]
+    span = out["d"].iloc[2] - out["d"].iloc[0]
+    assert span == pytest.approx(30.0)
+
+
+def test_apply_date_column_metadata():
+    cols = [
+        {"field": "when", "datatype": "double"},
+        {"field": "x", "datatype": "double"},
+    ]
+    apply_date_column_metadata(cols, {"when"})
+    assert cols[0]["is_date"] is True
+    assert cols[0]["date_unit"] == "days"
+    assert "is_date" not in cols[1]
+
+
+def test_add_datasource_datetime_column(tmp_path):
+    project = MDVProject(str(tmp_path), delete_existing=True)
+    df = pd.DataFrame(
+        {
+            "when": pd.to_datetime(["2020-01-01", "2020-01-11", "2020-02-01"]),
+            "value": [1.0, 2.0, 3.0],
+        }
+    )
+    project.add_datasource("events", df, add_to_view=None)
+    meta = project.get_datasource_metadata("events")
+    when_col = next(c for c in meta["columns"] if c["field"] == "when")
+    assert when_col["datatype"] == "double"
+    assert when_col["is_date"] is True
+    assert when_col["date_unit"] == "days"
+    assert when_col["minMax"][1] - when_col["minMax"][0] == pytest.approx(31.0)
+
+
+def test_polars_datetime_to_day_doubles():
+    s = pl.Series("d", ["1970-01-01", "2020-01-01"]).str.to_date()
+    days = map_polars_datetime_to_day_doubles(s)
+    assert days[0] == pytest.approx(0.0)
+    assert days[1] == pytest.approx(18262.0)
+
+
+def test_polars_is_date_on_already_numeric_preserves_day_doubles(tmp_path):
+    """is_date metadata on Float64 day doubles must not re-run datetime conversion."""
+    project = MDVProject(str(tmp_path), delete_existing=True)
+    df = pl.DataFrame(
+        {
+            "when": [0.0, 18262.0, 18300.0],
+            "value": [1.0, 2.0, 3.0],
+        }
+    )
+    columns = [
+        {
+            "field": "when",
+            "name": "when",
+            "datatype": "double",
+            "is_date": True,
+            "date_unit": "days",
+        },
+        {"field": "value", "name": "value", "datatype": "double"},
+    ]
+    project.add_datasource_polars("events", df, columns=columns, add_to_view=None)
+    meta = project.get_datasource_metadata("events")
+    when_col = next(c for c in meta["columns"] if c["field"] == "when")
+    assert when_col["is_date"] is True
+    assert when_col["date_unit"] == "days"
+    assert when_col["minMax"][0] == pytest.approx(0.0)
+    assert when_col["minMax"][1] == pytest.approx(18300.0)

src/charts/BaseChart.ts

@@ -16,6 +16,7 @@ import type { FieldSpec, FieldSpecs } from "@/lib/columnTypeHelpers";
 import getParamsGuiSpec from "./dialogs/utils/ParamsSettingGui";
 import tippy, {type Instance as TippyInstance} from "tippy.js";
 import 'tippy.js/dist/tippy.css';
+import { isDateColumn } from "@/lib/dateFormat";
 import { buildColorLegendSpec } from "@/react/legend/color_legend/buildColorLegendSpec";
 import type { ColorLegendSpec } from "@/react/legend/color_legend/types";
 import ColorLegend from "@/react/components/legend/ColorLegend";
@@ -569,9 +570,13 @@ class BaseChart<T extends BaseConfig> {
         if (!colorBy || typeof colorBy !== "string") {
             return null;
         }
+        const colorCol = this.dataStore.columnIndex[colorBy];
         const conf = {
             overideValues: {
-                colorLogScale: this.config.log_color_scale,
+                // Log color scales are not meaningful for calendar day numbers.
+                colorLogScale: isDateColumn(colorCol)
+                    ? false
+                    : this.config.log_color_scale,
             },
         };
         this._addTrimmedColor(colorBy, conf);