diff --git a/app/package.json b/app/package.json index 8dd5dd31..b5e39f15 100644 --- a/app/package.json +++ b/app/package.json @@ -37,7 +37,7 @@ "scripts": { "start": "react-scripts start", "build": "BROWSERSLIST_IGNORE_OLD_DATA=true react-scripts build", - "test": "react-scripts test", + "test": "TZ=America/New_York react-scripts test", "eject": "react-scripts eject", "cy:run": "cypress run", "cy:open": "cypress open" diff --git a/app/src/components/inventory/FieldCell.js b/app/src/components/inventory/FieldCell.js index 05138d79..2db5406e 100644 --- a/app/src/components/inventory/FieldCell.js +++ b/app/src/components/inventory/FieldCell.js @@ -1,5 +1,6 @@ import React from "react"; import {Select, TextInput} from "../ui"; +import {formatInventoryDateValue} from "./inventoryDates"; import {ALL_UNITS, evaluateExpression, NUMERIC_TYPES, UNIT_GROUPS} from "./units"; // Renders the correct editor for a single field/value, wiring up keyboard navigation. The `inputRef` is attached @@ -102,7 +103,7 @@ export function FieldCell({field, value, unitValue, onChange, onUnitChange, onEn } if (field.type === 'date') { - return ; + return ; } if (field.type === 'number' || field.type === 'calories') { diff --git a/app/src/components/inventory/InventoryTable.js b/app/src/components/inventory/InventoryTable.js index f9e4480f..15f2a0d5 100644 --- a/app/src/components/inventory/InventoryTable.js +++ b/app/src/components/inventory/InventoryTable.js @@ -2,6 +2,7 @@ import React, {useMemo, useRef, useState} from "react"; import {Button, Checkbox, Icon, Table} from "../ui"; import {FieldCell} from "./FieldCell"; import {applyComputedCounts, computedCountConfigs} from "./computeFields"; +import {formatInventoryDateValue, localTodayYmd, parseInventoryDateMs} from "./inventoryDates"; import {NUMERIC_TYPES} from "./units"; const LOCATION_LIST_ID = 'inventory-location-suggestions'; @@ -25,22 +26,37 @@ function nextId(items) { return items.reduce((max, i) => (typeof i.id === 'number' && i.id > max ? i.id : max), 0) + 1; } -// Type-aware comparison for sorting: numeric for number/quantity/calories, chronological for date, locale string -// otherwise. Blank values sort last (ascending). -function compareByField(a, b, field) { - const av = a[field.key], bv = b[field.key]; - const aBlank = av === '' || av == null; - const bBlank = bv === '' || bv == null; - if (aBlank || bBlank) { - return aBlank === bBlank ? 0 : (aBlank ? 1 : -1); +// Blank values sort last (ascending). Sort keys are precomputed once per row before sorting. +function fieldSortKey(item, field) { + const av = item[field.key]; + if (av === '' || av == null) { + return {kind: 'blank'}; } if (NUMERIC_TYPES.includes(field.type)) { - return (Number(av) || 0) - (Number(bv) || 0); + return {kind: 'number', v: Number(av) || 0}; } if (field.type === 'date') { - return (Date.parse(av) || 0) - (Date.parse(bv) || 0); + return {kind: 'date', v: parseInventoryDateMs(av) || 0}; } - return String(av).localeCompare(String(bv)); + return {kind: 'string', v: String(av)}; +} + +function compareFieldSortKeys(ak, bk) { + if (ak.kind === 'blank' || bk.kind === 'blank') { + const aBlank = ak.kind === 'blank'; + const bBlank = bk.kind === 'blank'; + return aBlank === bBlank ? 0 : (aBlank ? 1 : -1); + } + if (ak.kind === 'number' && bk.kind === 'number') { + return ak.v - bk.v; + } + if (ak.kind === 'date' && bk.kind === 'date') { + return ak.v - bk.v; + } + if (ak.kind === 'string' && bk.kind === 'string') { + return ak.v.localeCompare(bk.v); + } + return 0; } /** @@ -120,7 +136,10 @@ export function InventoryTable({slug, fields, items, locations, catalog, search, if (!field) { return list; } - const sorted = [...list].sort((a, b) => compareByField(a, b, field)); + const withKeys = list.map(item => ({item, key: fieldSortKey(item, field)})); + const sorted = withKeys + .sort((a, b) => compareFieldSortKeys(a.key, b.key)) + .map(({item}) => item); return sort.dir === 'desc' ? sorted.reverse() : sorted; }, [items, fields, sort]); @@ -296,19 +315,25 @@ export function formatValue(item, field) { const unit = item[`${field.key}_unit`] || field.unit || ''; return `${value} ${unit}`.trim(); } + if (field.type === 'date') { + return formatInventoryDateValue(value); + } return String(value); } -// True when any `date`-type field is before the start of today (item is expired). +// True when any `date`-type field is before today's local calendar date (item is expired). export function isItemExpired(item, fields) { - const start = new Date(); - start.setHours(0, 0, 0, 0); + const today = localTodayYmd(); return (fields || []).some(f => { if (f.type !== 'date') { return false; } - const t = Date.parse(item[f.key]); - return !isNaN(t) && t < start.getTime(); + const raw = item[f.key]; + if (raw === '' || raw == null) { + return false; + } + const ymd = formatInventoryDateValue(raw); + return /^\d{4}-\d{2}-\d{2}$/.test(ymd) && ymd < today; }); } diff --git a/app/src/components/inventory/InventoryTable.test.js b/app/src/components/inventory/InventoryTable.test.js index e28c1963..fe4b1ee4 100644 --- a/app/src/components/inventory/InventoryTable.test.js +++ b/app/src/components/inventory/InventoryTable.test.js @@ -205,6 +205,22 @@ describe('InventoryTable sorting', () => { // 5 < 12 < 20 numerically (lexical would put '12','20','5'). expect(bodyNames()).toEqual(['Rice', 'Apples', 'Beans']); }); + + test('date column sorts chronologically, including legacy epoch values', () => { + const dateFields = [ + {key: 'name', label: 'Name', type: 'text', order: 0}, + {key: 'expiration_date', label: 'Expires', type: 'date', order: 1}, + ]; + const datedItems = [ + {id: 1, name: 'Late', expiration_date: '1893456000'}, // 2030-01-01 + {id: 2, name: 'Early', expiration_date: '2000-01-01'}, + ]; + render(); + fireEvent.click(screen.getByText('Expires')); + expect(bodyNames()).toEqual(['Early', 'Late']); + expect(screen.getByText('2030-01-01')).toBeTruthy(); + }); }); describe('InventoryTable catalog pre-fill', () => { diff --git a/app/src/components/inventory/inventoryDates.js b/app/src/components/inventory/inventoryDates.js new file mode 100644 index 00000000..3626393b --- /dev/null +++ b/app/src/components/inventory/inventoryDates.js @@ -0,0 +1,106 @@ +const DATE_ONLY_RE = /^(\d{4})-(\d{2})-(\d{2})$/; +/** Legacy inventory dates stored as Unix epoch seconds (not arbitrary digit strings). */ +const EPOCH_SECONDS_RE = /^\d{9,10}(\.0+)?$/; + +function digitsToEpochMs(value) { + const n = Number(value); + if (!Number.isFinite(n)) { + return null; + } + return n * 1000; +} + +function isEpochSecondsNumber(value) { + if (typeof value !== 'number' || !Number.isFinite(value)) { + return false; + } + const digits = String(Math.trunc(Math.abs(value))).length; + return digits >= 9 && digits <= 10; +} + +function formatMsUtc(ms) { + const d = new Date(ms); + const y = d.getUTCFullYear(); + const m = String(d.getUTCMonth() + 1).padStart(2, '0'); + const day = String(d.getUTCDate()).padStart(2, '0'); + return `${y}-${m}-${day}`; +} + +function formatMsLocal(ms) { + const d = new Date(ms); + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, '0'); + const day = String(d.getDate()).padStart(2, '0'); + return `${y}-${m}-${day}`; +} + +function parseDateOnlyLocalMs(value) { + const m = DATE_ONLY_RE.exec(String(value).trim()); + if (!m) { + return null; + } + const y = Number(m[1]); + const mo = Number(m[2]); + const day = Number(m[3]); + const ms = new Date(y, mo - 1, day).getTime(); + return Number.isNaN(ms) ? null : ms; +} + +/** Today's calendar date in the local timezone as YYYY-MM-DD. */ +export function localTodayYmd() { + return formatMsLocal(Date.now()); +} + +// Parse inventory `date`-field values (YYYY-MM-DD or legacy Unix epoch seconds) to epoch ms, or null. +export function parseInventoryDateMs(value) { + if (value === '' || value == null) { + return null; + } + if (isEpochSecondsNumber(value)) { + return digitsToEpochMs(value); + } + const s = String(value).trim(); + if (EPOCH_SECONDS_RE.test(s)) { + return digitsToEpochMs(s); + } + const dateOnlyMs = parseDateOnlyLocalMs(s); + if (dateOnlyMs != null) { + return dateOnlyMs; + } + if (/^\d+$/.test(s)) { + return null; + } + const parsed = Date.parse(s); + return Number.isNaN(parsed) ? null : parsed; +} + +// Format for display and `` (calendar date; never shifted by timezone). +export function formatInventoryDateValue(value) { + if (value === '' || value == null) { + return value == null ? '' : String(value); + } + if (typeof value === 'number' && Number.isFinite(value)) { + // A YAML number is either legacy epoch seconds or something we can't interpret; never treat it as epoch + // milliseconds, so `2030` and `'2030'` display the same. + return isEpochSecondsNumber(value) ? formatMsUtc(digitsToEpochMs(value)) : String(value); + } + const s = String(value).trim(); + if (DATE_ONLY_RE.test(s)) { + return s; + } + if (EPOCH_SECONDS_RE.test(s)) { + return formatMsUtc(digitsToEpochMs(s)); + } + if (/^\d+$/.test(s)) { + return s; + } + const dateOnlyMs = parseDateOnlyLocalMs(s); + if (dateOnlyMs != null) { + return formatMsLocal(dateOnlyMs); + } + const parsed = Date.parse(s); + if (Number.isNaN(parsed)) { + return s; + } + return formatMsLocal(parsed); +} diff --git a/app/src/components/inventory/inventoryDates.test.js b/app/src/components/inventory/inventoryDates.test.js new file mode 100644 index 00000000..2b7e4273 --- /dev/null +++ b/app/src/components/inventory/inventoryDates.test.js @@ -0,0 +1,96 @@ +import {formatInventoryDateValue, localTodayYmd, parseInventoryDateMs} from "./inventoryDates"; +import {filterItems, formatValue, isItemExpired} from "./InventoryTable"; + +describe('inventoryDates', () => { + test('parseInventoryDateMs accepts ISO dates and epoch seconds', () => { + expect(parseInventoryDateMs('2030-01-01')).toBe(new Date(2030, 0, 1).getTime()); + expect(parseInventoryDateMs('1893456000')).toBe(1893456000 * 1000); + }); + + test('formatInventoryDateValue renders a calendar date', () => { + expect(formatInventoryDateValue('2030-01-01')).toBe('2030-01-01'); + expect(formatInventoryDateValue('1893456000')).toBe('2030-01-01'); + }); + + test('ambiguous digit strings are not treated as epoch seconds', () => { + expect(formatInventoryDateValue('2030')).toBe('2030'); + expect(formatInventoryDateValue('20300101')).toBe('20300101'); + expect(parseInventoryDateMs('2030')).toBeNull(); + expect(parseInventoryDateMs('20300101')).toBeNull(); + expect(formatInventoryDateValue('2030')).not.toMatch(/^1970/); + expect(formatInventoryDateValue('20300101')).not.toMatch(/^1970/); + // Unquoted YAML numbers display the same as the equivalent string. + expect(formatInventoryDateValue(2030)).toBe('2030'); + expect(formatInventoryDateValue(20300101)).toBe('20300101'); + expect(formatInventoryDateValue(1893456000)).toBe('2030-01-01'); + }); + + test('epoch seconds with a .0 suffix are accepted', () => { + expect(formatInventoryDateValue('1893456000.0')).toBe('2030-01-01'); + expect(parseInventoryDateMs('1893456000.0')).toBe(1893456000 * 1000); + }); + + test('date-only values round-trip without timezone shift (US Eastern)', () => { + const samples = ['2030-01-01', '2000-06-15', '2999-01-01']; + for (const stored of samples) { + const displayed = formatInventoryDateValue(stored); + expect(displayed).toBe(stored); + const fromInput = displayed; + expect(fromInput).toBe(stored); + expect(formatInventoryDateValue(fromInput)).toBe(stored); + const [y, m, d] = stored.split('-').map(Number); + expect(parseInventoryDateMs(fromInput)).toBe(new Date(y, m - 1, d).getTime()); + } + }); +}); + +describe('isItemExpired', () => { + const fields = [{key: 'expiration_date', label: 'Expires', type: 'date', order: 0}]; + + afterEach(() => jest.useRealTimers()); + + // The local calendar date of the (possibly faked) clock, independent of the process TZ. + const localYmd = () => { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; + }; + + test('a date that is still today or later is not expired late in the evening', () => { + jest.useFakeTimers(); + // 2030-01-01T02:00:00Z is 9 PM on 2029-12-31 in US Eastern, but already 2030-01-01 in UTC and east of it. + // Either way 2030-01-01 is not before the local date, so it must not be expired. + jest.setSystemTime(new Date('2030-01-01T02:00:00.000Z')); + expect(localTodayYmd()).toBe(localYmd()); + expect(isItemExpired({expiration_date: '2030-01-01'}, fields)).toBe(false); + expect(isItemExpired({expiration_date: '1893456000'}, fields)).toBe(false); + }); + + test('yesterday\'s calendar date is expired', () => { + // 2030-01-02T15:00:00Z is 2030-01-02 in every timezone from UTC-9 through UTC+8. + jest.useFakeTimers(); + jest.setSystemTime(new Date('2030-01-02T15:00:00.000Z')); + expect(localTodayYmd()).toBe(localYmd()); + expect(isItemExpired({expiration_date: '2030-01-01'}, fields)).toBe(true); + expect(isItemExpired({expiration_date: '1893456000'}, fields)).toBe(true); + }); +}); + +describe('formatValue date fields', () => { + const field = {key: 'expiration_date', label: 'Expires', type: 'date', order: 0}; + + test('legacy epoch seconds display as YYYY-MM-DD', () => { + expect(formatValue({expiration_date: '1893456000'}, field)).toBe('2030-01-01'); + }); +}); + +describe('filterItems epoch dates', () => { + const fields = [ + {key: 'name', label: 'Name', type: 'text', order: 0}, + {key: 'expiration_date', label: 'Expires', type: 'date', order: 1}, + ]; + const items = [{id: 1, name: 'Rice', expiration_date: '1893456000'}]; + + test('search matches formatted expiration date', () => { + expect(filterItems(items, fields, '2030-01-01')).toHaveLength(1); + }); +}); diff --git a/modules/inventory/common.py b/modules/inventory/common.py index 1acf5025..3a70cb9d 100644 --- a/modules/inventory/common.py +++ b/modules/inventory/common.py @@ -9,8 +9,7 @@ from typing import List, Optional from wrolpi.common import logger, MultiFileConfig, read_config_data from wrolpi.errors import ValidationError -from .defaults import (DEFAULT_INVENTORIES, FIELD_TYPES, default_fields, slugify) - +from .defaults import (DEFAULT_INVENTORIES, FIELD_TYPES, default_fields, normalize_item_dates, slugify) logger = logger.getChild(__name__) @@ -62,6 +61,7 @@ def _normalize_items(items: list, fields_: list) -> list: normalized = [] for raw in items: clean = {k: (v.strip() if isinstance(v, str) else v) for k, v in raw.items() if k in allowed} + normalize_item_dates([clean], fields_) item_id = raw.get('id') if not isinstance(item_id, int) or item_id in used: counter += 1 @@ -88,6 +88,27 @@ class InventoriesConfig(MultiFileConfig): # Exclude reserved files (e.g. catalog.yaml) that share the directory but are not inventories. return [slug for slug in super().discover() if slug not in RESERVED_SLUGS] + def import_all(self, send_events: bool = False): + super().import_all(send_events=send_events) + self._repair_legacy_dates() + + def _repair_legacy_dates(self): + """One-time repair: inventories migrated before dates were normalized hold epoch seconds in their date + fields. Rewrite those files as YYYY-MM-DD so every reader (API, MCP, frontend) sees a calendar date.""" + for slug in list(self._configs.keys()): + entity = dict(self._configs[slug]) + items = [dict(i) for i in entity.get('items') or [] if isinstance(i, dict)] + if not normalize_item_dates(items, entity.get('fields') or []): + continue + entity['items'] = items + try: + with self.save_lock(): + self._configs[slug] = entity + self._persist_slug(slug) + logger.info(f'Normalized legacy epoch dates in inventory {slug}') + except Exception as e: + logger.error(f'Failed to normalize legacy epoch dates in inventory {slug}', exc_info=e) + def all_inventories(self) -> List[dict]: """Return every inventory in full (fields + items), sorted by name. diff --git a/modules/inventory/defaults.py b/modules/inventory/defaults.py index 3e6de5e2..287c8092 100644 --- a/modules/inventory/defaults.py +++ b/modules/inventory/defaults.py @@ -5,6 +5,7 @@ add/remove/reorder/retype fields freely. Quantity-field default units use mathj not ``pound``); all unit math happens in the frontend via mathjs. """ import re +from datetime import datetime, timezone # Supported field types. The backend treats every value as an opaque string/number; only the frontend interprets # `quantity` (magnitude + `_unit`), `date`, `select` (enum `options`), `location`, and `calories` (a number of @@ -12,6 +13,42 @@ import re FIELD_TYPES = ('text', 'number', 'quantity', 'date', 'select', 'location', 'calories') +_DATE_ONLY_RE = re.compile(r'^\d{4}-\d{2}-\d{2}$') +# Legacy inventory dates were stored as Unix epoch seconds (optionally ``str(float)``, e.g. ``1893456000.0``). +_EPOCH_SECONDS_RE = re.compile(r'^\d{9,10}(\.0+)?$') + + +def normalize_date_value(value): + """Normalize a ``date``-field value: legacy epoch seconds become ``YYYY-MM-DD`` (UTC); anything else is + returned unchanged (stripped if a string).""" + if value is None or value == '': + return value + s = value.strip() if isinstance(value, str) else str(value) + if _DATE_ONLY_RE.match(s): + return s + if _EPOCH_SECONDS_RE.match(s): + return datetime.fromtimestamp(int(float(s)), tz=timezone.utc).strftime('%Y-%m-%d') + return s + + +def date_field_keys(fields_) -> set: + """The keys of every ``date``-type field in a field schema.""" + return {f['key'] for f in fields_ or [] if isinstance(f, dict) and f.get('type') == 'date'} + + +def normalize_item_dates(items: list, fields_: list) -> bool: + """Normalize every date-field value of ``items`` in place. Returns True if anything changed.""" + changed = False + for key in date_field_keys(fields_): + for item in items or []: + if isinstance(item, dict) and key in item: + new = normalize_date_value(item[key]) + if new != item[key]: + item[key] = new + changed = True + return changed + + def _field(key, label, type_, order, **kwargs): d = dict(key=key, label=label, type=type_, order=order) d.update({k: v for k, v in kwargs.items() if v is not None}) diff --git a/modules/inventory/migrate.py b/modules/inventory/migrate.py index 65e08293..b8713ccc 100644 --- a/modules/inventory/migrate.py +++ b/modules/inventory/migrate.py @@ -8,7 +8,7 @@ import shutil from wrolpi.common import logger, read_config_data from wrolpi.dates import now -from .defaults import default_fields, slugify +from .defaults import default_fields, normalize_item_dates, slugify logger = logger.getChild(__name__) @@ -42,6 +42,8 @@ def _convert_item(legacy: dict, item_id: int) -> dict: unit = normalize_unit(legacy.get('unit')) if unit: item['item_size_unit'] = unit + # Legacy expiration dates are epoch seconds; store calendar dates like the rest of the app. + normalize_item_dates([item], default_fields('food')) return item diff --git a/modules/inventory/test/test_inventory.py b/modules/inventory/test/test_inventory.py index a5e251d3..4fe4d4de 100644 --- a/modules/inventory/test/test_inventory.py +++ b/modules/inventory/test/test_inventory.py @@ -79,7 +79,20 @@ def test_save_inventory_strips_whitespace(food_inventory_factory, test_inventory assert item['name'] == 'Rice' assert item['category'] == 'grains' assert item['count'] == '4' - + + +def test_save_inventory_normalizes_legacy_epoch_expiration(food_inventory_factory, test_inventory_configs): + config = test_inventory_configs + slug = food_inventory_factory() + saved = config.save_inventory(slug, dict(items=[ + dict(name='Rice', expiration_date='1893456000'), + ])) + assert saved['items'][0]['expiration_date'] == '2030-01-01' + # Short digit strings are not mistaken for epoch seconds. + saved = config.save_inventory(slug, dict(items=[ + dict(name='Weird', expiration_date='2030'), + ])) + assert saved['items'][0]['expiration_date'] == '2030' def test_save_inventory_fields(food_inventory_factory, test_inventory_configs): config = test_inventory_configs diff --git a/modules/inventory/test/test_migration.py b/modules/inventory/test/test_migration.py index f9ef0c3f..68a1a2c1 100644 --- a/modules/inventory/test/test_migration.py +++ b/modules/inventory/test/test_migration.py @@ -55,6 +55,55 @@ def test_migrate_legacy_yaml(test_inventory_configs): assert legacy_file.with_suffix('.yaml.migrated').is_file() +def test_migrate_converts_epoch_expiration_dates(test_inventory_configs): + """Legacy items stored expiration dates as epoch seconds (sometimes ``str(float)``); the migration writes + calendar dates so no reader has to guess what an all-digit string means.""" + config = test_inventory_configs + _write_legacy(config, [ + dict(name='Food Storage', created_at='2024-01-01', items=[ + dict(id=1, name='Salt', item_size='25', unit='lb', count='8', expiration_date=1893456000), + dict(id=2, name='Rice', item_size='25', unit='lb', count='8', expiration_date='1893456000.0'), + dict(id=3, name='Beans', item_size='25', unit='lb', count='8', expiration_date='2030-01-01'), + dict(id=4, name='Odd', item_size='25', unit='lb', count='8', expiration_date='2030'), + ]), + ]) + + assert migrate_legacy_inventory(config) is True + config.import_all() + + inventory = config.get_inventory(config.all_inventories()[0]['slug']) + assert [i['expiration_date'] for i in inventory['items']] == ['2030-01-01', '2030-01-01', '2030-01-01', '2030'] + # The file on disk holds calendar dates too. + assert '1893456000' not in config.get_file(inventory['slug']).read_text() + + +def test_import_repairs_legacy_epoch_dates(test_inventory_configs): + """Inventories migrated before dates were normalized are rewritten once on import.""" + config = test_inventory_configs + directory = config.get_directory() + directory.mkdir(parents=True, exist_ok=True) + file = directory / 'food.yaml' + with file.open('wt') as fh: + yaml.dump(dict( + version=3, slug='food', name='Food', type='food', created_at='2024-01-01', viewed_at='2024-01-01', + fields=[dict(key='name', label='Name', type='text', order=0), + dict(key='expiration_date', label='Expires', type='date', order=1)], + items=[dict(id=1, name='Salt', expiration_date='1893456000'), + dict(id=2, name='Rice', expiration_date='2030-06-01')], + ), fh) + + config.import_all() + inventory = config.get_inventory('food') + assert [i['expiration_date'] for i in inventory['items']] == ['2030-01-01', '2030-06-01'] + # Repaired on disk, with a version bump so stale clients notice. + assert '1893456000' not in file.read_text() + assert inventory['version'] == 4 + + # A second import finds nothing to repair and does not rewrite the file. + config.import_all() + assert config.get_inventory('food')['version'] == 4 + + def test_migrate_is_idempotent(test_inventory_configs): config = test_inventory_configs _write_legacy(config, [dict(name='Food', items=[])])