diff --git a/docs/receipt-parsing.md b/docs/receipt-parsing.md index aaf854b..72b7182 100644 --- a/docs/receipt-parsing.md +++ b/docs/receipt-parsing.md @@ -23,21 +23,41 @@ For each receipt: | `store` | Best-effort store from the header + `known_stores` | | `receiptTotal` | First grand-total amount after the items | | `totalDiscount` | Receipt-level `Totaal korting` (and similar) | -| `validationPassed` | Item sum matches `receiptTotal` within €0.01 | +| `validationPassed` | Item sum matches `receiptTotal` within €0.01, item count matches when printed, and no line needs manual review | +| `items[].sourceBounds` | Normalized rect on the receipt photo for tap-to-highlight in Review | +| `items[].needsManualReview` | Parser could not resolve price/qty confidently | +| `expectedItemCount` | From `Aantal Artikel`-style lines when found | +| `ocrImageWidth` / `ocrImageHeight` | Image size used to normalize bounds | -If validation fails, Review shows: *Parsed totals do not match the receipt.* +If validation fails, Review shows a banner asking you to check items before saving. --- ## Pipeline +### Primary: spatial (ML Kit bounding boxes) + +Used automatically after scanning a receipt photo. + +1. **Extract elements** from ML Kit `RecognizedText` (blocks → lines → elements), keeping each element's pixel bounding box. +2. **Cluster rows** by Y-center (threshold ≈ 1.2× median element height — see `spatial_parsing` in the JSON config). +3. **Classify rows** with column logic: right ~25% of content width is the price column; pure `N X price` rows are quantity modifiers (checked first). +4. **Build items** with a one-row lookahead buffer (Jumbo split layout + Kruidvat inline layout). +5. **Validate** totals, printed item count, and manual-review flags. + +Implementation: `ReceiptOcr`, `ReceiptRowBuilder`, `ReceiptSpatialParser` in `lib/services/`. + +### Fallback: flat text + +`ReceiptParser.parseReceipt(String)` for tests and when only flattened OCR text exists. + 1. **Preprocess** each OCR line: trim, collapse spaces, drop empty lines. 2. **Choose a layout** - **Sequential** (default): name, price, quantity and discount appear in reading order. Jumbo (printed), Lidl, Kruidvat, World Toko. - **Column OCR fallback:** ML Kit sometimes dumps every name, then every amount. Detected when there is a run of 4+ price-only lines *and* enough - name-like lines. Used for the Jumbo sample dump in tests. + name-like lines. 3. **Classify** each line (first match wins) using the keyword JSON. 4. **Build items** (pending name, then quantity/discount/price). 5. **Validate** against the receipt total when one was found. diff --git a/lib/models/line_item.dart b/lib/models/line_item.dart index 1b88e96..a16e63d 100644 --- a/lib/models/line_item.dart +++ b/lib/models/line_item.dart @@ -1,6 +1,8 @@ /// A parsed receipt row shown on the review screen. library; +import 'receipt_bounds.dart'; + class LineItem { LineItem({ required this.id, @@ -9,6 +11,8 @@ class LineItem { this.quantity, this.unitPrice, this.discount = 0, + this.sourceBounds, + this.needsManualReview = false, }); final String id; @@ -22,6 +26,12 @@ class LineItem { /// Negative amount when a discount was folded into [price]. final double discount; + /// Region on the receipt photo (normalized 0–1) used when parsing this line. + final ReceiptBounds? sourceBounds; + + /// True when the parser could not confidently resolve price/qty for this line. + final bool needsManualReview; + bool get hasDiscount => discount != 0; /// List price before a folded discount. Equals [price] when there is none. @@ -37,7 +47,10 @@ class LineItem { int? quantity, double? unitPrice, double? discount, + ReceiptBounds? sourceBounds, + bool? needsManualReview, bool clearQuantity = false, + bool clearSourceBounds = false, }) { return LineItem( id: id ?? this.id, @@ -46,6 +59,8 @@ class LineItem { quantity: clearQuantity ? null : (quantity ?? this.quantity), unitPrice: clearQuantity ? null : (unitPrice ?? this.unitPrice), discount: discount ?? this.discount, + sourceBounds: clearSourceBounds ? null : (sourceBounds ?? this.sourceBounds), + needsManualReview: needsManualReview ?? this.needsManualReview, ); } } diff --git a/lib/review_screen.dart b/lib/review_screen.dart index c58a998..838bd24 100644 --- a/lib/review_screen.dart +++ b/lib/review_screen.dart @@ -13,6 +13,7 @@ import 'utils/money.dart'; import 'utils/supermarkets.dart'; import 'widgets/discounted_price.dart'; import 'widgets/item_edit_sheet.dart'; +import 'widgets/receipt_highlight_overlay.dart'; import 'widgets/store_tag.dart'; import 'widgets/tag_editor.dart'; @@ -26,6 +27,8 @@ class ReviewScreen extends StatefulWidget { this.supermarketNames = kSupermarkets, this.needsReview = false, this.detectedStore, + this.ocrImageWidth, + this.ocrImageHeight, }); final List items; @@ -35,6 +38,8 @@ class ReviewScreen extends StatefulWidget { final List supermarketNames; final bool needsReview; final String? detectedStore; + final double? ocrImageWidth; + final double? ocrImageHeight; @override State createState() => _ReviewScreenState(); @@ -49,6 +54,7 @@ class _ReviewScreenState extends State { final _tripName = TextEditingController(); List _tags = []; List _tagSuggestions = []; + String? _highlightedItemId; @override void initState() { @@ -88,6 +94,11 @@ class _ReviewScreenState extends State { return _supermarket; } + bool get _hasReceipt { + final path = widget.imagePath; + return path != null && path.isNotEmpty && File(path).existsSync(); + } + Future _edit({LineItem? existing}) async { final result = await showItemEditSheet(context, item: existing); if (result == null || !mounted) return; @@ -98,9 +109,20 @@ class _ReviewScreenState extends State { } else { final index = _items.indexWhere((item) => item.id == existing.id); if (index >= 0) { - _items[index] = result; + _items[index] = result.copyWith( + sourceBounds: existing.sourceBounds, + needsManualReview: false, + ); } } + _highlightedItemId = null; + }); + } + + void _toggleHighlight(LineItem item) { + if (item.sourceBounds == null || !_hasReceipt) return; + setState(() { + _highlightedItemId = _highlightedItemId == item.id ? null : item.id; }); } @@ -181,9 +203,12 @@ class _ReviewScreenState extends State { names.add(_supermarket!); } final receiptPath = widget.imagePath; - final hasReceipt = receiptPath != null && - receiptPath.isNotEmpty && - File(receiptPath).existsSync(); + final highlighted = _highlightedItemId == null + ? null + : _items.cast().firstWhere( + (item) => item?.id == _highlightedItemId, + orElse: () => null, + ); return Scaffold( appBar: AppBar( @@ -207,7 +232,7 @@ class _ReviewScreenState extends State { child: Padding( padding: const EdgeInsets.all(12), child: Text( - 'Parsed totals do not match the receipt. Check the items before saving.', + 'Item count or totals do not match the receipt, or some lines need manual review. Check highlighted items before saving.', style: theme.textTheme.bodyMedium?.copyWith( color: theme.colorScheme.onErrorContainer, ), @@ -325,6 +350,29 @@ class _ReviewScreenState extends State { }); }, child: ListTile( + selected: _highlightedItemId == item.id, + selectedTileColor: theme.colorScheme.primaryContainer + .withValues(alpha: 0.35), + leading: item.sourceBounds != null && _hasReceipt + ? IconButton( + icon: Icon( + _highlightedItemId == item.id + ? Icons.visibility + : Icons.visibility_outlined, + size: 20, + ), + tooltip: 'Show on receipt', + onPressed: _saving + ? null + : () => _toggleHighlight(item), + ) + : item.needsManualReview + ? Icon( + Icons.flag_outlined, + color: theme.colorScheme.error, + size: 20, + ) + : null, title: Text(item.name), subtitle: _itemSubtitle(item), trailing: DiscountedPrice(item: item), @@ -337,24 +385,30 @@ class _ReviewScreenState extends State { ], ), ), - if (hasReceipt) + if (_hasReceipt) ExpansionTile( + initiallyExpanded: highlighted?.sourceBounds != null, title: const Text('See receipt'), children: [ Padding( padding: const EdgeInsets.fromLTRB(16, 0, 16, 16), - child: GestureDetector( - onTap: _openReceipt, - child: ClipRRect( - borderRadius: BorderRadius.circular(12), - child: Image.file( - File(receiptPath), - fit: BoxFit.contain, - height: 220, - width: double.infinity, - ), - ), - ), + child: highlighted?.sourceBounds != null + ? ReceiptHighlightOverlay( + imagePath: receiptPath!, + bounds: highlighted!.sourceBounds!, + ) + : GestureDetector( + onTap: _openReceipt, + child: ClipRRect( + borderRadius: BorderRadius.circular(12), + child: Image.file( + File(receiptPath!), + fit: BoxFit.contain, + height: 220, + width: double.infinity, + ), + ), + ), ), ], ), diff --git a/lib/scan_screen.dart b/lib/scan_screen.dart index 312d681..1394b7f 100644 --- a/lib/scan_screen.dart +++ b/lib/scan_screen.dart @@ -2,6 +2,7 @@ library; import 'dart:io'; +import 'dart:ui' as ui; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; @@ -102,12 +103,22 @@ class _ScanScreenState extends State { }); final recognizer = TextRecognizer(script: TextRecognitionScript.latin); + RecognizedText? recognized; + double imageWidth = 0; + double imageHeight = 0; try { + final bytes = await File(imagePath).readAsBytes(); + final codec = await ui.instantiateImageCodec(bytes); + final frame = await codec.getNextFrame(); + imageWidth = frame.image.width.toDouble(); + imageHeight = frame.image.height.toDouble(); + frame.image.dispose(); + final inputImage = InputImage.fromFilePath(imagePath); - final recognized = await recognizer.processImage(inputImage); + recognized = await recognizer.processImage(inputImage); if (!mounted) return; setState(() { - _extractedText = recognized.text.trim(); + _extractedText = recognized!.text.trim(); if (_extractedText.isEmpty) { _error = 'No text found in this image.'; } @@ -131,8 +142,14 @@ class _ScanScreenState extends State { } } - if (mounted && _extractedText.isNotEmpty) { - await _openReview(_extractedText); + if (mounted && _extractedText.isNotEmpty && recognized != null) { + await _openReview( + _extractedText, + imagePath: imagePath, + recognized: recognized, + imageWidth: imageWidth, + imageHeight: imageHeight, + ); } } @@ -150,11 +167,22 @@ class _ScanScreenState extends State { String rawText, { List? items, String? imagePath, + RecognizedText? recognized, + double? imageWidth, + double? imageHeight, }) async { final parsedItems = items; ReceiptParseResult? parsed; if (parsedItems == null) { - parsed = ReceiptParser.parseReceipt(rawText); + if (recognized != null && imageWidth != null && imageHeight != null) { + parsed = ReceiptParser.parseRecognizedText( + recognized, + imageWidth: imageWidth, + imageHeight: imageHeight, + ); + } else { + parsed = ReceiptParser.parseReceipt(rawText); + } } if (!mounted) return; final names = @@ -170,6 +198,8 @@ class _ScanScreenState extends State { supermarketNames: names.toList(), needsReview: parsed != null && !parsed.validationPassed, detectedStore: parsed?.store, + ocrImageWidth: parsed?.ocrImageWidth, + ocrImageHeight: parsed?.ocrImageHeight, ), ), ); diff --git a/lib/services/receipt_parse_config.dart b/lib/services/receipt_parse_config.dart index 10e9a49..cd9bc6b 100644 --- a/lib/services/receipt_parse_config.dart +++ b/lib/services/receipt_parse_config.dart @@ -25,6 +25,8 @@ class ReceiptParseConfig { required this.trailingCategoryLetter, required this.noiseKeywords, required this.itemCountKeywords, + this.rowClusterHeightFactor = 1.2, + this.priceColumnWidthFraction = 0.25, }); final List knownStores; @@ -44,6 +46,8 @@ class ReceiptParseConfig { final RegExp trailingCategoryLetter; final List noiseKeywords; final List itemCountKeywords; + final double rowClusterHeightFactor; + final double priceColumnWidthFraction; factory ReceiptParseConfig.fromJson(Map json) { List list(String section, [String key = 'list']) { @@ -99,9 +103,31 @@ class ReceiptParseConfig { ), noiseKeywords: list('noise_footer_keywords'), itemCountKeywords: list('line_item_count_keywords'), + rowClusterHeightFactor: _spatialDouble( + json, + 'row_cluster_height_factor', + fallback: 1.2, + ), + priceColumnWidthFraction: _spatialDouble( + json, + 'price_column_width_fraction', + fallback: 0.25, + ), ); } + static double _spatialDouble( + Map json, + String key, { + required double fallback, + }) { + final spatial = json['spatial_parsing']; + if (spatial is! Map) return fallback; + final value = spatial[key]; + if (value is num && value > 0) return value.toDouble(); + return fallback; + } + static const assetPath = 'receipt-parsing-keywords.json'; /// True if [line] matches a config keyword. diff --git a/lib/services/receipt_parser.dart b/lib/services/receipt_parser.dart index b08655f..b8e4830 100644 --- a/lib/services/receipt_parser.dart +++ b/lib/services/receipt_parser.dart @@ -1,609 +1,42 @@ -/// Turns raw OCR text into [LineItem]s using receipt-parsing-keywords.json. +/// Receipt parsing entry point: spatial OCR pipeline with text fallback. library; +import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart'; + import '../models/line_item.dart'; -import '../utils/money.dart'; +import '../models/ocr_element.dart'; +import 'receipt_ocr.dart'; import 'receipt_parse_config.dart'; - export 'receipt_parse_config.dart' show loadReceiptParseConfig; - -/// Result of a full receipt parse, including totals used for review. -class ReceiptParseResult { - const ReceiptParseResult({ - required this.items, - this.store, - this.receiptTotal, - this.totalDiscount = 0, - this.validationPassed = true, - }); - - final List items; - final String? store; - final double? receiptTotal; - final double totalDiscount; - final bool validationPassed; -} - -class _Qty { - const _Qty({required this.count, required this.unitPrice, this.total}); - - final int count; - final double unitPrice; - final double? total; - - double get lineTotal => total ?? roundMoney(count * unitPrice); -} - -class _Pending { - _Pending({ - required this.name, - this.unitPrice, - this.quantity = 1, - this.lineTotal, - }); - - String name; - double? unitPrice; - int quantity; - double? lineTotal; - double discount = 0; -} - -enum _Kind { - skip, - item, - nameOnly, - quantity, - discount, - grandTotal, - totalDiscount, - priceOnly, -} - -class _Line { - const _Line({ - required this.kind, - required this.raw, - this.name, - this.amount, - this.qty, - }); - - final _Kind kind; - final String raw; - final String? name; - final double? amount; - final _Qty? qty; -} - -final _postalPattern = RegExp(r'\d{4}\s*[A-Za-z]{2}\b'); -final _phonePattern = RegExp(r'^\d{8,}$'); -final _streetPattern = RegExp(r"^[A-Za-z. ''-]+\s+\d{1,4}[A-Za-z]?$"); -final _urlPattern = RegExp(r'www\.|\.com|\.nl', caseSensitive: false); - -/// Jumbo-style OCR often adds a line total after `N X unit`. -final _qtyWithTotal = RegExp( - r'^\s*(\d+)\s*[xX×]\s*(-?\d+[,.]\d{2})(?:\s+(?:PER\s+STUK|PER\s+KG|ST\.?|STUKS?))?(?:\s+(-?\d+[,.]\d{2}))?\s*$', - caseSensitive: false, -); +export 'receipt_text_parser.dart' show ReceiptParseResult; +import 'receipt_spatial_parser.dart'; +import 'receipt_text_parser.dart'; class ReceiptParser { ReceiptParser._(); - static int _nextId = 0; - - static String _id() => 'item-${++_nextId}'; - static Future loadConfig() => loadReceiptParseConfig(); static List parse(String rawText) => parseReceipt(rawText).items; - static ReceiptParseResult parseReceipt(String rawText) { - final config = receiptParseConfig; - final lines = rawText - .split('\n') - .map(_preprocess) - .where((line) => line.isNotEmpty) - .toList(); + /// Fallback when only flattened OCR text is available (tests, re-parse). + static ReceiptParseResult parseReceipt(String rawText) => + ReceiptTextParser.parseReceipt(rawText); - if (_looksLikeColumnOcr(lines, config)) { - return _parseColumns(lines, config, rawText); - } - return _parseSequential(lines, config, rawText); - } + /// Primary path: positioned ML Kit elements → row clustering → lookahead. + static ReceiptParseResult parseReceiptFromOcr(OcrDocument document) => + ReceiptSpatialParser.parse(document); - static String _preprocess(String line) => - line.trim().replaceAll(RegExp(r'\s+'), ' '); - - static String _stripTax(String line, ReceiptParseConfig config) { - return line.replaceFirst(config.trailingCategoryLetter, '').trim(); - } - - static bool _looksLikeColumnOcr( - List lines, - ReceiptParseConfig config, - ) { - var run = 0; - var maxRun = 0; - var prices = 0; - var names = 0; - for (final line in lines) { - final stripped = _stripTax(line, config); - if (isPriceOnly(stripped)) { - prices++; - run++; - if (run > maxRun) maxRun = run; - } else { - run = 0; - if (_parseQty(stripped, config) == null && - !_isIgnorable(stripped, config) && - !config.matchesAny(stripped, [ - ...config.grandTotalKeywords, - ...config.subtotalKeywords, - ...config.taxInclusiveTotalKeywords, - ...config.totalDiscountKeywords, - ]) && - _trailingPrice(stripped, config) == null) { - names++; - } - } - } - if (maxRun >= 4 && prices >= 4 && names >= 4) return true; - // ML Kit sometimes splits a short receipt into names, then prices (2–3 items). - return maxRun >= 2 && prices >= 2 && names >= prices && maxRun >= prices; - } - - static ReceiptParseResult _parseSequential( - List lines, - ReceiptParseConfig config, - String rawText, - ) { - final items = []; - _Pending? pending; - var receiptTotal = _readAmountAfterKeyword( - lines, - config, - config.grandTotalKeywords, - skipIf: (line) => config.matchesAny(line, [ - ...config.totalDiscountKeywords, - ...config.taxInclusiveTotalKeywords, - ...config.subtotalKeywords, - ]), - ); - var totalDiscount = 0.0; - var stop = false; - - void flush() { - final current = pending; - pending = null; - if (current == null) return; - final total = current.lineTotal ?? current.unitPrice; - if (total == null) return; - items.add(_toItem(current, total)); - } - - for (var i = 0; i < lines.length && !stop; i++) { - final classified = _classify(lines[i], config); - switch (classified.kind) { - case _Kind.skip: - continue; - case _Kind.item: - flush(); - pending = _Pending( - name: classified.name!, - unitPrice: classified.amount, - quantity: classified.qty?.count ?? 1, - lineTotal: classified.qty == null - ? classified.amount - : classified.qty!.lineTotal, - ); - if (classified.qty != null) { - pending!.unitPrice = classified.qty!.unitPrice; - } - case _Kind.nameOnly: - flush(); - pending = _Pending(name: classified.name ?? classified.raw); - case _Kind.quantity: - final qty = classified.qty!; - if (pending != null) { - pending! - ..quantity = qty.count - ..unitPrice = qty.unitPrice - ..lineTotal = qty.lineTotal; - } - case _Kind.discount: - var amount = classified.amount; - if (amount == null && i + 1 < lines.length) { - amount = tryParsePrice(_stripTax(lines[i + 1], config)); - if (amount != null) i++; - } - if (amount == null) continue; - if (amount > 0) amount = -amount; - if (pending != null) { - pending!.discount += amount; - pending!.lineTotal = roundMoney( - (pending!.lineTotal ?? pending!.unitPrice ?? 0) + amount, - ); - } else { - totalDiscount += amount; - } - case _Kind.totalDiscount: - var amount = classified.amount; - if (amount == null && i + 1 < lines.length) { - amount = tryParsePrice(_stripTax(lines[i + 1], config)); - if (amount != null) i++; - } - if (amount != null) { - totalDiscount += amount > 0 ? -amount : amount; - } - case _Kind.grandTotal: - flush(); - if (classified.amount != null) { - receiptTotal = classified.amount; - stop = true; - } - case _Kind.priceOnly: - final amount = classified.amount!; - if (pending != null) { - if (amount < 0) { - pending!.discount += amount; - pending!.lineTotal = roundMoney( - (pending!.lineTotal ?? pending!.unitPrice ?? 0) + amount, - ); - } else if (pending!.lineTotal == null && - pending!.unitPrice == null) { - pending!.lineTotal = amount; - flush(); - } - } - } - } - flush(); - - return _result( - items: items, - rawText: rawText, - receiptTotal: receiptTotal, - totalDiscount: totalDiscount, - ); - } - - static ReceiptParseResult _parseColumns( - List lines, - ReceiptParseConfig config, - String rawText, - ) { - final names = <({String name, int? quantity, double? unitPrice})>[]; - final prices = []; - final discountSlots = []; - final discounts = []; - - for (final raw in lines) { - final line = _stripTax(raw, config); - if (isPriceOnly(line)) { - final price = tryParsePrice(line); - if (price == null) continue; - if (price < 0) { - discounts.add(price); - } else { - prices.add(price); - } - continue; - } - if (_isIgnorable(line, config)) continue; - if (config.matchesAny(line, [ - ...config.grandTotalKeywords, - ...config.subtotalKeywords, - ...config.taxInclusiveTotalKeywords, - ...config.totalDiscountKeywords, - ])) { - continue; - } - - final qty = _parseQty(line, config); - if (qty != null) { - if (names.isNotEmpty) { - final last = names.removeLast(); - names.add(( - name: last.name, - quantity: qty.count, - unitPrice: qty.unitPrice, - )); - } - continue; - } - - if (config.matchesAny(line, config.discountKeywords)) { - if (names.isNotEmpty) discountSlots.add(names.length - 1); - continue; - } - - names.add((name: line, quantity: null, unitPrice: null)); - } - - final count = names.length < prices.length ? names.length : prices.length; - final pairedNames = names.length > count - ? names.sublist(names.length - count) - : names; - final nameOffset = names.length - pairedNames.length; - final items = [ - for (var i = 0; i < count; i++) - LineItem( - id: _id(), - name: pairedNames[i].name, - price: prices[i], - quantity: pairedNames[i].quantity, - unitPrice: pairedNames[i].unitPrice, - ), - ]; - - for (var i = 0; i < discountSlots.length && i < discounts.length; i++) { - final index = discountSlots[i] - nameOffset; - if (index < 0 || index >= items.length) continue; - final item = items[index]; - final discount = discounts[i]; - items[index] = item.copyWith( - price: roundMoney(item.price + discount), - discount: roundMoney(item.discount + discount), - ); - } - - final receiptTotal = _readAmountAfterKeyword( - lines, - config, - config.grandTotalKeywords, - skipIf: (line) => config.matchesAny(line, [ - ...config.totalDiscountKeywords, - ...config.taxInclusiveTotalKeywords, - ...config.subtotalKeywords, - ]), - ); - var totalDiscount = discounts.fold(0, (sum, value) => sum + value); - totalDiscount += - _readAmountAfterKeyword(lines, config, config.totalDiscountKeywords) ?? - 0; - - return _result( - items: items, - rawText: rawText, - receiptTotal: receiptTotal, - totalDiscount: totalDiscount, - ); - } - - static LineItem _toItem(_Pending pending, double total) { - return LineItem( - id: _id(), - name: pending.name, - price: roundMoney(total), - quantity: pending.quantity == 1 ? null : pending.quantity, - unitPrice: pending.quantity == 1 ? null : pending.unitPrice, - discount: pending.discount == 0 ? 0 : roundMoney(pending.discount), - ); - } - - static ReceiptParseResult _result({ - required List items, - required String rawText, - double? receiptTotal, - double totalDiscount = 0, + static ReceiptParseResult parseRecognizedText( + RecognizedText recognized, { + required double imageWidth, + required double imageHeight, }) { - final kept = items - .where((item) => item.name.trim().isNotEmpty && item.price > 0) - .toList(); - final sum = roundMoney( - kept.fold(0, (total, item) => total + item.price), + final document = ReceiptOcr.fromRecognizedText( + recognized, + imageWidth: imageWidth, + imageHeight: imageHeight, ); - final passed = - receiptTotal == null || - (sum - receiptTotal).abs() <= 0.01 || - (sum + totalDiscount - receiptTotal).abs() <= 0.01; - return ReceiptParseResult( - items: kept, - store: _detectStore(rawText), - receiptTotal: receiptTotal, - totalDiscount: totalDiscount, - validationPassed: passed, - ); - } - - static String? _detectStore(String rawText) { - final config = receiptParseConfig; - final header = rawText - .split('\n') - .map(_preprocess) - .where((line) => line.isNotEmpty) - .take(8) - .join('\n') - .toUpperCase(); - final stores = [...config.knownStores] - ..sort((a, b) => b.length.compareTo(a.length)); - for (final store in stores) { - if (RegExp('\\b${RegExp.escape(store)}\\b').hasMatch(header)) { - return _canonicalStore(store); - } - } - return null; - } - - static String _canonicalStore(String raw) { - const aliases = { - 'AH': 'Albert Heijn', - 'ALBERT HEIJN': 'Albert Heijn', - 'JUMBO': 'Jumbo', - 'LIDL': 'Lidl', - 'KRUIDVAT': 'Kruidvat', - 'ALDI': 'Aldi', - 'PLUS': 'Plus', - 'DIRK': 'Dirk', - 'COOP': 'Coop', - 'SPAR': 'SPAR', - 'VOMAR': 'Vomar', - 'EKOPLAZA': 'Ekoplaza', - 'HOOGVLIET': 'Hoogvliet', - 'ACTION': 'Action', - 'ETOS': 'Etos', - 'WORLD TOKO': 'World Toko', - }; - return aliases[raw.toUpperCase()] ?? raw; - } - - static double? _readAmountAfterKeyword( - List lines, - ReceiptParseConfig config, - List keywords, { - bool Function(String line)? skipIf, - }) { - for (var i = 0; i < lines.length; i++) { - final line = _stripTax(lines[i], config); - if (!config.matchesAny(line, keywords)) continue; - if (skipIf != null && skipIf(line)) continue; - final trailing = _trailingPrice(line, config); - if (trailing != null) return trailing; - if (i + 1 < lines.length) { - final next = tryParsePrice(_stripTax(lines[i + 1], config)); - if (next != null) return next; - } - } - return null; - } - - static _Line _classify(String raw, ReceiptParseConfig config) { - final line = _stripTax(raw, config); - if (_isIgnorable(line, config)) { - return _Line(kind: _Kind.skip, raw: line); - } - if (config.matchesAny(line, config.totalDiscountKeywords)) { - return _Line( - kind: _Kind.totalDiscount, - raw: line, - amount: _trailingPrice(line, config), - ); - } - if (config.matchesAny(line, config.taxInclusiveTotalKeywords) || - config.matchesAny(line, config.subtotalKeywords)) { - return _Line(kind: _Kind.skip, raw: line); - } - if (config.matchesAny(line, config.grandTotalKeywords)) { - return _Line( - kind: _Kind.grandTotal, - raw: line, - amount: _trailingPrice(line, config), - ); - } - - final qty = _parseQty(line, config); - if (qty != null) { - return _Line(kind: _Kind.quantity, raw: line, qty: qty); - } - - final isDiscount = - config.matchesAny(line, config.discountKeywords) || - config.negativeAmount.hasMatch(line); - if (isDiscount && - (config.matchesAny(line, config.discountKeywords) || - isPriceOnly(line))) { - return _Line( - kind: _Kind.discount, - raw: line, - name: line, - amount: _trailingPrice(line, config) ?? tryParsePrice(line), - ); - } - - if (isPriceOnly(line)) { - return _Line( - kind: _Kind.priceOnly, - raw: line, - amount: tryParsePrice(line), - ); - } - - final inline = config.quantityInline.firstMatch(line); - final trailing = _trailingPrice(line, config); - if (trailing != null) { - var name = line.replaceFirst(RegExp(r'\s+-?\d+[.,]\d{2}\s*$'), '').trim(); - _Qty? inlineQty; - if (inline != null) { - name = line.substring(0, inline.start).trim(); - final count = int.tryParse(inline.group(1)!); - final unit = tryParsePrice(inline.group(2)!); - if (count != null && unit != null) { - inlineQty = _Qty(count: count, unitPrice: unit, total: trailing); - } - } - if (name.isEmpty || _isIgnorable(name, config)) { - return _Line(kind: _Kind.skip, raw: line); - } - return _Line( - kind: _Kind.item, - raw: line, - name: name, - amount: trailing, - qty: inlineQty, - ); - } - - if (inline != null) { - final name = line.substring(0, inline.start).trim(); - final count = int.tryParse(inline.group(1)!); - final unit = tryParsePrice(inline.group(2)!); - if (name.isNotEmpty && count != null && unit != null) { - return _Line( - kind: _Kind.item, - raw: line, - name: name, - amount: roundMoney(count * unit), - qty: _Qty(count: count, unitPrice: unit), - ); - } - } - - return _Line(kind: _Kind.nameOnly, raw: line, name: line); - } - - static bool _isIgnorable(String line, ReceiptParseConfig config) { - if (config.separatorPattern.hasMatch(line)) return true; - if (_urlPattern.hasMatch(line)) return true; - if (_postalPattern.hasMatch(line)) return true; - if (_phonePattern.hasMatch(line.replaceAll(RegExp(r'\s'), ''))) { - return true; - } - if (_streetPattern.hasMatch(line)) return true; - if (config.categoryRow.hasMatch(line)) return true; - if (config.isStandaloneKeywordLine(line, config.columnHeaderKeywords)) { - return true; - } - if (config.isStandaloneKeywordLine(line, config.itemCountKeywords)) { - return true; - } - if (config.matchesAny(line, config.taxKeywords)) return true; - if (config.matchesAny(line, config.paymentKeywords)) return true; - if (config.matchesAny(line, config.noiseKeywords)) return true; - return false; - } - - static double? _trailingPrice(String line, ReceiptParseConfig config) { - final match = RegExp(r'(-?\d+[.,]\d{2})\s*$').firstMatch(line); - if (match == null) return null; - return tryParsePrice(match.group(1)!); - } - - static _Qty? _parseQty(String line, ReceiptParseConfig config) { - final match = - _qtyWithTotal.firstMatch(line) ?? - config.quantityStandalone.firstMatch(line); - if (match == null) return null; - final count = int.tryParse(match.group(1)!); - final unit = tryParsePrice(match.group(2)!); - if (count == null || count < 1 || unit == null) return null; - double? total; - if (match.groupCount >= 3) { - final raw = match.group(3); - if (raw != null && RegExp(r'^-?\d+[.,]\d{2}$').hasMatch(raw)) { - total = tryParsePrice(raw); - } - } - return _Qty(count: count, unitPrice: unit, total: total); + return parseReceiptFromOcr(document); } } diff --git a/lib/widgets/item_edit_sheet.dart b/lib/widgets/item_edit_sheet.dart index 0f0fef9..ab8a1ce 100644 --- a/lib/widgets/item_edit_sheet.dart +++ b/lib/widgets/item_edit_sheet.dart @@ -170,6 +170,8 @@ class _ItemEditSheetState extends State { discount: widget.item?.price == roundMoney(resolvedTotal) ? widget.item?.discount ?? 0 : 0, + sourceBounds: widget.item?.sourceBounds, + needsManualReview: false, ), ); } diff --git a/receipt-parsing-keywords.json b/receipt-parsing-keywords.json index 729c5e0..4cf2646 100644 --- a/receipt-parsing-keywords.json +++ b/receipt-parsing-keywords.json @@ -182,5 +182,11 @@ "ART.", "ARTIKELEN" ] + }, + + "spatial_parsing": { + "_comment": "Bounding-box pipeline settings. row_cluster_height_factor multiplies median element height when grouping rows. price_column_width_fraction is the right-hand slice of receipt content width treated as the price column.", + "row_cluster_height_factor": 1.2, + "price_column_width_fraction": 0.25 } } diff --git a/receipt-parsing-spec.md b/receipt-parsing-spec.md index 1478b2b..21c04ce 100644 --- a/receipt-parsing-spec.md +++ b/receipt-parsing-spec.md @@ -16,30 +16,53 @@ without assuming a single fixed format. ## 1. Input assumptions -- Input is raw OCR text, one line per detected text line, in top-to-bottom - order as it appeared on the receipt. +- Input is structured OCR from ML Kit: **blocks → lines → elements**, each + element carrying its own pixel bounding box (`left`, `top`, `right`, + `bottom`) on the receipt image. Do **not** rely on the flattened `.text` + property for parsing — it discards the two-column layout. - Prices use European format: comma as decimal separator (e.g. `2,89`), period as thousands separator if present (rare on receipts this size). - OCR may introduce noise: misread characters, extra/missing spaces, occasional dropped or merged lines. The parser should be tolerant of minor spacing irregularities but does not need to do spell-correction. +- Receipt photos vary in resolution and crop; all column thresholds must be + computed **relative to the detected content width**, not as fixed pixels. ## 2. Preprocessing (run before classification) -1. Trim leading/trailing whitespace on every line; collapse multiple - internal spaces to one. -2. Normalize decimal separators: treat `,` as the decimal point in any - number matching `\d+[,.]\d{2}`. Convert to a standard `.` internally - for numeric storage, but remember the original format was comma-based - (useful for locale-aware re-parsing if needed). -3. Drop fully empty lines. -4. Do NOT drop short lines outright (e.g. a lone `BROCCOLI` or a lone +1. Extract every text **element** with its bounding box from ML Kit. +2. **Rebuild rows** — do not trust ML Kit's line segmentation: + - Sort elements by Y-center (top to bottom). + - Cluster into rows where Y-centers fall within one line height of each + other. Threshold = `median_element_height × row_cluster_height_factor` + (default factor **1.2**, configurable in `spatial_parsing`). + - Within each row, sort elements left-to-right by X position. +3. Join element text within a row (single spaces) for keyword/regex checks. +4. Trim leading/trailing whitespace; collapse multiple internal spaces to one. +5. Normalize decimal separators: treat `,` as the decimal point in any number + matching `\d+[,.]\d{2}`. Convert to `.` internally for storage. +6. Do NOT drop short rows outright (e.g. a lone `BROCCOLI` or a lone `2 X 0,95`) — both are meaningful in different receipt layouts. -## 3. Core line classification +## 3. Core row classification -Classify every line into one of these categories, in this priority order -(check top-down, first match wins): +Classify every **row** (not flat OCR line) into one of these categories. +Within a row, use **X-position column logic** alongside regex: + +- Compute content width = rightmost − leftmost element across the receipt. +- **Price column** = rightmost `price_column_width_fraction` (default **25%**) + of that width. A row has a price when a token in the price column matches + `\d+[,.]\d{2}`. Everything left of the price column is the candidate name. +- A row with no price-column match is **name-only**. + +Check categories in this priority order (first match wins): + +### 3.0 QUANTITY_MODIFIER (checked first — unconditional) + +- If the **entire row** matches (not a substring): + `^\d+\s*[xX]\s*\d+[,.]\d{2}\s*(PER\s+STUK|PER\s+KG|ST\.?|STUKS?)?$` +- Classify as QUANTITY_MODIFIER **unconditionally**. A pure `N X price` row + can never be a standalone product, even if a price-column token is present. ### 3.1 STORE_HEADER - The first 1–5 lines of the receipt, before the first recognizable @@ -72,13 +95,13 @@ Classify every line into one of these categories, in this priority order - This is the "complete" case: name + price on the same line, quantity of 1 implied unless a QUANTITY_MODIFIER line follows (section 3.4). -### 3.4 QUANTITY_MODIFIER (sub-line, belongs to previous or next ITEM) -- A line matching the pattern: ` X ` optionally followed by - a unit word (e.g. `PER STUK`), e.g.: - - `2 X 2,79` - - `2 X 3,99 PER STUK` -- This line does NOT introduce a new product — it modifies the item on - an adjacent line. +### 3.4 QUANTITY_MODIFIER (sub-row, belongs to previous or next ITEM) + +See **§3.0** — pure `N X price` rows are detected before any other type. +The rules below describe how they attach to items: + +- This row does NOT introduce a new product — it modifies the item on + an adjacent row. - Two observed variants — the parser must detect which applies: - **Variant A (Jumbo-style):** product name appears on its own line with NO price. The next line is `N X unit_price`. The line total @@ -154,66 +177,71 @@ Classify every line into one of these categories, in this priority order ## 4. Item construction algorithm (pseudocode) +Uses a **one-row lookahead buffer** — do not emit an item until the next +row is inspected (or the receipt ends). + ``` items = [] pending_item = None # name found, price/qty not yet resolved -for line in classified_lines: - if line.type == ITEM: - if pending_item: - items.append(pending_item) # close out previous item first - pending_item = { name: line.name, unit_price: line.price, - quantity: 1, line_total: line.price, - discount: 0 } - - elif line.type == QUANTITY_MODIFIER: - if pending_item and pending_item.unit_price is None: - # Variant A: name-only line was pending, this line completes it - pending_item.quantity = line.qty - pending_item.unit_price = line.price - pending_item.line_total = line.qty * line.price - elif pending_item: - # Variant B: name+price already set, this line refines/ - # confirms it (overwrite quantity/unit price, recompute total) - pending_item.quantity = line.qty - pending_item.unit_price = line.price - pending_item.line_total = line.qty * line.price - else: - # Orphan quantity line with no preceding item — flag for - # manual review rather than silently dropping - flag_for_review(line) - - elif line.type == DISCOUNT: - if pending_item: - pending_item.discount += line.amount # amount is negative - pending_item.line_total += line.amount - else: - receipt.total_discount += line.amount - - elif line.type in (SUBTOTAL, TOTAL): +for row in classified_rows: + if row.type == QUANTITY_MODIFIER: if pending_item: + pending_item.quantity = row.qty + pending_item.unit_price = row.price + pending_item.line_total = row.qty * row.price + merge bounding boxes from row into pending_item items.append(pending_item) pending_item = None - if is_grand_total(line): - receipt.total = line.amount - break # stop item parsing here + continue # orphan modifier without pending → ignore -# catch any trailing pending item if receipt ended oddly -if pending_item: - items.append(pending_item) + if row.type == NAME_ONLY: + if pending_item: + items.append(pending_item with needs_review if no price) + pending_item = { name: row.name, unit_price: None, quantity: 1, + line_total: None, discount: 0, bounds: row.bounds } + peek next row: + if next.type == QUANTITY_MODIFIER → defer emit (handled above) + else → append pending_item flagged needs_review, clear pending + continue + + if row.type == ITEM: + if pending_item: + items.append(pending_item with needs_review if no price) + pending_item = { name: row.name, unit_price: row.price, + quantity: 1, line_total: row.price, + discount: 0, bounds: row.bounds } + peek next row: + if next.type == QUANTITY_MODIFIER → defer emit (modifier overrides qty/unit) + else → append pending_item, clear pending + continue + + if row.type == DISCOUNT: + ... (unchanged: attach to pending_item or receipt.total_discount) + + if row.type in (SUBTOTAL, TOTAL): + flush pending_item + if is_grand_total(row): + receipt.total = row.amount + break + +flush any trailing pending_item ``` +Each emitted item keeps **source bounding box(es)** (normalized 0–1 on the +receipt image) for highlight overlays in the review UI. + ## 5. Validation pass (do this after parsing, before showing results to user) 1. Sum `line_total` across all parsed items. 2. Add `receipt.total_discount` if it wasn't already folded into individual item discounts. 3. Compare the result to `receipt.total` (the grand total line). -4. If they match within 0.01 → high confidence, no flags needed. -5. If they don't match → mark the WHOLE receipt as "needs review" and - surface it clearly in the review/edit screen (don't just flag - individual lines — a mismatch means something in the whole parse is - probably off, e.g. a missed line or a misclassified discount). +4. If the receipt prints an item count (e.g. `Aantal Artikel: N`), compare + `N` to the number of parsed items. +5. If totals match within 0.01 **and** item count matches (when present) + **and** no item is flagged `needs_manual_review` → high confidence. +6. If any check fails → mark the **whole receipt** as "needs review". ## 6. Known store-specific quirks to encode as configurable rules