parser logic added

This commit is contained in:
davmar
2026-08-30 12:24:29 +02:00
parent 35d5c90d81
commit 3b698a3667
28 changed files with 3018 additions and 346 deletions
+499 -113
View File
@@ -1,57 +1,31 @@
/// Turns raw OCR text into [LineItem]s using line-end prices and a keyword filter.
/// Turns raw OCR text into [LineItem]s using receipt-parsing-keywords.json.
library;
import '../models/line_item.dart';
import '../utils/money.dart';
import 'receipt_parse_config.dart';
/// Headers, payments, and totals — never treated as products.
const Set<String> kHeaderKeywords = {
'TOTAL',
'SUBTOTAL',
'TOTAAL',
'TAX',
'VAT',
'BTW',
'CASH',
'CARD',
'CHANGE',
'VISA',
'MASTERCARD',
'MAESTRO',
'PIN',
'DEBIT',
'CREDIT',
'OMSCHRIJVING',
'BEDRAG',
'AMOUNT',
'DESCRIPTION',
'THANK',
'BEDANKT',
'WISSELGELD',
'BETAALD',
'BETAAL',
'CONTANT',
'KASSA',
'RECEIPT',
'WWW',
'HTTP',
};
export 'receipt_parse_config.dart' show loadReceiptParseConfig;
/// Discount / promo rows. Paired with a price, then dropped.
const Set<String> kDropKeywords = {
'DISCOUNT',
'KORTING',
'ACTIE',
};
/// Result of a full receipt parse, including totals used for review.
class ReceiptParseResult {
const ReceiptParseResult({
required this.items,
this.store,
this.receiptTotal,
this.totalDiscount = 0,
this.validationPassed = true,
});
final _separatorPattern = RegExp(r'^[=\-_.:\s]+$');
final _postalPattern = RegExp(r'\d{4}\s*[A-Za-z]{2}\b');
final _phonePattern = RegExp(r'^\d{8,}$');
final _streetPattern = RegExp(r"^[A-Za-z. ''-]+\s+\d{1,4}$");
final _urlPattern = RegExp(r'www\.|\.com|\.nl', caseSensitive: false);
final List<LineItem> items;
final String? store;
final double? receiptTotal;
final double totalDiscount;
final bool validationPassed;
}
class _QtyLine {
const _QtyLine({required this.count, required this.unitPrice, this.total});
class _Qty {
const _Qty({required this.count, required this.unitPrice, this.total});
final int count;
final double unitPrice;
@@ -60,6 +34,59 @@ class _QtyLine {
double get lineTotal => total ?? roundMoney(count * unitPrice);
}
class _Pending {
_Pending({
required this.name,
this.unitPrice,
this.quantity = 1,
this.lineTotal,
});
String name;
double? unitPrice;
int quantity;
double? lineTotal;
double discount = 0;
}
enum _Kind {
skip,
item,
nameOnly,
quantity,
discount,
grandTotal,
totalDiscount,
priceOnly,
}
class _Line {
const _Line({
required this.kind,
required this.raw,
this.name,
this.amount,
this.qty,
});
final _Kind kind;
final String raw;
final String? name;
final double? amount;
final _Qty? qty;
}
final _postalPattern = RegExp(r'\d{4}\s*[A-Za-z]{2}\b');
final _phonePattern = RegExp(r'^\d{8,}$');
final _streetPattern = RegExp(r"^[A-Za-z. ''-]+\s+\d{1,4}[A-Za-z]?$");
final _urlPattern = RegExp(r'www\.|\.com|\.nl', caseSensitive: false);
/// Jumbo-style OCR often adds a line total after `N X unit`.
final _qtyWithTotal = RegExp(
r'^\s*(\d+)\s*[xX×]\s*(-?\d+[,.]\d{2})(?:\s+(?:PER\s+STUK|PER\s+KG|ST\.?|STUKS?))?(?:\s+(-?\d+[,.]\d{2}))?\s*$',
caseSensitive: false,
);
class ReceiptParser {
ReceiptParser._();
@@ -67,74 +94,198 @@ class ReceiptParser {
static String _id() => 'item-${++_nextId}';
/// Parses [rawText] into product rows.
///
/// Prefers same-line `name … price` (with a following `2 x 1.50` line when
/// the name has no price). If OCR split the receipt into a name column and a
/// price column, falls back to pairing those columns.
static List<LineItem> parse(String rawText) {
static Future<void> loadConfig() => loadReceiptParseConfig();
static List<LineItem> parse(String rawText) => parseReceipt(rawText).items;
static ReceiptParseResult parseReceipt(String rawText) {
final config = receiptParseConfig;
final lines = rawText
.split('\n')
.map((line) => line.trim())
.map(_preprocess)
.where((line) => line.isNotEmpty)
.toList();
final inline = _parseInline(lines);
final columns = _parseColumns(lines);
final chosen = columns.length > inline.length ? columns : inline;
return chosen.where(_keepItem).toList();
if (_looksLikeColumnOcr(lines, config)) {
return _parseColumns(lines, config, rawText);
}
return _parseSequential(lines, config, rawText);
}
static List<LineItem> _parseInline(List<String> lines) {
final items = <LineItem>[];
static String _preprocess(String line) =>
line.trim().replaceAll(RegExp(r'\s+'), ' ');
for (var i = 0; i < lines.length; i++) {
final line = lines[i];
if (_shouldSkipLine(line) || isPriceOnly(line)) continue;
static String _stripTax(String line, ReceiptParseConfig config) {
return line.replaceFirst(config.trailingCategoryLetter, '').trim();
}
final qty = _parseQty(line);
if (qty != null) continue;
final trailing = trailingPricePattern.firstMatch(line);
if (trailing != null) {
final name = trailing.group(1)!.trim();
final price = tryParsePrice(trailing.group(2)!);
if (name.isEmpty || price == null || _shouldSkipLine(name)) continue;
items.add(LineItem(id: _id(), name: name, price: price));
continue;
static bool _looksLikeColumnOcr(
List<String> lines,
ReceiptParseConfig config,
) {
var run = 0;
var maxRun = 0;
var prices = 0;
var names = 0;
for (final line in lines) {
final stripped = _stripTax(line, config);
if (isPriceOnly(stripped)) {
prices++;
run++;
if (run > maxRun) maxRun = run;
} else {
run = 0;
if (_parseQty(stripped, config) == null &&
!_isIgnorable(stripped, config)) {
names++;
}
}
}
return maxRun >= 4 && prices >= 4 && names >= 4;
}
if (i + 1 >= lines.length) continue;
final nextQty = _parseQty(lines[i + 1]);
if (nextQty == null) continue;
items.add(
LineItem(
id: _id(),
name: line,
price: nextQty.lineTotal,
quantity: nextQty.count,
unitPrice: nextQty.unitPrice,
),
);
i++;
static ReceiptParseResult _parseSequential(
List<String> lines,
ReceiptParseConfig config,
String rawText,
) {
final items = <LineItem>[];
_Pending? pending;
var receiptTotal = _readAmountAfterKeyword(
lines,
config,
config.grandTotalKeywords,
skipIf: (line) => config.matchesAny(line, [
...config.totalDiscountKeywords,
...config.taxInclusiveTotalKeywords,
...config.subtotalKeywords,
]),
);
var totalDiscount = 0.0;
var stop = false;
void flush() {
final current = pending;
pending = null;
if (current == null) return;
final total = current.lineTotal ?? current.unitPrice;
if (total == null) return;
items.add(_toItem(current, total));
}
return items;
for (var i = 0; i < lines.length && !stop; i++) {
final classified = _classify(lines[i], config);
switch (classified.kind) {
case _Kind.skip:
continue;
case _Kind.item:
flush();
pending = _Pending(
name: classified.name!,
unitPrice: classified.amount,
quantity: classified.qty?.count ?? 1,
lineTotal: classified.qty == null
? classified.amount
: classified.qty!.lineTotal,
);
if (classified.qty != null) {
pending!.unitPrice = classified.qty!.unitPrice;
}
case _Kind.nameOnly:
flush();
pending = _Pending(name: classified.name ?? classified.raw);
case _Kind.quantity:
final qty = classified.qty!;
if (pending != null) {
pending!
..quantity = qty.count
..unitPrice = qty.unitPrice
..lineTotal = qty.lineTotal;
}
case _Kind.discount:
var amount = classified.amount;
if (amount == null && i + 1 < lines.length) {
amount = tryParsePrice(_stripTax(lines[i + 1], config));
if (amount != null) i++;
}
if (amount == null) continue;
if (amount > 0) amount = -amount;
if (pending != null) {
pending!.discount += amount;
pending!.lineTotal = roundMoney(
(pending!.lineTotal ?? pending!.unitPrice ?? 0) + amount,
);
} else {
totalDiscount += amount;
}
case _Kind.totalDiscount:
var amount = classified.amount;
if (amount == null && i + 1 < lines.length) {
amount = tryParsePrice(_stripTax(lines[i + 1], config));
if (amount != null) i++;
}
if (amount != null) {
totalDiscount += amount > 0 ? -amount : amount;
}
case _Kind.grandTotal:
flush();
if (classified.amount != null) {
receiptTotal = classified.amount;
stop = true;
}
case _Kind.priceOnly:
final amount = classified.amount!;
if (amount < 0 && pending != null) {
pending!.discount += amount;
pending!.lineTotal = roundMoney(
(pending!.lineTotal ?? pending!.unitPrice ?? 0) + amount,
);
}
}
}
flush();
return _result(
items: items,
rawText: rawText,
receiptTotal: receiptTotal,
totalDiscount: totalDiscount,
);
}
static List<LineItem> _parseColumns(List<String> lines) {
static ReceiptParseResult _parseColumns(
List<String> lines,
ReceiptParseConfig config,
String rawText,
) {
final names = <({String name, int? quantity, double? unitPrice})>[];
final prices = <double>[];
final discountSlots = <int>[];
final discounts = <double>[];
for (final line in lines) {
for (final raw in lines) {
final line = _stripTax(raw, config);
if (isPriceOnly(line)) {
final price = tryParsePrice(line);
if (price != null) prices.add(price);
if (price == null) continue;
if (price < 0) {
discounts.add(price);
} else {
prices.add(price);
}
continue;
}
if (_isIgnorable(line, config)) continue;
if (config.matchesAny(line, [
...config.grandTotalKeywords,
...config.subtotalKeywords,
...config.taxInclusiveTotalKeywords,
...config.totalDiscountKeywords,
])) {
continue;
}
if (_shouldSkipLine(line)) continue;
final qty = _parseQty(line);
final qty = _parseQty(line, config);
if (qty != null) {
if (names.isNotEmpty) {
final last = names.removeLast();
@@ -147,11 +298,16 @@ class ReceiptParser {
continue;
}
if (config.matchesAny(line, config.discountKeywords)) {
if (names.isNotEmpty) discountSlots.add(names.length - 1);
continue;
}
names.add((name: line, quantity: null, unitPrice: null));
}
final count = names.length < prices.length ? names.length : prices.length;
return [
final items = <LineItem>[
for (var i = 0; i < count; i++)
LineItem(
id: _id(),
@@ -161,44 +317,274 @@ class ReceiptParser {
unitPrice: names[i].unitPrice,
),
];
for (var i = 0; i < discountSlots.length && i < discounts.length; i++) {
final index = discountSlots[i];
if (index < 0 || index >= items.length) continue;
final item = items[index];
final discount = discounts[i];
items[index] = item.copyWith(
price: roundMoney(item.price + discount),
discount: roundMoney(item.discount + discount),
);
}
final receiptTotal = _readAmountAfterKeyword(
lines,
config,
config.grandTotalKeywords,
skipIf: (line) => config.matchesAny(line, [
...config.totalDiscountKeywords,
...config.taxInclusiveTotalKeywords,
...config.subtotalKeywords,
]),
);
var totalDiscount = discounts.fold<double>(0, (sum, value) => sum + value);
totalDiscount +=
_readAmountAfterKeyword(lines, config, config.totalDiscountKeywords) ??
0;
return _result(
items: items,
rawText: rawText,
receiptTotal: receiptTotal,
totalDiscount: totalDiscount,
);
}
static bool _keepItem(LineItem item) {
if (item.price < 0) return false;
if (item.name.trim().isEmpty) return false;
return !_containsKeyword(item.name, kDropKeywords);
static LineItem _toItem(_Pending pending, double total) {
return LineItem(
id: _id(),
name: pending.name,
price: roundMoney(total),
quantity: pending.quantity == 1 ? null : pending.quantity,
unitPrice: pending.quantity == 1 ? null : pending.unitPrice,
discount: pending.discount == 0 ? 0 : roundMoney(pending.discount),
);
}
static bool _shouldSkipLine(String line) {
if (_separatorPattern.hasMatch(line)) return true;
static ReceiptParseResult _result({
required List<LineItem> items,
required String rawText,
double? receiptTotal,
double totalDiscount = 0,
}) {
final kept = items
.where((item) => item.name.trim().isNotEmpty && item.price > 0)
.toList();
final sum = roundMoney(
kept.fold<double>(0, (total, item) => total + item.price),
);
final passed =
receiptTotal == null ||
(sum - receiptTotal).abs() <= 0.01 ||
(sum + totalDiscount - receiptTotal).abs() <= 0.01;
return ReceiptParseResult(
items: kept,
store: _detectStore(rawText),
receiptTotal: receiptTotal,
totalDiscount: totalDiscount,
validationPassed: passed,
);
}
static String? _detectStore(String rawText) {
final config = receiptParseConfig;
final header = rawText
.split('\n')
.map(_preprocess)
.where((line) => line.isNotEmpty)
.take(8)
.join('\n')
.toUpperCase();
final stores = [...config.knownStores]
..sort((a, b) => b.length.compareTo(a.length));
for (final store in stores) {
if (RegExp('\\b${RegExp.escape(store)}\\b').hasMatch(header)) {
return _canonicalStore(store);
}
}
return null;
}
static String _canonicalStore(String raw) {
const aliases = {
'AH': 'Albert Heijn',
'ALBERT HEIJN': 'Albert Heijn',
'JUMBO': 'Jumbo',
'LIDL': 'Lidl',
'KRUIDVAT': 'Kruidvat',
'ALDI': 'Aldi',
'PLUS': 'Plus',
'DIRK': 'Dirk',
'COOP': 'Coop',
'SPAR': 'SPAR',
'VOMAR': 'Vomar',
'EKOPLAZA': 'Ekoplaza',
'HOOGVLIET': 'Hoogvliet',
'ACTION': 'Action',
'ETOS': 'Etos',
'WORLD TOKO': 'World Toko',
};
return aliases[raw.toUpperCase()] ?? raw;
}
static double? _readAmountAfterKeyword(
List<String> lines,
ReceiptParseConfig config,
List<String> keywords, {
bool Function(String line)? skipIf,
}) {
for (var i = 0; i < lines.length; i++) {
final line = _stripTax(lines[i], config);
if (!config.matchesAny(line, keywords)) continue;
if (skipIf != null && skipIf(line)) continue;
final trailing = _trailingPrice(line, config);
if (trailing != null) return trailing;
if (i + 1 < lines.length) {
final next = tryParsePrice(_stripTax(lines[i + 1], config));
if (next != null) return next;
}
}
return null;
}
static _Line _classify(String raw, ReceiptParseConfig config) {
final line = _stripTax(raw, config);
if (_isIgnorable(line, config)) {
return _Line(kind: _Kind.skip, raw: line);
}
if (config.matchesAny(line, config.totalDiscountKeywords)) {
return _Line(
kind: _Kind.totalDiscount,
raw: line,
amount: _trailingPrice(line, config),
);
}
if (config.matchesAny(line, config.taxInclusiveTotalKeywords) ||
config.matchesAny(line, config.subtotalKeywords)) {
return _Line(kind: _Kind.skip, raw: line);
}
if (config.matchesAny(line, config.grandTotalKeywords)) {
return _Line(
kind: _Kind.grandTotal,
raw: line,
amount: _trailingPrice(line, config),
);
}
final qty = _parseQty(line, config);
if (qty != null) {
return _Line(kind: _Kind.quantity, raw: line, qty: qty);
}
final isDiscount =
config.matchesAny(line, config.discountKeywords) ||
config.negativeAmount.hasMatch(line);
if (isDiscount &&
(config.matchesAny(line, config.discountKeywords) ||
isPriceOnly(line))) {
return _Line(
kind: _Kind.discount,
raw: line,
name: line,
amount: _trailingPrice(line, config) ?? tryParsePrice(line),
);
}
if (isPriceOnly(line)) {
return _Line(
kind: _Kind.priceOnly,
raw: line,
amount: tryParsePrice(line),
);
}
final inline = config.quantityInline.firstMatch(line);
final trailing = _trailingPrice(line, config);
if (trailing != null) {
var name = line.replaceFirst(RegExp(r'\s+-?\d+[.,]\d{2}\s*$'), '').trim();
_Qty? inlineQty;
if (inline != null) {
name = line.substring(0, inline.start).trim();
final count = int.tryParse(inline.group(1)!);
final unit = tryParsePrice(inline.group(2)!);
if (count != null && unit != null) {
inlineQty = _Qty(count: count, unitPrice: unit, total: trailing);
}
}
if (name.isEmpty || _isIgnorable(name, config)) {
return _Line(kind: _Kind.skip, raw: line);
}
return _Line(
kind: _Kind.item,
raw: line,
name: name,
amount: trailing,
qty: inlineQty,
);
}
if (inline != null) {
final name = line.substring(0, inline.start).trim();
final count = int.tryParse(inline.group(1)!);
final unit = tryParsePrice(inline.group(2)!);
if (name.isNotEmpty && count != null && unit != null) {
return _Line(
kind: _Kind.item,
raw: line,
name: name,
amount: roundMoney(count * unit),
qty: _Qty(count: count, unitPrice: unit),
);
}
}
return _Line(kind: _Kind.nameOnly, raw: line, name: line);
}
static bool _isIgnorable(String line, ReceiptParseConfig config) {
if (config.separatorPattern.hasMatch(line)) return true;
if (_urlPattern.hasMatch(line)) return true;
if (_postalPattern.hasMatch(line)) return true;
if (_phonePattern.hasMatch(line.replaceAll(RegExp(r'\s'), ''))) {
return true;
}
if (_streetPattern.hasMatch(line)) return true;
if (_containsKeyword(line, kHeaderKeywords)) return true;
return false;
}
static bool _containsKeyword(String line, Set<String> keywords) {
final upper = line.toUpperCase();
for (final keyword in keywords) {
if (RegExp('\\b${RegExp.escape(keyword)}\\b').hasMatch(upper)) {
return true;
}
if (config.categoryRow.hasMatch(line)) return true;
if (config.isStandaloneKeywordLine(line, config.columnHeaderKeywords)) {
return true;
}
if (config.isStandaloneKeywordLine(line, config.itemCountKeywords)) {
return true;
}
if (config.matchesAny(line, config.taxKeywords)) return true;
if (config.matchesAny(line, config.paymentKeywords)) return true;
if (config.matchesAny(line, config.noiseKeywords)) return true;
return false;
}
static _QtyLine? _parseQty(String line) {
final match = quantityPattern.firstMatch(line.trim());
static double? _trailingPrice(String line, ReceiptParseConfig config) {
final match = RegExp(r'(-?\d+[.,]\d{2})\s*$').firstMatch(line);
if (match == null) return null;
return tryParsePrice(match.group(1)!);
}
static _Qty? _parseQty(String line, ReceiptParseConfig config) {
final match =
_qtyWithTotal.firstMatch(line) ??
config.quantityStandalone.firstMatch(line);
if (match == null) return null;
final count = int.tryParse(match.group(1)!);
final unit = tryParsePrice(match.group(2)!);
if (count == null || count < 1 || unit == null) return null;
final totalRaw = match.group(3);
final total = totalRaw == null ? null : tryParsePrice(totalRaw);
return _QtyLine(count: count, unitPrice: unit, total: total);
double? total;
if (match.groupCount >= 3) {
final raw = match.group(3);
if (raw != null && RegExp(r'^-?\d+[.,]\d{2}$').hasMatch(raw)) {
total = tryParsePrice(raw);
}
}
return _Qty(count: count, unitPrice: unit, total: total);
}
}