591 lines
17 KiB
Dart
591 lines
17 KiB
Dart
/// Turns raw OCR text into [LineItem]s using receipt-parsing-keywords.json.
|
||
library;
|
||
|
||
import '../models/line_item.dart';
|
||
import '../utils/money.dart';
|
||
import 'receipt_parse_config.dart';
|
||
|
||
export 'receipt_parse_config.dart' show loadReceiptParseConfig;
|
||
|
||
/// Result of a full receipt parse, including totals used for review.
|
||
class ReceiptParseResult {
|
||
const ReceiptParseResult({
|
||
required this.items,
|
||
this.store,
|
||
this.receiptTotal,
|
||
this.totalDiscount = 0,
|
||
this.validationPassed = true,
|
||
});
|
||
|
||
final List<LineItem> items;
|
||
final String? store;
|
||
final double? receiptTotal;
|
||
final double totalDiscount;
|
||
final bool validationPassed;
|
||
}
|
||
|
||
class _Qty {
|
||
const _Qty({required this.count, required this.unitPrice, this.total});
|
||
|
||
final int count;
|
||
final double unitPrice;
|
||
final double? total;
|
||
|
||
double get lineTotal => total ?? roundMoney(count * unitPrice);
|
||
}
|
||
|
||
class _Pending {
|
||
_Pending({
|
||
required this.name,
|
||
this.unitPrice,
|
||
this.quantity = 1,
|
||
this.lineTotal,
|
||
});
|
||
|
||
String name;
|
||
double? unitPrice;
|
||
int quantity;
|
||
double? lineTotal;
|
||
double discount = 0;
|
||
}
|
||
|
||
enum _Kind {
|
||
skip,
|
||
item,
|
||
nameOnly,
|
||
quantity,
|
||
discount,
|
||
grandTotal,
|
||
totalDiscount,
|
||
priceOnly,
|
||
}
|
||
|
||
class _Line {
|
||
const _Line({
|
||
required this.kind,
|
||
required this.raw,
|
||
this.name,
|
||
this.amount,
|
||
this.qty,
|
||
});
|
||
|
||
final _Kind kind;
|
||
final String raw;
|
||
final String? name;
|
||
final double? amount;
|
||
final _Qty? qty;
|
||
}
|
||
|
||
final _postalPattern = RegExp(r'\d{4}\s*[A-Za-z]{2}\b');
|
||
final _phonePattern = RegExp(r'^\d{8,}$');
|
||
final _streetPattern = RegExp(r"^[A-Za-z. ''-]+\s+\d{1,4}[A-Za-z]?$");
|
||
final _urlPattern = RegExp(r'www\.|\.com|\.nl', caseSensitive: false);
|
||
|
||
/// Jumbo-style OCR often adds a line total after `N X unit`.
|
||
final _qtyWithTotal = RegExp(
|
||
r'^\s*(\d+)\s*[xX×]\s*(-?\d+[,.]\d{2})(?:\s+(?:PER\s+STUK|PER\s+KG|ST\.?|STUKS?))?(?:\s+(-?\d+[,.]\d{2}))?\s*$',
|
||
caseSensitive: false,
|
||
);
|
||
|
||
class ReceiptParser {
|
||
ReceiptParser._();
|
||
|
||
static int _nextId = 0;
|
||
|
||
static String _id() => 'item-${++_nextId}';
|
||
|
||
static Future<void> loadConfig() => loadReceiptParseConfig();
|
||
|
||
static List<LineItem> parse(String rawText) => parseReceipt(rawText).items;
|
||
|
||
static ReceiptParseResult parseReceipt(String rawText) {
|
||
final config = receiptParseConfig;
|
||
final lines = rawText
|
||
.split('\n')
|
||
.map(_preprocess)
|
||
.where((line) => line.isNotEmpty)
|
||
.toList();
|
||
|
||
if (_looksLikeColumnOcr(lines, config)) {
|
||
return _parseColumns(lines, config, rawText);
|
||
}
|
||
return _parseSequential(lines, config, rawText);
|
||
}
|
||
|
||
static String _preprocess(String line) =>
|
||
line.trim().replaceAll(RegExp(r'\s+'), ' ');
|
||
|
||
static String _stripTax(String line, ReceiptParseConfig config) {
|
||
return line.replaceFirst(config.trailingCategoryLetter, '').trim();
|
||
}
|
||
|
||
static bool _looksLikeColumnOcr(
|
||
List<String> lines,
|
||
ReceiptParseConfig config,
|
||
) {
|
||
var run = 0;
|
||
var maxRun = 0;
|
||
var prices = 0;
|
||
var names = 0;
|
||
for (final line in lines) {
|
||
final stripped = _stripTax(line, config);
|
||
if (isPriceOnly(stripped)) {
|
||
prices++;
|
||
run++;
|
||
if (run > maxRun) maxRun = run;
|
||
} else {
|
||
run = 0;
|
||
if (_parseQty(stripped, config) == null &&
|
||
!_isIgnorable(stripped, config)) {
|
||
names++;
|
||
}
|
||
}
|
||
}
|
||
return maxRun >= 4 && prices >= 4 && names >= 4;
|
||
}
|
||
|
||
static ReceiptParseResult _parseSequential(
|
||
List<String> lines,
|
||
ReceiptParseConfig config,
|
||
String rawText,
|
||
) {
|
||
final items = <LineItem>[];
|
||
_Pending? pending;
|
||
var receiptTotal = _readAmountAfterKeyword(
|
||
lines,
|
||
config,
|
||
config.grandTotalKeywords,
|
||
skipIf: (line) => config.matchesAny(line, [
|
||
...config.totalDiscountKeywords,
|
||
...config.taxInclusiveTotalKeywords,
|
||
...config.subtotalKeywords,
|
||
]),
|
||
);
|
||
var totalDiscount = 0.0;
|
||
var stop = false;
|
||
|
||
void flush() {
|
||
final current = pending;
|
||
pending = null;
|
||
if (current == null) return;
|
||
final total = current.lineTotal ?? current.unitPrice;
|
||
if (total == null) return;
|
||
items.add(_toItem(current, total));
|
||
}
|
||
|
||
for (var i = 0; i < lines.length && !stop; i++) {
|
||
final classified = _classify(lines[i], config);
|
||
switch (classified.kind) {
|
||
case _Kind.skip:
|
||
continue;
|
||
case _Kind.item:
|
||
flush();
|
||
pending = _Pending(
|
||
name: classified.name!,
|
||
unitPrice: classified.amount,
|
||
quantity: classified.qty?.count ?? 1,
|
||
lineTotal: classified.qty == null
|
||
? classified.amount
|
||
: classified.qty!.lineTotal,
|
||
);
|
||
if (classified.qty != null) {
|
||
pending!.unitPrice = classified.qty!.unitPrice;
|
||
}
|
||
case _Kind.nameOnly:
|
||
flush();
|
||
pending = _Pending(name: classified.name ?? classified.raw);
|
||
case _Kind.quantity:
|
||
final qty = classified.qty!;
|
||
if (pending != null) {
|
||
pending!
|
||
..quantity = qty.count
|
||
..unitPrice = qty.unitPrice
|
||
..lineTotal = qty.lineTotal;
|
||
}
|
||
case _Kind.discount:
|
||
var amount = classified.amount;
|
||
if (amount == null && i + 1 < lines.length) {
|
||
amount = tryParsePrice(_stripTax(lines[i + 1], config));
|
||
if (amount != null) i++;
|
||
}
|
||
if (amount == null) continue;
|
||
if (amount > 0) amount = -amount;
|
||
if (pending != null) {
|
||
pending!.discount += amount;
|
||
pending!.lineTotal = roundMoney(
|
||
(pending!.lineTotal ?? pending!.unitPrice ?? 0) + amount,
|
||
);
|
||
} else {
|
||
totalDiscount += amount;
|
||
}
|
||
case _Kind.totalDiscount:
|
||
var amount = classified.amount;
|
||
if (amount == null && i + 1 < lines.length) {
|
||
amount = tryParsePrice(_stripTax(lines[i + 1], config));
|
||
if (amount != null) i++;
|
||
}
|
||
if (amount != null) {
|
||
totalDiscount += amount > 0 ? -amount : amount;
|
||
}
|
||
case _Kind.grandTotal:
|
||
flush();
|
||
if (classified.amount != null) {
|
||
receiptTotal = classified.amount;
|
||
stop = true;
|
||
}
|
||
case _Kind.priceOnly:
|
||
final amount = classified.amount!;
|
||
if (amount < 0 && pending != null) {
|
||
pending!.discount += amount;
|
||
pending!.lineTotal = roundMoney(
|
||
(pending!.lineTotal ?? pending!.unitPrice ?? 0) + amount,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
flush();
|
||
|
||
return _result(
|
||
items: items,
|
||
rawText: rawText,
|
||
receiptTotal: receiptTotal,
|
||
totalDiscount: totalDiscount,
|
||
);
|
||
}
|
||
|
||
static ReceiptParseResult _parseColumns(
|
||
List<String> lines,
|
||
ReceiptParseConfig config,
|
||
String rawText,
|
||
) {
|
||
final names = <({String name, int? quantity, double? unitPrice})>[];
|
||
final prices = <double>[];
|
||
final discountSlots = <int>[];
|
||
final discounts = <double>[];
|
||
|
||
for (final raw in lines) {
|
||
final line = _stripTax(raw, config);
|
||
if (isPriceOnly(line)) {
|
||
final price = tryParsePrice(line);
|
||
if (price == null) continue;
|
||
if (price < 0) {
|
||
discounts.add(price);
|
||
} else {
|
||
prices.add(price);
|
||
}
|
||
continue;
|
||
}
|
||
if (_isIgnorable(line, config)) continue;
|
||
if (config.matchesAny(line, [
|
||
...config.grandTotalKeywords,
|
||
...config.subtotalKeywords,
|
||
...config.taxInclusiveTotalKeywords,
|
||
...config.totalDiscountKeywords,
|
||
])) {
|
||
continue;
|
||
}
|
||
|
||
final qty = _parseQty(line, config);
|
||
if (qty != null) {
|
||
if (names.isNotEmpty) {
|
||
final last = names.removeLast();
|
||
names.add((
|
||
name: last.name,
|
||
quantity: qty.count,
|
||
unitPrice: qty.unitPrice,
|
||
));
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (config.matchesAny(line, config.discountKeywords)) {
|
||
if (names.isNotEmpty) discountSlots.add(names.length - 1);
|
||
continue;
|
||
}
|
||
|
||
names.add((name: line, quantity: null, unitPrice: null));
|
||
}
|
||
|
||
final count = names.length < prices.length ? names.length : prices.length;
|
||
final items = <LineItem>[
|
||
for (var i = 0; i < count; i++)
|
||
LineItem(
|
||
id: _id(),
|
||
name: names[i].name,
|
||
price: prices[i],
|
||
quantity: names[i].quantity,
|
||
unitPrice: names[i].unitPrice,
|
||
),
|
||
];
|
||
|
||
for (var i = 0; i < discountSlots.length && i < discounts.length; i++) {
|
||
final index = discountSlots[i];
|
||
if (index < 0 || index >= items.length) continue;
|
||
final item = items[index];
|
||
final discount = discounts[i];
|
||
items[index] = item.copyWith(
|
||
price: roundMoney(item.price + discount),
|
||
discount: roundMoney(item.discount + discount),
|
||
);
|
||
}
|
||
|
||
final receiptTotal = _readAmountAfterKeyword(
|
||
lines,
|
||
config,
|
||
config.grandTotalKeywords,
|
||
skipIf: (line) => config.matchesAny(line, [
|
||
...config.totalDiscountKeywords,
|
||
...config.taxInclusiveTotalKeywords,
|
||
...config.subtotalKeywords,
|
||
]),
|
||
);
|
||
var totalDiscount = discounts.fold<double>(0, (sum, value) => sum + value);
|
||
totalDiscount +=
|
||
_readAmountAfterKeyword(lines, config, config.totalDiscountKeywords) ??
|
||
0;
|
||
|
||
return _result(
|
||
items: items,
|
||
rawText: rawText,
|
||
receiptTotal: receiptTotal,
|
||
totalDiscount: totalDiscount,
|
||
);
|
||
}
|
||
|
||
static LineItem _toItem(_Pending pending, double total) {
|
||
return LineItem(
|
||
id: _id(),
|
||
name: pending.name,
|
||
price: roundMoney(total),
|
||
quantity: pending.quantity == 1 ? null : pending.quantity,
|
||
unitPrice: pending.quantity == 1 ? null : pending.unitPrice,
|
||
discount: pending.discount == 0 ? 0 : roundMoney(pending.discount),
|
||
);
|
||
}
|
||
|
||
static ReceiptParseResult _result({
|
||
required List<LineItem> items,
|
||
required String rawText,
|
||
double? receiptTotal,
|
||
double totalDiscount = 0,
|
||
}) {
|
||
final kept = items
|
||
.where((item) => item.name.trim().isNotEmpty && item.price > 0)
|
||
.toList();
|
||
final sum = roundMoney(
|
||
kept.fold<double>(0, (total, item) => total + item.price),
|
||
);
|
||
final passed =
|
||
receiptTotal == null ||
|
||
(sum - receiptTotal).abs() <= 0.01 ||
|
||
(sum + totalDiscount - receiptTotal).abs() <= 0.01;
|
||
return ReceiptParseResult(
|
||
items: kept,
|
||
store: _detectStore(rawText),
|
||
receiptTotal: receiptTotal,
|
||
totalDiscount: totalDiscount,
|
||
validationPassed: passed,
|
||
);
|
||
}
|
||
|
||
static String? _detectStore(String rawText) {
|
||
final config = receiptParseConfig;
|
||
final header = rawText
|
||
.split('\n')
|
||
.map(_preprocess)
|
||
.where((line) => line.isNotEmpty)
|
||
.take(8)
|
||
.join('\n')
|
||
.toUpperCase();
|
||
final stores = [...config.knownStores]
|
||
..sort((a, b) => b.length.compareTo(a.length));
|
||
for (final store in stores) {
|
||
if (RegExp('\\b${RegExp.escape(store)}\\b').hasMatch(header)) {
|
||
return _canonicalStore(store);
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
static String _canonicalStore(String raw) {
|
||
const aliases = {
|
||
'AH': 'Albert Heijn',
|
||
'ALBERT HEIJN': 'Albert Heijn',
|
||
'JUMBO': 'Jumbo',
|
||
'LIDL': 'Lidl',
|
||
'KRUIDVAT': 'Kruidvat',
|
||
'ALDI': 'Aldi',
|
||
'PLUS': 'Plus',
|
||
'DIRK': 'Dirk',
|
||
'COOP': 'Coop',
|
||
'SPAR': 'SPAR',
|
||
'VOMAR': 'Vomar',
|
||
'EKOPLAZA': 'Ekoplaza',
|
||
'HOOGVLIET': 'Hoogvliet',
|
||
'ACTION': 'Action',
|
||
'ETOS': 'Etos',
|
||
'WORLD TOKO': 'World Toko',
|
||
};
|
||
return aliases[raw.toUpperCase()] ?? raw;
|
||
}
|
||
|
||
static double? _readAmountAfterKeyword(
|
||
List<String> lines,
|
||
ReceiptParseConfig config,
|
||
List<String> keywords, {
|
||
bool Function(String line)? skipIf,
|
||
}) {
|
||
for (var i = 0; i < lines.length; i++) {
|
||
final line = _stripTax(lines[i], config);
|
||
if (!config.matchesAny(line, keywords)) continue;
|
||
if (skipIf != null && skipIf(line)) continue;
|
||
final trailing = _trailingPrice(line, config);
|
||
if (trailing != null) return trailing;
|
||
if (i + 1 < lines.length) {
|
||
final next = tryParsePrice(_stripTax(lines[i + 1], config));
|
||
if (next != null) return next;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
static _Line _classify(String raw, ReceiptParseConfig config) {
|
||
final line = _stripTax(raw, config);
|
||
if (_isIgnorable(line, config)) {
|
||
return _Line(kind: _Kind.skip, raw: line);
|
||
}
|
||
if (config.matchesAny(line, config.totalDiscountKeywords)) {
|
||
return _Line(
|
||
kind: _Kind.totalDiscount,
|
||
raw: line,
|
||
amount: _trailingPrice(line, config),
|
||
);
|
||
}
|
||
if (config.matchesAny(line, config.taxInclusiveTotalKeywords) ||
|
||
config.matchesAny(line, config.subtotalKeywords)) {
|
||
return _Line(kind: _Kind.skip, raw: line);
|
||
}
|
||
if (config.matchesAny(line, config.grandTotalKeywords)) {
|
||
return _Line(
|
||
kind: _Kind.grandTotal,
|
||
raw: line,
|
||
amount: _trailingPrice(line, config),
|
||
);
|
||
}
|
||
|
||
final qty = _parseQty(line, config);
|
||
if (qty != null) {
|
||
return _Line(kind: _Kind.quantity, raw: line, qty: qty);
|
||
}
|
||
|
||
final isDiscount =
|
||
config.matchesAny(line, config.discountKeywords) ||
|
||
config.negativeAmount.hasMatch(line);
|
||
if (isDiscount &&
|
||
(config.matchesAny(line, config.discountKeywords) ||
|
||
isPriceOnly(line))) {
|
||
return _Line(
|
||
kind: _Kind.discount,
|
||
raw: line,
|
||
name: line,
|
||
amount: _trailingPrice(line, config) ?? tryParsePrice(line),
|
||
);
|
||
}
|
||
|
||
if (isPriceOnly(line)) {
|
||
return _Line(
|
||
kind: _Kind.priceOnly,
|
||
raw: line,
|
||
amount: tryParsePrice(line),
|
||
);
|
||
}
|
||
|
||
final inline = config.quantityInline.firstMatch(line);
|
||
final trailing = _trailingPrice(line, config);
|
||
if (trailing != null) {
|
||
var name = line.replaceFirst(RegExp(r'\s+-?\d+[.,]\d{2}\s*$'), '').trim();
|
||
_Qty? inlineQty;
|
||
if (inline != null) {
|
||
name = line.substring(0, inline.start).trim();
|
||
final count = int.tryParse(inline.group(1)!);
|
||
final unit = tryParsePrice(inline.group(2)!);
|
||
if (count != null && unit != null) {
|
||
inlineQty = _Qty(count: count, unitPrice: unit, total: trailing);
|
||
}
|
||
}
|
||
if (name.isEmpty || _isIgnorable(name, config)) {
|
||
return _Line(kind: _Kind.skip, raw: line);
|
||
}
|
||
return _Line(
|
||
kind: _Kind.item,
|
||
raw: line,
|
||
name: name,
|
||
amount: trailing,
|
||
qty: inlineQty,
|
||
);
|
||
}
|
||
|
||
if (inline != null) {
|
||
final name = line.substring(0, inline.start).trim();
|
||
final count = int.tryParse(inline.group(1)!);
|
||
final unit = tryParsePrice(inline.group(2)!);
|
||
if (name.isNotEmpty && count != null && unit != null) {
|
||
return _Line(
|
||
kind: _Kind.item,
|
||
raw: line,
|
||
name: name,
|
||
amount: roundMoney(count * unit),
|
||
qty: _Qty(count: count, unitPrice: unit),
|
||
);
|
||
}
|
||
}
|
||
|
||
return _Line(kind: _Kind.nameOnly, raw: line, name: line);
|
||
}
|
||
|
||
static bool _isIgnorable(String line, ReceiptParseConfig config) {
|
||
if (config.separatorPattern.hasMatch(line)) return true;
|
||
if (_urlPattern.hasMatch(line)) return true;
|
||
if (_postalPattern.hasMatch(line)) return true;
|
||
if (_phonePattern.hasMatch(line.replaceAll(RegExp(r'\s'), ''))) {
|
||
return true;
|
||
}
|
||
if (_streetPattern.hasMatch(line)) return true;
|
||
if (config.categoryRow.hasMatch(line)) return true;
|
||
if (config.isStandaloneKeywordLine(line, config.columnHeaderKeywords)) {
|
||
return true;
|
||
}
|
||
if (config.isStandaloneKeywordLine(line, config.itemCountKeywords)) {
|
||
return true;
|
||
}
|
||
if (config.matchesAny(line, config.taxKeywords)) return true;
|
||
if (config.matchesAny(line, config.paymentKeywords)) return true;
|
||
if (config.matchesAny(line, config.noiseKeywords)) return true;
|
||
return false;
|
||
}
|
||
|
||
static double? _trailingPrice(String line, ReceiptParseConfig config) {
|
||
final match = RegExp(r'(-?\d+[.,]\d{2})\s*$').firstMatch(line);
|
||
if (match == null) return null;
|
||
return tryParsePrice(match.group(1)!);
|
||
}
|
||
|
||
static _Qty? _parseQty(String line, ReceiptParseConfig config) {
|
||
final match =
|
||
_qtyWithTotal.firstMatch(line) ??
|
||
config.quantityStandalone.firstMatch(line);
|
||
if (match == null) return null;
|
||
final count = int.tryParse(match.group(1)!);
|
||
final unit = tryParsePrice(match.group(2)!);
|
||
if (count == null || count < 1 || unit == null) return null;
|
||
double? total;
|
||
if (match.groupCount >= 3) {
|
||
final raw = match.group(3);
|
||
if (raw != null && RegExp(r'^-?\d+[.,]\d{2}$').hasMatch(raw)) {
|
||
total = tryParsePrice(raw);
|
||
}
|
||
}
|
||
return _Qty(count: count, unitPrice: unit, total: total);
|
||
}
|
||
}
|