improve parsing 2
This commit is contained in:
davmar
@@ -0,0 +1,346 @@
|
||||
/// Cluster OCR elements into rows and classify using column positions.
|
||||
library;
|
||||
|
||||
import '../models/ocr_element.dart';
|
||||
import '../models/receipt_bounds.dart';
|
||||
import '../models/receipt_row.dart';
|
||||
import '../utils/money.dart';
|
||||
import 'receipt_parse_config.dart';
|
||||
|
||||
final _postalPattern = RegExp(r'\d{4}\s*[A-Za-z]{2}\b');
|
||||
final _phonePattern = RegExp(r'^\d{8,}$');
|
||||
final _streetPattern = RegExp(r"^[A-Za-z. ''-]+\s+\d{1,4}[A-Za-z]?$");
|
||||
final _urlPattern = RegExp(r'www\.|\.com|\.nl', caseSensitive: false);
|
||||
|
||||
/// Pure quantity row — checked before any other row classification.
|
||||
final _pureQuantityRow = RegExp(
|
||||
r'^\s*(\d+)\s*[xX×]\s*(\d+[,.]\d{2})\s*(PER\s+STUK|PER\s+KG|ST\.?|STUKS?)?\s*$',
|
||||
caseSensitive: false,
|
||||
);
|
||||
|
||||
final _priceToken = RegExp(r'^-?\d+[,.]\d{2}$');
|
||||
|
||||
class ReceiptRowBuilder {
|
||||
ReceiptRowBuilder(this.config);
|
||||
|
||||
final ReceiptParseConfig config;
|
||||
|
||||
List<ReceiptRow> buildRows(OcrDocument document) {
|
||||
if (document.elements.isEmpty) return const [];
|
||||
|
||||
final rowClusterThreshold = _rowClusterThreshold(document.elements);
|
||||
final clustered = _clusterByY(document.elements, rowClusterThreshold);
|
||||
final contentWidth = _contentWidth(document.elements);
|
||||
final contentLeft = _contentLeft(document.elements);
|
||||
final priceColumnStart =
|
||||
contentLeft + contentWidth * (1 - config.priceColumnWidthFraction);
|
||||
|
||||
return [
|
||||
for (final rowElements in clustered)
|
||||
_classifyRow(
|
||||
rowElements,
|
||||
priceColumnStart: priceColumnStart,
|
||||
imageWidth: document.imageWidth,
|
||||
imageHeight: document.imageHeight,
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
double _rowClusterThreshold(List<OcrElement> elements) {
|
||||
final heights = [
|
||||
for (final element in elements)
|
||||
if (element.height > 0) element.height,
|
||||
]..sort();
|
||||
if (heights.isEmpty) return 12;
|
||||
final median = heights[heights.length ~/ 2];
|
||||
return median * config.rowClusterHeightFactor;
|
||||
}
|
||||
|
||||
List<List<OcrElement>> _clusterByY(
|
||||
List<OcrElement> elements,
|
||||
double threshold,
|
||||
) {
|
||||
final sorted = [...elements]..sort((a, b) {
|
||||
final dy = a.centerY.compareTo(b.centerY);
|
||||
if (dy != 0) return dy;
|
||||
return a.left.compareTo(b.left);
|
||||
});
|
||||
|
||||
final rows = <List<OcrElement>>[];
|
||||
List<OcrElement>? current;
|
||||
double? anchorY;
|
||||
|
||||
for (final element in sorted) {
|
||||
if (current == null) {
|
||||
current = [element];
|
||||
anchorY = element.centerY;
|
||||
continue;
|
||||
}
|
||||
if ((element.centerY - anchorY!).abs() <= threshold) {
|
||||
current.add(element);
|
||||
} else {
|
||||
current.sort((a, b) => a.left.compareTo(b.left));
|
||||
rows.add(current);
|
||||
current = [element];
|
||||
anchorY = element.centerY;
|
||||
}
|
||||
}
|
||||
if (current != null) {
|
||||
current.sort((a, b) => a.left.compareTo(b.left));
|
||||
rows.add(current);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
double _contentWidth(List<OcrElement> elements) {
|
||||
var minLeft = double.infinity;
|
||||
var maxRight = 0.0;
|
||||
for (final element in elements) {
|
||||
if (element.left < minLeft) minLeft = element.left;
|
||||
if (element.right > maxRight) maxRight = element.right;
|
||||
}
|
||||
if (minLeft == double.infinity) return 1;
|
||||
return (maxRight - minLeft).clamp(1, double.infinity);
|
||||
}
|
||||
|
||||
double _contentLeft(List<OcrElement> elements) {
|
||||
var minLeft = double.infinity;
|
||||
for (final element in elements) {
|
||||
if (element.left < minLeft) minLeft = element.left;
|
||||
}
|
||||
return minLeft == double.infinity ? 0 : minLeft;
|
||||
}
|
||||
|
||||
ReceiptRow _classifyRow(
|
||||
List<OcrElement> rowElements, {
|
||||
required double priceColumnStart,
|
||||
required double imageWidth,
|
||||
required double imageHeight,
|
||||
}) {
|
||||
final text = rowElements.map((e) => e.text).join(' ').trim();
|
||||
final normalized = _preprocess(text);
|
||||
final bounds = _rowBounds(rowElements, imageWidth, imageHeight);
|
||||
|
||||
// 4. Quantity modifier — entire row, checked first.
|
||||
final pureQty = _pureQuantityRow.firstMatch(normalized);
|
||||
if (pureQty != null) {
|
||||
final count = int.tryParse(pureQty.group(1)!);
|
||||
final unit = tryParsePrice(pureQty.group(2)!);
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.quantityModifier,
|
||||
text: normalized,
|
||||
nameText: '',
|
||||
quantity: count,
|
||||
unitPrice: unit,
|
||||
lineTotal: count != null && unit != null
|
||||
? roundMoney(count * unit)
|
||||
: null,
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
|
||||
if (_isIgnorable(normalized)) {
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.skip,
|
||||
text: normalized,
|
||||
nameText: '',
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
|
||||
if (config.matchesAny(normalized, config.totalDiscountKeywords)) {
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.totalDiscount,
|
||||
text: normalized,
|
||||
nameText: normalized,
|
||||
price: _rowPrice(rowElements, priceColumnStart) ??
|
||||
_trailingPrice(normalized),
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
|
||||
if (config.matchesAny(normalized, config.taxInclusiveTotalKeywords) ||
|
||||
config.matchesAny(normalized, config.subtotalKeywords)) {
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.skip,
|
||||
text: normalized,
|
||||
nameText: '',
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
|
||||
if (config.matchesAny(normalized, config.grandTotalKeywords)) {
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.grandTotal,
|
||||
text: normalized,
|
||||
nameText: normalized,
|
||||
price: _rowPrice(rowElements, priceColumnStart) ??
|
||||
_trailingPrice(normalized),
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
|
||||
final isDiscount =
|
||||
config.matchesAny(normalized, config.discountKeywords) ||
|
||||
config.negativeAmount.hasMatch(normalized);
|
||||
if (isDiscount) {
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.discount,
|
||||
text: normalized,
|
||||
nameText: normalized,
|
||||
price: _rowPrice(rowElements, priceColumnStart) ??
|
||||
_trailingPrice(normalized) ??
|
||||
tryParsePrice(normalized),
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
|
||||
final price = _rowPrice(rowElements, priceColumnStart);
|
||||
final inlineQty = config.quantityInline.firstMatch(normalized);
|
||||
var nameText = _nameFromElements(rowElements, priceColumnStart);
|
||||
|
||||
if (price != null) {
|
||||
if (inlineQty != null) {
|
||||
nameText = normalized.substring(0, inlineQty.start).trim();
|
||||
final count = int.tryParse(inlineQty.group(1)!);
|
||||
final unit = tryParsePrice(inlineQty.group(2)!);
|
||||
if (nameText.isNotEmpty && count != null && unit != null) {
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.item,
|
||||
text: normalized,
|
||||
nameText: nameText,
|
||||
price: price,
|
||||
quantity: count,
|
||||
unitPrice: unit,
|
||||
lineTotal: price,
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (nameText.isEmpty || _isIgnorable(nameText)) {
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.skip,
|
||||
text: normalized,
|
||||
nameText: nameText,
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.item,
|
||||
text: normalized,
|
||||
nameText: nameText,
|
||||
price: price,
|
||||
lineTotal: price,
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
|
||||
if (nameText.isEmpty) nameText = normalized;
|
||||
return ReceiptRow(
|
||||
kind: ReceiptRowKind.nameOnly,
|
||||
text: normalized,
|
||||
nameText: nameText,
|
||||
elements: rowElements,
|
||||
bounds: bounds,
|
||||
);
|
||||
}
|
||||
|
||||
double? _rowPrice(List<OcrElement> elements, double priceColumnStart) {
|
||||
double? best;
|
||||
var bestX = -1.0;
|
||||
for (final element in elements) {
|
||||
if (element.centerX < priceColumnStart) continue;
|
||||
final token = element.text.trim().replaceAll(
|
||||
config.trailingCategoryLetter,
|
||||
'',
|
||||
);
|
||||
if (!_priceToken.hasMatch(token)) continue;
|
||||
final value = tryParsePrice(token);
|
||||
if (value == null) continue;
|
||||
if (element.centerX >= bestX) {
|
||||
bestX = element.centerX;
|
||||
best = value;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
String _nameFromElements(List<OcrElement> elements, double priceColumnStart) {
|
||||
final parts = <String>[];
|
||||
for (final element in elements) {
|
||||
if (element.centerX >= priceColumnStart &&
|
||||
_priceToken.hasMatch(element.text.trim())) {
|
||||
continue;
|
||||
}
|
||||
final cleaned = element.text
|
||||
.replaceFirst(config.trailingCategoryLetter, '')
|
||||
.trim();
|
||||
if (cleaned.isNotEmpty) parts.add(cleaned);
|
||||
}
|
||||
return _preprocess(parts.join(' '));
|
||||
}
|
||||
|
||||
ReceiptBounds? _rowBounds(
|
||||
List<OcrElement> elements,
|
||||
double imageWidth,
|
||||
double imageHeight,
|
||||
) {
|
||||
if (elements.isEmpty || imageWidth <= 0 || imageHeight <= 0) return null;
|
||||
var left = double.infinity;
|
||||
var top = double.infinity;
|
||||
var right = 0.0;
|
||||
var bottom = 0.0;
|
||||
for (final element in elements) {
|
||||
if (element.left < left) left = element.left;
|
||||
if (element.top < top) top = element.top;
|
||||
if (element.right > right) right = element.right;
|
||||
if (element.bottom > bottom) bottom = element.bottom;
|
||||
}
|
||||
return ReceiptBounds.fromPixelRect(
|
||||
left: left,
|
||||
top: top,
|
||||
right: right,
|
||||
bottom: bottom,
|
||||
imageWidth: imageWidth,
|
||||
imageHeight: imageHeight,
|
||||
);
|
||||
}
|
||||
|
||||
String _preprocess(String line) =>
|
||||
line.trim().replaceAll(RegExp(r'\s+'), ' ');
|
||||
|
||||
double? _trailingPrice(String line) {
|
||||
final stripped = line.replaceFirst(config.trailingCategoryLetter, '').trim();
|
||||
final match = RegExp(r'(-?\d+[.,]\d{2})\s*$').firstMatch(stripped);
|
||||
if (match == null) return null;
|
||||
return tryParsePrice(match.group(1)!);
|
||||
}
|
||||
|
||||
bool _isIgnorable(String line) {
|
||||
if (config.separatorPattern.hasMatch(line)) return true;
|
||||
if (_urlPattern.hasMatch(line)) return true;
|
||||
if (_postalPattern.hasMatch(line)) return true;
|
||||
if (_phonePattern.hasMatch(line.replaceAll(RegExp(r'\s'), ''))) return true;
|
||||
if (_streetPattern.hasMatch(line)) return true;
|
||||
if (config.categoryRow.hasMatch(line)) return true;
|
||||
if (config.isStandaloneKeywordLine(line, config.columnHeaderKeywords)) {
|
||||
return true;
|
||||
}
|
||||
if (config.isStandaloneKeywordLine(line, config.itemCountKeywords)) {
|
||||
return true;
|
||||
}
|
||||
if (config.matchesAny(line, config.taxKeywords)) return true;
|
||||
if (config.matchesAny(line, config.paymentKeywords)) return true;
|
||||
if (config.matchesAny(line, config.noiseKeywords)) return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user