gitea/receipity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

improve parsing 3

Browse Source
This commit is contained in:
davmar
2026-09-02 13:38:47 +02:00
parent 98574af903
commit ae42ba8193
10 changed files with 417 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/models/ocr_element.dart
+23
View File
@@ -22,16 +22,39 @@ class OcrElement {
double get width => (right - left).abs();
}
/// One ML Kit text line with its word-level elements.
class OcrLine {
const OcrLine({
required this.text,
required this.elements,
required this.left,
required this.top,
required this.right,
required this.bottom,
});
final String text;
final List<OcrElement> elements;
final double left;
final double top;
final double right;
final double bottom;
double get centerY => (top + bottom) / 2;
}
/// All positioned elements from one ML Kit recognition pass.
class OcrDocument {
const OcrDocument({
required this.elements,
required this.lines,
required this.imageWidth,
required this.imageHeight,
required this.flattenedText,
});
final List<OcrElement> elements;
final List<OcrLine> lines;
final double imageWidth;
final double imageHeight;
final String flattenedText;
lib/review_screen.dart
+184 -89
View File
@@ -55,6 +55,7 @@ class _ReviewScreenState extends State<ReviewScreen> {
List<String> _tags = [];
List<String> _tagSuggestions = [];
String? _highlightedItemId;
bool _showTripExtras = false;
@override
void initState() {
@@ -137,18 +138,91 @@ class _ReviewScreenState extends State<ReviewScreen> {
setState(() => _shoppedAt = picked);
}
Future<void> _openReceipt() async {
Future<void> _openReceiptPanel() async {
final path = widget.imagePath;
if (path == null || path.isEmpty) return;
final file = File(path);
if (!file.existsSync()) return;
await showDialog<void>(
final highlighted = _highlightedItemId == null
? null
: _items.cast<LineItem?>().firstWhere(
(item) => item?.id == _highlightedItemId,
orElse: () => null,
);
await showModalBottomSheet<void>(
context: context,
isScrollControlled: true,
showDragHandle: true,
builder: (context) {
return Dialog(
insetPadding: const EdgeInsets.all(16),
child: InteractiveViewer(
child: Image.file(file, fit: BoxFit.contain),
return SafeArea(
child: Padding(
padding: const EdgeInsets.fromLTRB(16, 0, 16, 16),
child: Column(
mainAxisSize: MainAxisSize.min,
crossAxisAlignment: CrossAxisAlignment.stretch,
children: [
Text('Receipt', style: Theme.of(context).textTheme.titleMedium),
const SizedBox(height: 12),
if (highlighted?.sourceBounds != null)
ReceiptHighlightOverlay(
imagePath: path,
bounds: highlighted!.sourceBounds!,
maxHeight: 360,
)
else
GestureDetector(
onTap: () async {
await showDialog<void>(
context: context,
builder: (ctx) => Dialog(
insetPadding: const EdgeInsets.all(16),
child: InteractiveViewer(
child: Image.file(file, fit: BoxFit.contain),
),
),
);
},
child: ClipRRect(
borderRadius: BorderRadius.circular(12),
child: Image.file(file, fit: BoxFit.contain, height: 360),
),
),
],
),
),
);
},
);
}
Future<void> _openOcrPanel() async {
if (widget.rawText.isEmpty) return;
await showModalBottomSheet<void>(
context: context,
isScrollControlled: true,
showDragHandle: true,
builder: (context) {
return SafeArea(
child: Padding(
padding: const EdgeInsets.fromLTRB(16, 0, 16, 16),
child: Column(
mainAxisSize: MainAxisSize.min,
crossAxisAlignment: CrossAxisAlignment.stretch,
children: [
Text('Raw OCR text', style: Theme.of(context).textTheme.titleMedium),
const SizedBox(height: 12),
ConstrainedBox(
constraints: BoxConstraints(
maxHeight: MediaQuery.sizeOf(context).height * 0.55,
),
child: SingleChildScrollView(
child: SelectableText(widget.rawText),
),
),
],
),
),
);
},
@@ -202,13 +276,14 @@ class _ReviewScreenState extends State<ReviewScreen> {
if (_supermarket != null && !names.contains(_supermarket)) {
names.add(_supermarket!);
}
final receiptPath = widget.imagePath;
final highlighted = _highlightedItemId == null
? null
: _items.cast<LineItem?>().firstWhere(
(item) => item?.id == _highlightedItemId,
orElse: () => null,
);
final flaggedCount =
_items.where((item) => item.needsManualReview).length;
final reviewMessage = flaggedCount > 0
? 'Some lines could not be parsed confidently — '
'${flaggedCount == 1 ? '1 item is' : '$flaggedCount items are'} '
'marked below. Check them against the receipt before saving.'
: 'Item count or totals do not match the receipt. '
'Check the items below before saving.';
return Scaffold(
appBar: AppBar(
@@ -232,7 +307,7 @@ class _ReviewScreenState extends State<ReviewScreen> {
child: Padding(
padding: const EdgeInsets.all(12),
child: Text(
'Item count or totals do not match the receipt, or some lines need manual review. Check highlighted items before saving.',
reviewMessage,
style: theme.textTheme.bodyMedium?.copyWith(
color: theme.colorScheme.onErrorContainer,
),
@@ -273,43 +348,57 @@ class _ReviewScreenState extends State<ReviewScreen> {
const SizedBox(height: 8),
ListTile(
contentPadding: EdgeInsets.zero,
leading: const Icon(Icons.calendar_today_outlined),
leading: Row(
mainAxisSize: MainAxisSize.min,
children: [
IconButton(
icon: Icon(
_showTripExtras
? Icons.expand_less
: Icons.edit_note_outlined,
),
tooltip: _showTripExtras
? 'Hide trip name & tags'
: 'Trip name & tags',
onPressed: _saving
? null
: () => setState(
() => _showTripExtras = !_showTripExtras,
),
),
const Icon(Icons.calendar_today_outlined),
],
),
title: Text(formatDate(_shoppedAt)),
subtitle: const Text('Shopping date'),
onTap: _saving ? null : _pickDate,
),
if (_showTripExtras) ...[
const SizedBox(height: 8),
TextField(
controller: _tripName,
enabled: !_saving,
textCapitalization: TextCapitalization.sentences,
decoration: const InputDecoration(
labelText: 'Trip name',
hintText: 'Optional, e.g. Weekly shop',
),
),
const SizedBox(height: 12),
TagEditor(
tags: _tags,
suggestions: _tagSuggestions,
enabled: !_saving,
onChanged: (tags) => setState(() => _tags = tags),
),
const SizedBox(height: 8),
],
],
),
),
Expanded(
child: CustomScrollView(
slivers: [
SliverPadding(
padding: const EdgeInsets.fromLTRB(16, 0, 16, 8),
sliver: SliverToBoxAdapter(
child: Column(
crossAxisAlignment: CrossAxisAlignment.stretch,
children: [
TextField(
controller: _tripName,
enabled: !_saving,
textCapitalization: TextCapitalization.sentences,
decoration: const InputDecoration(
labelText: 'Trip name',
hintText: 'Optional, e.g. Weekly shop',
),
),
const SizedBox(height: 12),
TagEditor(
tags: _tags,
suggestions: _tagSuggestions,
enabled: !_saving,
onChanged: (tags) => setState(() => _tags = tags),
),
],
),
),
),
if (_items.isEmpty)
SliverFillRemaining(
hasScrollBody: false,
@@ -329,6 +418,8 @@ class _ReviewScreenState extends State<ReviewScreen> {
separatorBuilder: (_, _) => const SizedBox(height: 4),
itemBuilder: (context, index) {
final item = _items[index];
final flagged = item.needsManualReview;
final onReceipt = _highlightedItemId == item.id;
return Dismissible(
key: ValueKey(item.id),
direction: DismissDirection.endToStart,
@@ -350,13 +441,26 @@ class _ReviewScreenState extends State<ReviewScreen> {
});
},
child: ListTile(
selected: _highlightedItemId == item.id,
selectedTileColor: theme.colorScheme.primaryContainer
.withValues(alpha: 0.35),
shape: RoundedRectangleBorder(
borderRadius: BorderRadius.circular(12),
side: flagged
? BorderSide(
color: theme.colorScheme.error
.withValues(alpha: 0.55),
)
: BorderSide.none,
),
tileColor: flagged
? theme.colorScheme.errorContainer
.withValues(alpha: 0.35)
: onReceipt
? theme.colorScheme.primaryContainer
.withValues(alpha: 0.35)
: null,
leading: item.sourceBounds != null && _hasReceipt
? IconButton(
icon: Icon(
_highlightedItemId == item.id
onReceipt
? Icons.visibility
: Icons.visibility_outlined,
size: 20,
@@ -366,7 +470,7 @@ class _ReviewScreenState extends State<ReviewScreen> {
? null
: () => _toggleHighlight(item),
)
: item.needsManualReview
: flagged
? Icon(
Icons.flag_outlined,
color: theme.colorScheme.error,
@@ -374,7 +478,7 @@ class _ReviewScreenState extends State<ReviewScreen> {
)
: null,
title: Text(item.name),
subtitle: _itemSubtitle(item),
subtitle: _itemSubtitle(item, flagged: flagged),
trailing: DiscountedPrice(item: item),
onTap: _saving ? null : () => _edit(existing: item),
),
@@ -385,45 +489,31 @@ class _ReviewScreenState extends State<ReviewScreen> {
],
),
),
if (_hasReceipt)
ExpansionTile(
initiallyExpanded: highlighted?.sourceBounds != null,
title: const Text('See receipt'),
children: [
Padding(
padding: const EdgeInsets.fromLTRB(16, 0, 16, 16),
child: highlighted?.sourceBounds != null
? ReceiptHighlightOverlay(
imagePath: receiptPath!,
bounds: highlighted!.sourceBounds!,
)
: GestureDetector(
onTap: _openReceipt,
child: ClipRRect(
borderRadius: BorderRadius.circular(12),
child: Image.file(
File(receiptPath!),
fit: BoxFit.contain,
height: 220,
width: double.infinity,
),
),
),
),
],
),
if (widget.rawText.isNotEmpty)
ExpansionTile(
title: const Text('Raw OCR text'),
children: [
ConstrainedBox(
constraints: const BoxConstraints(maxHeight: 180),
child: SingleChildScrollView(
padding: const EdgeInsets.fromLTRB(16, 0, 16, 16),
child: SelectableText(widget.rawText),
),
),
],
if (_hasReceipt || widget.rawText.isNotEmpty)
Padding(
padding: const EdgeInsets.fromLTRB(16, 0, 16, 4),
child: Row(
children: [
if (_hasReceipt)
Expanded(
child: OutlinedButton.icon(
onPressed: _saving ? null : _openReceiptPanel,
icon: const Icon(Icons.receipt_long_outlined, size: 18),
label: const Text('Receipt'),
),
),
if (_hasReceipt && widget.rawText.isNotEmpty)
const SizedBox(width: 8),
if (widget.rawText.isNotEmpty)
Expanded(
child: OutlinedButton.icon(
onPressed: _saving ? null : _openOcrPanel,
icon: const Icon(Icons.text_snippet_outlined, size: 18),
label: const Text('OCR text'),
),
),
],
),
),
SafeArea(
child: Padding(
@@ -452,8 +542,13 @@ class _ReviewScreenState extends State<ReviewScreen> {
);
}
Widget? _itemSubtitle(LineItem item) {
if (item.quantity == null) return null;
return Text('${item.quantity} × ${formatEuro(item.unitPrice ?? 0)}');
Widget? _itemSubtitle(LineItem item, {bool flagged = false}) {
final parts = <String>[];
if (flagged) parts.add('Needs review');
if (item.quantity != null) {
parts.add('${item.quantity} × ${formatEuro(item.unitPrice ?? 0)}');
}
if (parts.isEmpty) return null;
return Text(parts.join(' · '));
}
}
lib/services/receipt_ocr.dart
+17 -2
View File
@@ -16,15 +16,17 @@ class ReceiptOcr {
required double imageHeight,
}) {
final elements = <OcrElement>[];
final lines = <OcrLine>[];
var maxRight = 0.0;
var maxBottom = 0.0;
for (final block in recognized.blocks) {
for (final line in block.lines) {
final lineElements = <OcrElement>[];
if (line.elements.isNotEmpty) {
for (final element in line.elements) {
final box = element.boundingBox;
elements.add(
lineElements.add(
OcrElement(
text: element.text,
left: box.left,
@@ -38,7 +40,7 @@ class ReceiptOcr {
}
} else {
final box = line.boundingBox;
elements.add(
lineElements.add(
OcrElement(
text: line.text,
left: box.left,
@@ -50,6 +52,18 @@ class ReceiptOcr {
if (box.right > maxRight) maxRight = box.right;
if (box.bottom > maxBottom) maxBottom = box.bottom;
}
elements.addAll(lineElements);
final box = line.boundingBox;
lines.add(
OcrLine(
text: line.text,
elements: lineElements,
left: box.left,
top: box.top,
right: box.right,
bottom: box.bottom,
),
);
}
}
@@ -58,6 +72,7 @@ class ReceiptOcr {
return OcrDocument(
elements: elements,
lines: lines,
imageWidth: width,
imageHeight: height,
flattenedText: recognized.text.trim(),
lib/services/receipt_parse_config.dart
+2 -2
View File
@@ -25,7 +25,7 @@ class ReceiptParseConfig {
required this.trailingCategoryLetter,
required this.noiseKeywords,
required this.itemCountKeywords,
this.rowClusterHeightFactor = 1.2,
this.rowClusterHeightFactor = 0.55,
this.priceColumnWidthFraction = 0.25,
});
@@ -106,7 +106,7 @@ class ReceiptParseConfig {
rowClusterHeightFactor: _spatialDouble(
json,
'row_cluster_height_factor',
fallback: 1.2,
fallback: 0.55,
),
priceColumnWidthFraction: _spatialDouble(
json,
lib/services/receipt_parser.dart
+46 -2
View File
@@ -25,7 +25,7 @@ class ReceiptParser {
/// Primary path: positioned ML Kit elements → row clustering → lookahead.
static ReceiptParseResult parseReceiptFromOcr(OcrDocument document) =>
ReceiptSpatialParser.parse(document);
_parseDocument(document);
static ReceiptParseResult parseRecognizedText(
RecognizedText recognized, {
@@ -37,6 +37,50 @@ class ReceiptParser {
imageWidth: imageWidth,
imageHeight: imageHeight,
);
return parseReceiptFromOcr(document);
return _parseDocument(document);
}
static ReceiptParseResult _parseDocument(OcrDocument document) {
if (ReceiptTextParser.looksLikeColumnOcr(document.flattenedText)) {
return _withImageDimensions(
ReceiptTextParser.parseReceipt(document.flattenedText),
document,
);
}
final spatial = ReceiptSpatialParser.parse(document);
if (_shouldFallbackToText(spatial)) {
return _withImageDimensions(
ReceiptTextParser.parseReceipt(document.flattenedText),
document,
);
}
return spatial;
}
static bool _shouldFallbackToText(ReceiptParseResult spatial) {
if (spatial.items.isEmpty) return true;
final flagged =
spatial.items.where((item) => item.needsManualReview).length;
if (flagged > 0) return true;
if (spatial.items.any((item) => item.price <= 0)) return true;
if (spatial.receiptTotal != null && !spatial.validationPassed) return true;
return false;
}
static ReceiptParseResult _withImageDimensions(
ReceiptParseResult result,
OcrDocument document,
) {
return ReceiptParseResult(
items: result.items,
store: result.store,
receiptTotal: result.receiptTotal,
totalDiscount: result.totalDiscount,
validationPassed: result.validationPassed,
expectedItemCount: result.expectedItemCount,
ocrImageWidth: document.imageWidth,
ocrImageHeight: document.imageHeight,
);
}
}
lib/services/receipt_row_builder.dart
+19 -3
View File
@@ -28,13 +28,28 @@ class ReceiptRowBuilder {
List<ReceiptRow> buildRows(OcrDocument document) {
if (document.elements.isEmpty) return const [];
final rowClusterThreshold = _rowClusterThreshold(document.elements);
final clustered = _clusterByY(document.elements, rowClusterThreshold);
final contentWidth = _contentWidth(document.elements);
final contentLeft = _contentLeft(document.elements);
final priceColumnStart =
contentLeft + contentWidth * (1 - config.priceColumnWidthFraction);
// Prefer ML Kit line boundaries — element Y-clustering merges adjacent lines.
if (document.lines.isNotEmpty) {
return [
for (final line in document.lines)
_classifyRow(
line.elements,
priceColumnStart: priceColumnStart,
imageWidth: document.imageWidth,
imageHeight: document.imageHeight,
rowText: line.text,
),
];
}
final rowClusterThreshold = _rowClusterThreshold(document.elements);
final clustered = _clusterByY(document.elements, rowClusterThreshold);
return [
for (final rowElements in clustered)
_classifyRow(
@@ -116,8 +131,9 @@ class ReceiptRowBuilder {
required double priceColumnStart,
required double imageWidth,
required double imageHeight,
String? rowText,
}) {
final text = rowElements.map((e) => e.text).join(' ').trim();
final text = rowText ?? rowElements.map((e) => e.text).join(' ').trim();
final normalized = _preprocess(text);
final bounds = _rowBounds(rowElements, imageWidth, imageHeight);
lib/services/receipt_text_parser.dart
+11
View File
@@ -116,6 +116,17 @@ class ReceiptTextParser {
return _parseSequential(lines, config, rawText);
}
/// True when OCR dumped an entire name column before a price column (Jumbo-style).
static bool looksLikeColumnOcr(String rawText) {
final config = receiptParseConfig;
final lines = rawText
.split('\n')
.map(_preprocess)
.where((line) => line.isNotEmpty)
.toList();
return _looksLikeColumnOcr(lines, config);
}
static String _preprocess(String line) =>
line.trim().replaceAll(RegExp(r'\s+'), ' ');