diff --git a/lib/main.dart b/lib/main.dart index a24f2f5..ed4892c 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -56,8 +56,8 @@ class _MyAppState extends State { void initState() { super.initState(); _meta = TxMetaController(widget.storage); - _sync = SyncController(auth: widget.auth, meta: _meta); _bank = BankController(storage: widget.storage)..refresh(); + _sync = SyncController(auth: widget.auth, meta: _meta, bank: _bank); _appLinks = AppLinks(); _listenForBankRedirects(); } diff --git a/lib/providers/bank_provider.dart b/lib/providers/bank_provider.dart index 26a8243..f0efe99 100644 --- a/lib/providers/bank_provider.dart +++ b/lib/providers/bank_provider.dart @@ -155,6 +155,35 @@ class BankController extends ChangeNotifier { notifyListeners(); } + /// Union cloud txs into the local bank cache without dropping either side. + /// + /// Rows land in the API cache; CSV imports that match are deduped away. + Future applySyncedTransactions(List fromCloud) async { + if (fromCloud.isEmpty) return; + + _remoteTransactions = mergeTransactions(_remoteTransactions, fromCloud); + _importedTransactions = filterNewTransactions( + _remoteTransactions, + _importedTransactions, + ); + _invalidateMerged(); + + if (!_status.linked && _remoteTransactions.isNotEmpty) { + _status = ConnectStatus( + status: _status.status == 'disconnected' ? 'synced' : _status.status, + linked: true, + accountId: _status.accountId, + sessionId: _status.sessionId, + state: _status.state, + validUntil: _status.validUntil, + ); + } + + await _persistCache(); + await _persistImported(); + notifyListeners(); + } + /// Load status; if linked, also load balance + transactions. Future refresh() async { _isLoading = true; diff --git a/lib/providers/sync_provider.dart b/lib/providers/sync_provider.dart new file mode 100644 index 0000000..e183ef3 --- /dev/null +++ b/lib/providers/sync_provider.dart @@ -0,0 +1,290 @@ +/// Coordinates PocketBase ↔ local annotation + bank-transaction sync. +library; + +import 'dart:async'; + +import 'package:flutter/foundation.dart'; + +import '../services/pb_sync_service.dart'; +import 'auth_provider.dart'; +import 'bank_provider.dart'; +import 'tx_meta_provider.dart'; + +/// User-visible sync toast state (watched by [SyncFeedbackListener]). +class SyncFeedback { + const SyncFeedback({ + required this.epoch, + required this.message, + this.isProgress = false, + this.isError = false, + }); + + final int epoch; + final String message; + final bool isProgress; + final bool isError; +} + +class SyncController extends ChangeNotifier { + SyncController({ + required AuthController auth, + required TxMetaController meta, + required BankController bank, + }) : _auth = auth, + _meta = meta, + _bank = bank { + _wasLoggedIn = auth.isLoggedIn; + _meta.onPersisted = _schedulePush; + _auth.addListener(_onAuthChanged); + if (_auth.isLoggedIn) { + unawaited(syncNow()); + } + } + + final AuthController _auth; + final TxMetaController _meta; + final BankController _bank; + + bool _wasLoggedIn = false; + bool _syncing = false; + bool _pushQueued = false; + bool _queuedFullSync = false; + Timer? _pushDebounce; + Timer? _progressHold; + String? _lastError; + DateTime? _lastSyncedAt; + String _lastSyncSummary = ''; + int _feedbackEpoch = 0; + SyncFeedback? _feedback; + + /// Live detail for the avatar menu / tooltip (does not re-fire SnackBars). + String _statusDetail = ''; + + /// 0–1 while the avatar ring is visible. + double _syncProgress = 0; + bool _showProgressRing = false; + bool _progressSuccess = false; + + bool get isSyncing => _syncing; + String? get lastError => _lastError; + DateTime? get lastSyncedAt => _lastSyncedAt; + bool get canSync => _auth.isLoggedIn; + SyncFeedback? get feedback => _feedback; + + /// After a successful sync, e.g. annotation + bank tx counts. + String get lastSyncSummary => _lastSyncSummary; + + /// Short live status, e.g. `Writing tx_meta (40/200)…`. + String get statusDetail => _statusDetail; + + /// Determinate progress for the avatar ring (0–1). + double get syncProgress => _syncProgress; + + /// Whether to paint the progress ring (includes brief success flash). + bool get showProgressRing => _showProgressRing; + + /// Ring turns green when this is true (sync finished successfully). + bool get progressSuccess => _progressSuccess; + + void _announce(String message, {bool isProgress = false, bool isError = false}) { + _feedbackEpoch++; + _feedback = SyncFeedback( + epoch: _feedbackEpoch, + message: message, + isProgress: isProgress, + isError: isError, + ); + notifyListeners(); + } + + void _setProgress(double value, {bool success = false}) { + _syncProgress = value.clamp(0.0, 1.0); + _showProgressRing = true; + _progressSuccess = success; + notifyListeners(); + } + + void _setStatus(String message) { + if (_statusDetail == message) return; + _statusDetail = message; + notifyListeners(); + } + + void _finishProgress({required bool success}) { + _progressHold?.cancel(); + _statusDetail = ''; + if (success) { + _setProgress(1, success: true); + _progressHold = Timer(const Duration(milliseconds: 900), () { + _showProgressRing = false; + _progressSuccess = false; + _syncProgress = 0; + notifyListeners(); + }); + } else { + _showProgressRing = false; + _progressSuccess = false; + _syncProgress = 0; + notifyListeners(); + } + } + + void _onAuthChanged() { + final loggedIn = _auth.isLoggedIn; + if (loggedIn && !_wasLoggedIn) { + unawaited(syncNow()); + } + _wasLoggedIn = loggedIn; + if (!loggedIn) { + _pushDebounce?.cancel(); + _progressHold?.cancel(); + _lastError = null; + _statusDetail = ''; + _showProgressRing = false; + _progressSuccess = false; + _syncProgress = 0; + notifyListeners(); + } + } + + void _schedulePush() { + if (!_auth.isLoggedIn) return; + _pushDebounce?.cancel(); + _pushDebounce = Timer(const Duration(seconds: 2), () { + unawaited(_pushOnly()); + }); + } + + /// Pull → merge with local (no data loss) → save → push. + /// + /// Includes annotations and the full bank transaction list. + Future syncNow() async { + if (!_auth.isLoggedIn) return; + if (_syncing) { + _queuedFullSync = true; + return; + } + + _progressHold?.cancel(); + _syncing = true; + _lastError = null; + _progressSuccess = false; + _statusDetail = 'Starting…'; + _setProgress(0.02); + _announce('Syncing…', isProgress: true); + + try { + final service = PbSyncService(_auth.client); + final remoteMeta = await service.pull( + onProgress: (p) { + _setProgress(p * 0.35); + }, + onStatus: _setStatus, + ); + _setProgress(0.36); + final remoteTxs = await service.pullTransactions( + onProgress: (p) { + _setProgress(0.36 + p * 0.12); + }, + onStatus: _setStatus, + ); + _setProgress(0.5); + _setStatus('Merging…'); + final mergedMeta = mergeTxMetaStores(_meta.exportStore(), remoteMeta); + await _meta.replaceStore(mergedMeta); + await _bank.applySyncedTransactions(remoteTxs); + _setProgress(0.55); + _setStatus('Pushing…'); + final metaSummary = await service.push( + mergedMeta, + progressStart: 0.55, + progressEnd: 0.82, + onProgress: _setProgress, + onStatus: _setStatus, + ); + final bankSummary = await service.pushTransactions( + _bank.transactions, + progressStart: 0.82, + progressEnd: 1.0, + onProgress: _setProgress, + onStatus: _setStatus, + ); + _lastSyncedAt = DateTime.now(); + _lastError = null; + _lastSyncSummary = '$metaSummary\n$bankSummary'; + _announce('Sync complete'); + _syncing = false; + _finishProgress(success: true); + } catch (error, stackTrace) { + debugPrint('SyncController.syncNow failed: $error\n$stackTrace'); + _lastError = error.toString().replaceFirst(RegExp(r'^Bad state:\s*'), ''); + _announce('Sync failed', isError: true); + _syncing = false; + _finishProgress(success: false); + } finally { + if (_queuedFullSync) { + _queuedFullSync = false; + _pushQueued = false; + unawaited(syncNow()); + } else if (_pushQueued) { + _pushQueued = false; + unawaited(_pushOnly()); + } + } + } + + /// Annotation-only push after local meta edits (avoids re-uploading 4k+ txs). + Future _pushOnly() async { + if (!_auth.isLoggedIn || _syncing) { + if (_syncing) _pushQueued = true; + return; + } + + _progressHold?.cancel(); + _syncing = true; + _progressSuccess = false; + _statusDetail = 'Pushing…'; + _setProgress(0.05); + _announce('Syncing…', isProgress: true); + try { + final service = PbSyncService(_auth.client); + final summary = await service.push( + _meta.exportStore(), + progressStart: 0.05, + progressEnd: 1.0, + onProgress: _setProgress, + onStatus: _setStatus, + ); + _lastSyncedAt = DateTime.now(); + _lastError = null; + _lastSyncSummary = summary; + _announce('Sync complete'); + _syncing = false; + _finishProgress(success: true); + } catch (error, stackTrace) { + debugPrint('SyncController.push failed: $error\n$stackTrace'); + _lastError = error.toString().replaceFirst(RegExp(r'^Bad state:\s*'), ''); + _announce('Sync failed', isError: true); + _syncing = false; + _finishProgress(success: false); + } finally { + if (_queuedFullSync) { + _queuedFullSync = false; + _pushQueued = false; + unawaited(syncNow()); + } else if (_pushQueued) { + _pushQueued = false; + unawaited(_pushOnly()); + } + } + } + + @override + void dispose() { + _pushDebounce?.cancel(); + _progressHold?.cancel(); + _auth.removeListener(_onAuthChanged); + _meta.onPersisted = null; + super.dispose(); + } +} diff --git a/lib/providers/tx_meta_provider.dart b/lib/providers/tx_meta_provider.dart index 84f8df9..92b0ae2 100644 --- a/lib/providers/tx_meta_provider.dart +++ b/lib/providers/tx_meta_provider.dart @@ -127,6 +127,17 @@ class TxMetaController extends ChangeNotifier { return n; } + /// How many labeled txs get their labels only from merchant rules (no per-tx row). + int merchantOnlyLabeledCount(List txs) { + var n = 0; + for (final tx in txs) { + final stored = _byId[tx.id]; + if (stored != null && stored.hasLabel) continue; + if (labelIdsFor(tx).isNotEmpty) n++; + } + return n; + } + List get customLabels => List.unmodifiable(_customLabels); /// Effective parent id (hierarchy map wins over embedded [TxLabel.parentId]). diff --git a/lib/screens/account_screen.dart b/lib/screens/account_screen.dart index 27d7013..73868cf 100644 --- a/lib/screens/account_screen.dart +++ b/lib/screens/account_screen.dart @@ -34,6 +34,10 @@ class AccountScreen extends StatelessWidget { ? 'Not synced yet' : 'Last sync ${_formatTime(lastSync)}'; + final categorized = meta.labeledTransactionCount(bank.transactions); + final perTxAnnotations = meta.syncableAnnotationCount; + final viaMerchantOnly = meta.merchantOnlyLabeledCount(bank.transactions); + return Scaffold( appBar: AppBar(title: const Text('Account')), body: ResponsiveBody( @@ -119,13 +123,12 @@ class AccountScreen extends StatelessWidget { leading: const Icon(Icons.phone_android_outlined), title: const Text('What syncs'), subtitle: Text( - 'Bank cache stays on this device ' - '(${bank.transactions.length} transactions).\n' - 'Cloud gets labels/notes only: ' - '${meta.syncableAnnotationCount} annotated txs, ' - '${meta.merchantRuleCount} merchant rules, ' - '${meta.customLabelCount} custom labels.\n' - 'PocketBase tx_meta ≈ annotated count, not bank total.', + 'Categorized on device: $categorized\n' + '· Per-tx annotations → PocketBase tx_meta: $perTxAnnotations\n' + '· Via merchant rules only (not separate tx_meta rows): $viaMerchantOnly\n' + 'Merchant rules synced: ${meta.merchantRuleCount}\n' + 'Bank transactions → PocketBase transactions: ' + '${bank.transactions.length}', ), isThreeLine: true, ), diff --git a/lib/screens/login_screen.dart b/lib/screens/login_screen.dart index 19ec87c..16e4233 100644 --- a/lib/screens/login_screen.dart +++ b/lib/screens/login_screen.dart @@ -77,8 +77,8 @@ class _LoginScreenState extends State { ), const SizedBox(height: 8), Text( - 'Sign in to link this device with PocketBase. Your labels, ' - 'notes, and bank cache stay on this phone until sync is added.', + 'Sign in to link this device with PocketBase. Labels, notes, ' + 'and bank transactions sync across your devices.', style: theme.textTheme.bodyMedium?.copyWith( color: theme.colorScheme.onSurfaceVariant, ), diff --git a/lib/services/pb_sync_service.dart b/lib/services/pb_sync_service.dart new file mode 100644 index 0000000..287948a --- /dev/null +++ b/lib/services/pb_sync_service.dart @@ -0,0 +1,897 @@ +/// PocketBase pull/push for labels, annotations, and bank transactions. +library; + +import 'dart:async'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:pocketbase/pocketbase.dart'; + +import '../models/bank.dart'; +import '../models/transaction_meta.dart'; +import 'ing_csv_import.dart'; +import 'storage_service.dart'; + +/// Union local + remote bank rows (no data loss). Newest date first. +List mergeBankTransactionLists( + List local, + List remote, +) { + return mergeTransactions(local, remote); +} + +/// Union-merge so neither device loses annotations on first sync. +TxMetaStoreData mergeTxMetaStores(TxMetaStoreData a, TxMetaStoreData b) { + final byId = {...a.byId}; + for (final e in b.byId.entries) { + final existing = byId[e.key]; + byId[e.key] = existing == null ? e.value : _mergeMeta(existing, e.value); + } + + final labelsByMerchant = >{ + for (final e in a.labelsByMerchant.entries) e.key: List.of(e.value), + }; + for (final e in b.labelsByMerchant.entries) { + labelsByMerchant[e.key] = _unionIds(labelsByMerchant[e.key], e.value); + } + + final notesByMerchant = {...a.notesByMerchant}; + for (final e in b.notesByMerchant.entries) { + final cur = notesByMerchant[e.key]; + if (cur == null || cur.trim().isEmpty) { + notesByMerchant[e.key] = e.value; + } else if (e.value.trim().length > cur.trim().length) { + notesByMerchant[e.key] = e.value; + } + } + + final exactTemplates = {...a.exactTemplates}; + for (final e in b.exactTemplates.entries) { + final existing = exactTemplates[e.key]; + exactTemplates[e.key] = + existing == null ? e.value : _mergeExact(existing, e.value); + } + + final labelsByClientId = { + for (final label in a.customLabels) label.id: label, + }; + for (final label in b.customLabels) { + labelsByClientId.putIfAbsent(label.id, () => label); + } + + final labelParents = {...a.labelParents, ...b.labelParents}; + + return TxMetaStoreData( + byId: byId, + labelsByMerchant: labelsByMerchant, + notesByMerchant: notesByMerchant, + exactTemplates: exactTemplates, + customLabels: labelsByClientId.values.toList(growable: false), + labelParents: labelParents, + ); +} + +TxMeta _mergeMeta(TxMeta a, TxMeta b) { + return TxMeta( + labelIds: _unionIds(a.labelIds, b.labelIds), + recurring: a.recurring || b.recurring, + internalTransfer: a.internalTransfer || b.internalTransfer, + notes: _preferNotes(a.notes, b.notes), + ); +} + +TxExactTemplate _mergeExact(TxExactTemplate a, TxExactTemplate b) { + return TxExactTemplate( + notes: _preferNotes(a.notes, b.notes), + recurring: a.recurring || b.recurring, + internalTransfer: a.internalTransfer || b.internalTransfer, + ); +} + +List _unionIds(List? a, List? b) { + final out = []; + final seen = {}; + for (final id in [...?a, ...?b]) { + if (id.isEmpty || id == 'uncategorized' || id == 'other') continue; + if (seen.add(id)) out.add(id); + } + return out; +} + +String _preferNotes(String a, String b) { + final at = a.trim(); + final bt = b.trim(); + if (at.isEmpty) return b; + if (bt.isEmpty) return a; + return bt.length > at.length ? b : a; +} + +bool _sameIdList(List a, List b) { + if (a.length != b.length) return false; + if (listEquals(a, b)) return true; + final sa = [...a]..sort(); + final sb = [...b]..sort(); + return listEquals(sa, sb); +} + +enum _BatchOpKind { create, update, delete } + +class _BatchOp { + const _BatchOp.create(this.body) + : kind = _BatchOpKind.create, + id = null; + const _BatchOp.update(this.id, this.body) : kind = _BatchOpKind.update; + const _BatchOp.delete(this.id) + : kind = _BatchOpKind.delete, + body = null; + + final _BatchOpKind kind; + final String? id; + final Map? body; +} + +/// Talks to PocketBase collections created by the sync migration. +class PbSyncService { + PbSyncService(this._pb); + + final PocketBase _pb; + + /// Parallel HTTP writes — batch API often hangs/stalls on large sets. + static const _writeConcurrency = 12; + + /// Per-request timeout so a single hung call can't freeze sync forever. + static const _requestTimeout = Duration(seconds: 45); + + /// Page size for list fetches (progress updates after each page). + static const _listPageSize = 200; + + String get _userId { + final id = _pb.authStore.record?.id; + if (id == null || id.isEmpty) { + throw StateError('Not authenticated'); + } + return id; + } + + Future _withTimeout(Future future, {String? label}) { + return future.timeout( + _requestTimeout, + onTimeout: () => throw TimeoutException( + label == null ? 'PocketBase request timed out' : '$label timed out', + ), + ); + } + + /// Fetches all matching records page-by-page so UI progress can move. + Future> _fullList( + String collection, + String filter, { + void Function(int loaded)? onPage, + }) async { + final out = []; + var page = 1; + while (true) { + final result = await _withTimeout( + _pb.collection(collection).getList( + page: page, + perPage: _listPageSize, + filter: filter, + skipTotal: true, + ), + label: '$collection list page $page', + ); + out.addAll(result.items); + onPage?.call(out.length); + if (result.items.length < _listPageSize) break; + page++; + } + return out; + } + + Future pull({ + void Function(double progress)? onProgress, + void Function(String message)? onStatus, + }) async { + final userId = _userId; + final filter = 'user="$userId"'; + + void report(double p) => onProgress?.call(p.clamp(0.0, 1.0)); + + report(0.05); + onStatus?.call('Pulling labels…'); + final labelRecords = await _fullList('labels', filter); + report(0.12); + onStatus?.call('Pulling tx_meta…'); + final metaRecords = await _fullList( + 'tx_meta', + filter, + onPage: (n) => onStatus?.call('Pulling tx_meta ($n)…'), + ); + report(0.28); + onStatus?.call('Pulling merchant rules…'); + final merchantRecords = await _fullList('merchant_rules', filter); + report(0.36); + onStatus?.call('Pulling templates…'); + final exactRecords = await _fullList('exact_templates', filter); + report(0.45); + + final customLabels = []; + final labelParents = {}; + for (final r in labelRecords) { + final clientId = r.getStringValue('client_id'); + if (clientId.isEmpty) continue; + final parent = r.getStringValue('parent_client_id'); + customLabels.add( + TxLabel( + id: clientId, + name: r.getStringValue('name').isEmpty + ? 'Label' + : r.getStringValue('name'), + color: Color(r.getIntValue('color', 0xFF64748B)), + custom: true, + parentId: parent.isEmpty ? null : parent, + ), + ); + if (parent.isNotEmpty) labelParents[clientId] = parent; + } + + final byId = {}; + for (final r in metaRecords) { + final txId = r.getStringValue('tx_id'); + if (txId.isEmpty) continue; + byId[txId] = TxMeta( + labelIds: _jsonStringList(r.get('label_ids')), + recurring: r.getBoolValue('recurring'), + internalTransfer: r.getBoolValue('internal_transfer'), + notes: r.getStringValue('notes'), + ); + } + + final labelsByMerchant = >{}; + final notesByMerchant = {}; + for (final r in merchantRecords) { + final key = r.getStringValue('merchant_key'); + if (key.isEmpty) continue; + final ids = _jsonStringList(r.get('label_ids')); + if (ids.isNotEmpty) labelsByMerchant[key] = ids; + final notes = r.getStringValue('notes'); + if (notes.trim().isNotEmpty) notesByMerchant[key] = notes; + } + + final exactTemplates = {}; + for (final r in exactRecords) { + final key = r.getStringValue('exact_key'); + if (key.isEmpty) continue; + exactTemplates[key] = TxExactTemplate( + notes: r.getStringValue('notes'), + recurring: r.getBoolValue('recurring'), + internalTransfer: r.getBoolValue('internal_transfer'), + ); + } + + debugPrint( + 'PbSyncService pull: labels=${labelRecords.length} ' + 'tx_meta=${metaRecords.length} merchants=${merchantRecords.length} ' + 'exact=${exactRecords.length}', + ); + + return TxMetaStoreData( + byId: byId, + labelsByMerchant: labelsByMerchant, + notesByMerchant: notesByMerchant, + exactTemplates: exactTemplates, + customLabels: customLabels, + labelParents: labelParents, + ); + } + + /// Pulls the full bank transaction list for the signed-in user. + Future> pullTransactions({ + void Function(double progress)? onProgress, + void Function(String message)? onStatus, + }) async { + final userId = _userId; + final filter = 'user="$userId"'; + + onProgress?.call(0.05); + onStatus?.call('Pulling bank transactions…'); + final records = await _fullList( + 'transactions', + filter, + onPage: (n) => onStatus?.call('Pulling bank transactions ($n)…'), + ); + onProgress?.call(0.95); + + final out = []; + for (final r in records) { + final tx = _bankTxFromRecord(r); + if (tx != null) out.add(tx); + } + debugPrint('PbSyncService pull: transactions=${out.length}'); + onProgress?.call(1.0); + return out; + } + + /// Upserts [store] and removes remote rows that are no longer local. + /// + /// Returns a short summary of tx_meta sync for the Account screen. + Future push( + TxMetaStoreData store, { + void Function(double progress)? onProgress, + void Function(String message)? onStatus, + double progressStart = 0.5, + double progressEnd = 1.0, + }) async { + final userId = _userId; + final filter = 'user="$userId"'; + final span = progressEnd - progressStart; + + void report(double t) => + onProgress?.call((progressStart + span * t).clamp(0.0, 1.0)); + + report(0.02); + onStatus?.call('Syncing labels…'); + await _syncLabels(userId, filter, store); + report(0.08); + onStatus?.call('Syncing annotations…'); + final txSummary = await _syncTxMeta( + userId, + filter, + store, + onProgress: (t) => report(0.08 + 0.82 * t), + onStatus: onStatus, + ); + report(0.92); + onStatus?.call('Syncing merchant rules…'); + await _syncMerchantRules(userId, filter, store); + report(0.96); + onStatus?.call('Syncing templates…'); + await _syncExactTemplates(userId, filter, store); + report(1.0); + return txSummary; + } + + /// Upserts local bank txs to PocketBase. Does **not** delete remote rows + /// (partial local caches must not wipe another device). + Future pushTransactions( + List local, { + void Function(double progress)? onProgress, + void Function(String message)? onStatus, + double progressStart = 0.0, + double progressEnd = 1.0, + }) async { + final userId = _userId; + final filter = 'user="$userId"'; + final span = progressEnd - progressStart; + + void report(double t) => + onProgress?.call((progressStart + span * t).clamp(0.0, 1.0)); + + report(0.02); + onStatus?.call('Loading remote bank transactions…'); + final existing = { + for (final r in await _fullList( + 'transactions', + filter, + onPage: (n) => + onStatus?.call('Loading remote bank transactions ($n)…'), + )) + r.getStringValue('tx_id'): r, + }; + report(0.12); + + final ops = <_BatchOp>[]; + var skipped = 0; + final seen = {}; + + for (final tx in local) { + final syncId = _clip(tx.id, 2000); + if (syncId.isEmpty || !seen.add(syncId)) continue; + final body = { + 'user': userId, + 'tx_id': syncId, + 'date': _clip(tx.date, 32), + 'description': _clip( + tx.description.isEmpty ? 'Unknown' : tx.description, + 4000, + ), + 'amount': tx.amount, + 'currency': _clip( + tx.currency.isEmpty ? 'EUR' : tx.currency.toUpperCase(), + 8, + ), + 'pending': tx.pending, + }; + final remote = existing[syncId]; + if (remote == null) { + ops.add(_BatchOp.create(body)); + } else if (!_bankTxUnchanged(remote, tx)) { + ops.add(_BatchOp.update(remote.id, body)); + } else { + skipped++; + } + } + + final creates = ops.where((o) => o.kind == _BatchOpKind.create).length; + final updates = ops.where((o) => o.kind == _BatchOpKind.update).length; + debugPrint( + 'PbSyncService transactions: local=${local.length} ' + 'remote=${existing.length} create=$creates update=$updates skip=$skipped', + ); + onStatus?.call( + ops.isEmpty + ? 'Bank txs up to date (local ${local.length} · remote ${existing.length})' + : 'Writing bank txs (0/${ops.length})…', + ); + await _runWriteOps( + 'transactions', + ops, + onProgress: (t) => report(0.12 + 0.88 * t), + onStatus: onStatus, + ); + return 'Bank txs: local ${local.length} → remote ' + '${existing.length + creates} ' + '(+$creates ~$updates · $skipped unchanged)'; + } + + BankTransaction? _bankTxFromRecord(RecordModel r) { + final date = r.getStringValue('date'); + final description = r.getStringValue('description'); + if (date.isEmpty && description.isEmpty) return null; + final rawAmount = r.get('amount'); + final amount = switch (rawAmount) { + num n => n.toDouble(), + String s => double.tryParse(s) ?? 0, + _ => 0.0, + }; + return BankTransaction( + date: date, + description: description.isEmpty ? 'Unknown' : description, + amount: amount, + currency: r.getStringValue('currency').isEmpty + ? 'EUR' + : r.getStringValue('currency'), + pending: r.getBoolValue('pending'), + ); + } + + bool _bankTxUnchanged(RecordModel remote, BankTransaction tx) { + final rawAmount = remote.get('amount'); + final remoteAmount = switch (rawAmount) { + num n => n.toDouble(), + String s => double.tryParse(s) ?? 0, + _ => 0.0, + }; + return remote.getStringValue('date') == tx.date && + remote.getStringValue('description') == + _clip(tx.description.isEmpty ? 'Unknown' : tx.description, 4000) && + (remoteAmount * 100).round() == (tx.amount * 100).round() && + remote.getStringValue('currency').toUpperCase() == + tx.currency.toUpperCase() && + remote.getBoolValue('pending') == tx.pending; + } + + Future _runOne(String collection, _BatchOp op) { + final col = _pb.collection(collection); + final Future future; + switch (op.kind) { + case _BatchOpKind.create: + future = col.create(body: op.body!); + case _BatchOpKind.update: + future = col.update(op.id!, body: op.body!); + case _BatchOpKind.delete: + future = col.delete(op.id!); + } + return _withTimeout(future, label: '$collection ${op.kind.name}'); + } + + String _errorDetail(Object error) { + if (error is ClientException) { + final data = error.response['data']; + final message = error.response['message']; + if (data is Map && data.isNotEmpty) { + final parts = []; + for (final e in data.entries) { + final v = e.value; + if (v is Map && v['message'] != null) { + parts.add('${e.key}: ${v['message']}'); + } else { + parts.add('${e.key}: $v'); + } + } + if (parts.isNotEmpty) return parts.join('; '); + } + if (message is String && message.isNotEmpty) { + return 'HTTP ${error.statusCode}: $message'; + } + return 'HTTP ${error.statusCode}'; + } + return error.toString(); + } + + /// Parallel chunked writes with live progress (avoids hung /api/batch). + Future _runWriteOps( + String collection, + List<_BatchOp> ops, { + void Function(double progress)? onProgress, + void Function(String message)? onStatus, + }) async { + if (ops.isEmpty) { + onProgress?.call(1); + return; + } + + final total = ops.length; + var done = 0; + final errors = []; + + for (var i = 0; i < ops.length; i += _writeConcurrency) { + final end = (i + _writeConcurrency > ops.length) + ? ops.length + : i + _writeConcurrency; + final chunk = ops.sublist(i, end); + await Future.wait([ + for (final op in chunk) + () async { + try { + await _runOne(collection, op); + } catch (error, stackTrace) { + final detail = _errorDetail(error); + // Unique conflict on create → try update if we can resolve the id. + if (op.kind == _BatchOpKind.create && + error is ClientException && + (error.statusCode == 400 || error.statusCode == 409)) { + try { + await _upsertAfterConflict(collection, op.body!); + return; + } catch (retryError, retryStack) { + errors.add(detail); + debugPrint( + 'PbSyncService $collection create+upsert failed: ' + '$detail / ${_errorDetail(retryError)}\n$retryStack', + ); + return; + } + } + errors.add(detail); + debugPrint( + 'PbSyncService $collection ${op.kind.name} failed: $detail\n' + '$stackTrace', + ); + } + }(), + ]); + done = end; + onProgress?.call(done / total); + onStatus?.call('Syncing $collection ($done/$total)…'); + } + + if (errors.isNotEmpty) { + throw StateError( + 'Sync $collection: ${errors.length}/$total failed — ${errors.first}', + ); + } + } + + /// After a unique-key create conflict, load the existing row and PATCH it. + Future _upsertAfterConflict( + String collection, + Map body, + ) async { + final userId = body['user'] as String?; + String? filter; + switch (collection) { + case 'merchant_rules': + final key = body['merchant_key']; + if (userId != null && key is String) { + filter = 'user="$userId" && merchant_key="${key.replaceAll('"', '')}"'; + } + case 'labels': + final key = body['client_id']; + if (userId != null && key is String) { + filter = 'user="$userId" && client_id="${key.replaceAll('"', '')}"'; + } + case 'tx_meta': + final key = body['tx_id']; + if (userId != null && key is String) { + filter = + 'user="$userId" && tx_id="${key.replaceAll('"', '\\"')}"'; + } + case 'exact_templates': + final key = body['exact_key']; + if (userId != null && key is String) { + filter = + 'user="$userId" && exact_key="${key.replaceAll('"', '\\"')}"'; + } + case 'transactions': + final key = body['tx_id']; + if (userId != null && key is String) { + filter = + 'user="$userId" && tx_id="${key.replaceAll('"', '\\"')}"'; + } + } + if (filter == null) { + throw StateError('Cannot upsert $collection without filter'); + } + final existing = await _withTimeout( + _pb.collection(collection).getList( + page: 1, + perPage: 1, + filter: filter, + ), + label: '$collection conflict lookup', + ); + if (existing.items.isEmpty) { + throw StateError('Conflict on $collection but record not found'); + } + await _withTimeout( + _pb.collection(collection).update(existing.items.first.id, body: body), + label: '$collection conflict update', + ); + } + + static String _clip(String value, int max) { + final t = value.trim(); + if (t.length <= max) return t; + return t.substring(0, max); + } + + Future _syncLabels( + String userId, + String filter, + TxMetaStoreData store, + ) async { + final existing = { + for (final r in await _fullList('labels', filter)) + r.getStringValue('client_id'): r, + }; + final keep = {}; + final ops = <_BatchOp>[]; + + for (final label in store.customLabels) { + keep.add(label.id); + final parent = store.labelParents[label.id] ?? label.parentId ?? ''; + final body = { + 'user': userId, + 'client_id': _clip(label.id, 80), + 'name': _clip(label.name.isEmpty ? 'Label' : label.name, 80), + 'color': label.color.toARGB32(), + 'parent_client_id': _clip(parent, 80), + }; + final remote = existing[label.id]; + if (remote == null) { + ops.add(_BatchOp.create(body)); + } else if (!_labelUnchanged(remote, body)) { + ops.add(_BatchOp.update(remote.id, body)); + } + } + + for (final e in existing.entries) { + if (!keep.contains(e.key)) { + ops.add(_BatchOp.delete(e.value.id)); + } + } + + debugPrint('PbSyncService labels: remote=${existing.length} ops=${ops.length}'); + await _runWriteOps('labels', ops); + } + + Future _syncTxMeta( + String userId, + String filter, + TxMetaStoreData store, { + void Function(double progress)? onProgress, + void Function(String message)? onStatus, + }) async { + onProgress?.call(0.02); + onStatus?.call('Loading remote tx_meta…'); + final existing = { + for (final r in await _fullList( + 'tx_meta', + filter, + onPage: (n) => onStatus?.call('Loading remote tx_meta ($n)…'), + )) + r.getStringValue('tx_id'): r, + }; + onProgress?.call(0.12); + final keep = {}; + final ops = <_BatchOp>[]; + var skipped = 0; + var localSyncable = 0; + + for (final e in store.byId.entries) { + final meta = e.value; + if (!meta.hasLabel && + !meta.hasNotes && + !meta.recurring && + !meta.internalTransfer) { + continue; + } + localSyncable++; + final syncId = _clip(e.key, 2000); + keep.add(syncId); + final body = { + 'user': userId, + 'tx_id': syncId, + 'label_ids': meta.labelIds, + 'recurring': meta.recurring, + 'internal_transfer': meta.internalTransfer, + 'notes': _clip(meta.notes, 4000), + }; + final remote = existing[syncId] ?? existing[e.key]; + if (remote == null) { + ops.add(_BatchOp.create(body)); + } else if (!_txMetaUnchanged(remote, meta)) { + ops.add(_BatchOp.update(remote.id, body)); + } else { + skipped++; + } + } + + for (final e in existing.entries) { + if (!keep.contains(e.key)) { + ops.add(_BatchOp.delete(e.value.id)); + } + } + + final creates = ops.where((o) => o.kind == _BatchOpKind.create).length; + final updates = ops.where((o) => o.kind == _BatchOpKind.update).length; + final deletes = ops.where((o) => o.kind == _BatchOpKind.delete).length; + debugPrint( + 'PbSyncService tx_meta: local=$localSyncable remote=${existing.length} ' + 'create=$creates update=$updates delete=$deletes skip=$skipped', + ); + onStatus?.call( + ops.isEmpty + ? 'tx_meta up to date (local $localSyncable · remote ${existing.length})' + : 'Writing tx_meta (0/${ops.length})…', + ); + await _runWriteOps( + 'tx_meta', + ops, + onProgress: (t) => onProgress?.call(0.12 + 0.88 * t), + onStatus: onStatus, + ); + return 'Annotations: local $localSyncable → remote ' + '${existing.length + creates - deletes} ' + '(+$creates ~$updates -$deletes · $skipped unchanged)'; + } + + Future _syncMerchantRules( + String userId, + String filter, + TxMetaStoreData store, + ) async { + final existing = { + for (final r in await _fullList('merchant_rules', filter)) + r.getStringValue('merchant_key'): r, + }; + final keep = {}; + final ops = <_BatchOp>[]; + final keys = { + ...store.labelsByMerchant.keys, + ...store.notesByMerchant.keys, + }; + + for (final rawKey in keys) { + final key = _clip(rawKey, 200); + if (key.isEmpty) continue; + final ids = store.labelsByMerchant[rawKey] ?? const []; + final notes = _clip(store.notesByMerchant[rawKey] ?? '', 4000); + if (ids.isEmpty && notes.isEmpty) continue; + keep.add(key); + final body = { + 'user': userId, + 'merchant_key': key, + 'label_ids': ids, + 'notes': notes, + }; + final remote = existing[key] ?? existing[rawKey]; + if (remote == null) { + ops.add(_BatchOp.create(body)); + } else if (!_merchantUnchanged(remote, ids, notes)) { + ops.add(_BatchOp.update(remote.id, body)); + } + } + + for (final e in existing.entries) { + if (!keep.contains(e.key)) { + ops.add(_BatchOp.delete(e.value.id)); + } + } + + debugPrint( + 'PbSyncService merchant_rules: remote=${existing.length} ops=${ops.length}', + ); + await _runWriteOps('merchant_rules', ops); + } + + Future _syncExactTemplates( + String userId, + String filter, + TxMetaStoreData store, + ) async { + final existing = { + for (final r in await _fullList('exact_templates', filter)) + r.getStringValue('exact_key'): r, + }; + final keep = {}; + final ops = <_BatchOp>[]; + + for (final e in store.exactTemplates.entries) { + final t = e.value; + if (t.notes.trim().isEmpty && !t.recurring && !t.internalTransfer) { + continue; + } + final key = _clip(e.key, 2000); + if (key.isEmpty) continue; + keep.add(key); + final body = { + 'user': userId, + 'exact_key': key, + 'notes': _clip(t.notes, 4000), + 'recurring': t.recurring, + 'internal_transfer': t.internalTransfer, + }; + final remote = existing[key] ?? existing[e.key]; + if (remote == null) { + ops.add(_BatchOp.create(body)); + } else if (!_exactUnchanged(remote, t)) { + ops.add(_BatchOp.update(remote.id, body)); + } + } + + for (final e in existing.entries) { + if (!keep.contains(e.key)) { + ops.add(_BatchOp.delete(e.value.id)); + } + } + + debugPrint( + 'PbSyncService exact_templates: remote=${existing.length} ops=${ops.length}', + ); + await _runWriteOps('exact_templates', ops); + } + + bool _labelUnchanged(RecordModel remote, Map body) { + return remote.getStringValue('name') == body['name'] && + remote.getIntValue('color') == body['color'] && + remote.getStringValue('parent_client_id') == body['parent_client_id']; + } + + bool _txMetaUnchanged(RecordModel remote, TxMeta meta) { + final remoteIds = _jsonStringList(remote.get('label_ids')); + return _sameIdList(remoteIds, meta.labelIds) && + remote.getBoolValue('recurring') == meta.recurring && + remote.getBoolValue('internal_transfer') == meta.internalTransfer && + remote.getStringValue('notes') == meta.notes; + } + + bool _merchantUnchanged( + RecordModel remote, + List ids, + String notes, + ) { + final remoteIds = _jsonStringList(remote.get('label_ids')); + return _sameIdList(remoteIds, ids) && + remote.getStringValue('notes') == notes; + } + + bool _exactUnchanged(RecordModel remote, TxExactTemplate t) { + return remote.getStringValue('notes') == t.notes && + remote.getBoolValue('recurring') == t.recurring && + remote.getBoolValue('internal_transfer') == t.internalTransfer; + } +} + +List _jsonStringList(dynamic raw) { + if (raw is! List) return const []; + final out = []; + for (final item in raw) { + if (item == null) continue; + final id = item is String ? item : '$item'; + if (id.isEmpty || id == 'uncategorized' || id == 'other') continue; + out.add(id); + } + return out; +} diff --git a/lib/widgets/sync_feedback_listener.dart b/lib/widgets/sync_feedback_listener.dart new file mode 100644 index 0000000..045438a --- /dev/null +++ b/lib/widgets/sync_feedback_listener.dart @@ -0,0 +1,13 @@ +/// Formerly showed SnackBars for sync; avatar/Account feedback is enough. +library; + +import 'package:flutter/material.dart'; + +class SyncFeedbackListener extends StatelessWidget { + const SyncFeedbackListener({super.key, required this.child}); + + final Widget child; + + @override + Widget build(BuildContext context) => child; +} diff --git a/pocketbase/pb_migrations/1740000001_ingtrak_transactions_collection.js b/pocketbase/pb_migrations/1740000001_ingtrak_transactions_collection.js new file mode 100644 index 0000000..cdf4749 --- /dev/null +++ b/pocketbase/pb_migrations/1740000001_ingtrak_transactions_collection.js @@ -0,0 +1,87 @@ +/// PocketBase migration: sync full bank transaction list (BankTransaction rows). +/// +/// Apply on the server (from /opt/pocketbase or wherever the binary lives): +/// +/// sudo cp this_file /opt/pocketbase/pb_migrations/ +/// sudo chown pocketbase:pocketbase /opt/pocketbase/pb_migrations/*.js +/// sudo -u pocketbase /opt/pocketbase/pocketbase migrate up +/// sudo systemctl restart pocketbase +/// +/// Depends on usersapp (same owner rules as labels / tx_meta). +migrate((app) => { + const users = app.findCollectionByNameOrId("usersapp"); + + const ownerRules = { + listRule: "user = @request.auth.id", + viewRule: "user = @request.auth.id", + createRule: "@request.auth.id != '' && user = @request.auth.id", + updateRule: "user = @request.auth.id", + deleteRule: "user = @request.auth.id", + }; + + // --- bank transactions (API cache + CSV imports) --- + // Components are stored so clients can rebuild BankTransaction; tx_id is the + // clipped BankTransaction.id used for unique upserts (same clip as tx_meta). + const transactions = new Collection({ + type: "base", + name: "transactions", + ...ownerRules, + fields: [ + { + type: "relation", + name: "user", + required: true, + collectionId: users.id, + cascadeDelete: true, + maxSelect: 1, + }, + { + // BankTransaction.id → "date|currency|amount|description" (clipped) + type: "text", + name: "tx_id", + required: true, + min: 1, + max: 2000, + }, + { + type: "text", + name: "date", + required: true, + min: 1, + max: 32, + }, + { + type: "text", + name: "description", + required: true, + min: 1, + max: 4000, + }, + { + type: "number", + name: "amount", + required: true, + }, + { + type: "text", + name: "currency", + required: true, + min: 1, + max: 8, + }, + { + type: "bool", + name: "pending", + required: false, + }, + ], + indexes: [ + "CREATE UNIQUE INDEX idx_transactions_user_tx ON transactions (user, tx_id)", + ], + }); + app.save(transactions); +}, (app) => { + try { + app.delete(app.findCollectionByNameOrId("transactions")); + } catch (_) {} +}); diff --git a/test/filter_detail_smoke_test.dart b/test/filter_detail_smoke_test.dart index 03cde54..a6201da 100644 --- a/test/filter_detail_smoke_test.dart +++ b/test/filter_detail_smoke_test.dart @@ -41,7 +41,7 @@ void main() { await auth.init(); bank = BankController(storage: storage); meta = TxMetaController(storage); - sync = SyncController(auth: auth, meta: meta); + sync = SyncController(auth: auth, meta: meta, bank: bank); }); tearDown(() { diff --git a/test/overview_smoke_test.dart b/test/overview_smoke_test.dart index eaa6953..2c2414e 100644 --- a/test/overview_smoke_test.dart +++ b/test/overview_smoke_test.dart @@ -33,7 +33,7 @@ void main() { // seed a couple txs // ignore: invalid_use_of_visible_for_testing_member meta = TxMetaController(storage); - sync = SyncController(auth: auth, meta: meta); + sync = SyncController(auth: auth, meta: meta, bank: bank); }); tearDown(() { diff --git a/test/pb_sync_merge_test.dart b/test/pb_sync_merge_test.dart new file mode 100644 index 0000000..1e37974 --- /dev/null +++ b/test/pb_sync_merge_test.dart @@ -0,0 +1,100 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:myapp/models/bank.dart'; +import 'package:myapp/models/transaction_meta.dart'; +import 'package:myapp/services/pb_sync_service.dart'; +import 'package:myapp/services/storage_service.dart'; + +void main() { + test('mergeTxMetaStores unions labels and metas without dropping either side', + () { + final local = TxMetaStoreData( + byId: { + 'tx1': const TxMeta(labelIds: ['groceries'], notes: 'local note'), + 'tx2': const TxMeta(recurring: true), + }, + labelsByMerchant: { + 'albert heijn': ['groceries'], + }, + customLabels: const [ + TxLabel( + id: 'custom_ah', + name: 'AH', + color: Color(0xFF0F766E), + custom: true, + ), + ], + labelParents: const {'custom_ah': 'groceries'}, + ); + final remote = TxMetaStoreData( + byId: { + 'tx1': const TxMeta(labelIds: ['dining'], notes: ''), + 'tx3': const TxMeta(notes: 'remote only'), + }, + labelsByMerchant: { + 'albert heijn': ['shopping'], + 'jumbo': ['groceries'], + }, + notesByMerchant: const {'jumbo': 'weekly'}, + customLabels: const [ + TxLabel( + id: 'custom_fuel', + name: 'Fuel', + color: Color(0xFFEA580C), + custom: true, + ), + ], + ); + + final merged = mergeTxMetaStores(local, remote); + + expect(merged.byId['tx1']!.labelIds, containsAll(['groceries', 'dining'])); + expect(merged.byId['tx1']!.notes, 'local note'); + expect(merged.byId['tx2']!.recurring, isTrue); + expect(merged.byId['tx3']!.notes, 'remote only'); + expect(merged.labelsByMerchant['albert heijn'], + containsAll(['groceries', 'shopping'])); + expect(merged.labelsByMerchant['jumbo'], ['groceries']); + expect(merged.notesByMerchant['jumbo'], 'weekly'); + expect(merged.customLabels.map((l) => l.id), + containsAll(['custom_ah', 'custom_fuel'])); + expect(merged.labelParents['custom_ah'], 'groceries'); + }); + + test('mergeBankTransactionLists unions without dropping either side', () { + const localOnly = BankTransaction( + date: '2024-01-02', + description: 'Local Cafe', + amount: -4.5, + currency: 'EUR', + pending: false, + ); + const shared = BankTransaction( + date: '2024-01-01', + description: 'Shared Shop', + amount: -12.0, + currency: 'EUR', + pending: false, + ); + const remoteOnly = BankTransaction( + date: '2024-01-03', + description: 'Remote Market', + amount: -9.25, + currency: 'EUR', + pending: true, + ); + + final merged = mergeBankTransactionLists( + [localOnly, shared], + [shared, remoteOnly], + ); + + expect(merged.length, 3); + expect(merged.map((t) => t.id), containsAll([ + localOnly.id, + shared.id, + remoteOnly.id, + ])); + expect(merged.first.date, '2024-01-03'); + }); +}