Watch
1
0
Fork
You've already forked arc
0
forked from hesabix/arc
arc/hesabixAPI/app/services/legacy_sql/receipt_payment_importer.py

266 lines
7.5 KiB
Python

from __future__ import annotations
from collections import defaultdict
from datetime import date
from typing import Any, Callable, Dict, List, Optional, Set, Tuple
from sqlalchemy import text
from sqlalchemy.orm import Session
from app.services.legacy_sql.mappers import (
convert_amount,
convert_persian_date_to_date,
convert_timestamp_to_datetime,
is_valid_mapped_id,
)
from app.services.legacy_sql.sql_dump_reader import LegacySqlData
from app.services.receipt_payment_service import create_receipt_payment
ProgressCb = Optional[Callable[[int, str], None]]
RECEIPT_PAYMENT_DOC_TYPES: Dict[str, str] = {
"person_receive": "receipt",
"person_send": "payment",
"sell_receive": "receipt",
"buy_send": "payment",
}
class LegacyReceiptPaymentImporter:
def __init__(self, db: Session, data: LegacySqlData, *, dry_run: bool = False):
self.db = db
self.data = data
self.dry_run = dry_run
self.stats = {
"processed": 0,
"migrated": 0,
"skipped": 0,
"errors": 0,
"error_samples": [],
}
self._rows_by_doc: Dict[int, List[Dict[str, Any]]] | None = None
self._migrated_old_ids: Set[int] = set()
def _build_rows_index(self) -> Dict[int, List[Dict[str, Any]]]:
index: Dict[int, List[Dict[str, Any]]] = defaultdict(list)
for row in self.data.rows("hesabdari_row"):
doc_id = row.get("doc_id")
if doc_id is None:
continue
try:
did = int(doc_id)
except (TypeError, ValueError):
continue
index[did].append(row)
return dict(index)
def _load_migrated_cache(self, business_id: int) -> None:
rows = self.db.execute(
text("""
SELECT (extra_info->>'old_document_id')::int AS old_id
FROM documents
WHERE business_id = :bid
AND extra_info->>'source' = 'legacy_sql'
AND document_type IN ('receipt', 'payment')
AND extra_info->>'old_document_id' IS NOT NULL
"""),
{"bid": business_id},
).fetchall()
for r in rows:
if r[0] is not None:
self._migrated_old_ids.add(int(r[0]))
def run(
self,
*,
business_id_map: Dict[int, int],
user_id_map: Dict[int, int],
person_id_map: Dict[Tuple[int, int], int],
bank_id_map: Dict[Tuple[int, int], int],
cashdesk_id_map: Dict[Tuple[int, int], int],
petty_id_map: Dict[Tuple[int, int], int],
currency_id_map: Dict[int, int],
on_progress: ProgressCb = None,
) -> Dict[str, Any]:
self._rows_by_doc = self._build_rows_index()
docs = [
d for d in self.data.rows("hesabdari_doc")
if str(d.get("type") or "") in RECEIPT_PAYMENT_DOC_TYPES
]
docs.sort(key=lambda d: (str(d.get("date") or ""), int(d.get("id") or 0)))
total = len(docs) or 1
for i, doc in enumerate(docs):
self.stats["processed"] += 1
if on_progress and i % 10 == 0:
pct = 82 + int(10 * i / total)
on_progress(min(pct, 92), f"دریافت/پرداخت {i + 1}/{total}")
try:
self._migrate_one(
doc,
business_id_map=business_id_map,
user_id_map=user_id_map,
person_id_map=person_id_map,
bank_id_map=bank_id_map,
cashdesk_id_map=cashdesk_id_map,
petty_id_map=petty_id_map,
currency_id_map=currency_id_map,
)
except Exception as exc:
self.stats["errors"] += 1
if len(self.stats["error_samples"]) < 20:
self.stats["error_samples"].append({
"old_doc_id": doc.get("id"),
"type": doc.get("type"),
"error": str(exc),
})
return dict(self.stats)
def _line_amount(self, row: Dict[str, Any]) -> float:
d = convert_amount(row.get("bs"))
c = convert_amount(row.get("bd"))
return float(d if d > 0 else c)
def _detect_transaction(self, row: Dict[str, Any]) -> Tuple[Optional[str], Dict[str, Any]]:
if row.get("cheque_id"):
return "check", {"check_id": int(row["cheque_id"])}
if row.get("bank_id"):
return "bank", {"bank_id": int(row["bank_id"])}
if row.get("cashdesk_id"):
return "cash_register", {"cash_register_id": int(row["cashdesk_id"])}
if row.get("salary_id"):
return "petty_cash", {"petty_cash_id": int(row["salary_id"])}
return None, {}
def _migrate_one(
self,
doc: Dict[str, Any],
*,
business_id_map: Dict[int, int],
user_id_map: Dict[int, int],
person_id_map: Dict[Tuple[int, int], int],
bank_id_map: Dict[int, int],
cashdesk_id_map: Dict[Tuple[int, int], int],
petty_id_map: Dict[Tuple[int, int], int],
currency_id_map: Dict[int, int],
) -> None:
old_doc_id = int(doc["id"])
old_bid = int(doc["bid_id"])
new_bid = business_id_map.get(old_bid)
if not new_bid or new_bid < 0:
self.stats["skipped"] += 1
return
if not self._migrated_old_ids and self.stats["processed"] == 1:
self._load_migrated_cache(new_bid)
if old_doc_id in self._migrated_old_ids:
self.stats["skipped"] += 1
return
old_type = str(doc.get("type") or "")
document_type = RECEIPT_PAYMENT_DOC_TYPES.get(old_type)
if not document_type:
self.stats["skipped"] += 1
return
new_user_id = user_id_map.get(int(doc.get("submitter_id") or 0))
if not new_user_id or new_user_id < 0:
self.stats["skipped"] += 1
return
currency_id = currency_id_map.get(int(doc.get("money_id") or 1))
if not currency_id:
self.stats["skipped"] += 1
return
rows = self._rows_by_doc.get(old_doc_id, []) if self._rows_by_doc else []
if not rows:
self.stats["skipped"] += 1
return
person_lines: List[Dict[str, Any]] = []
account_lines: List[Dict[str, Any]] = []
for row in rows:
amount = self._line_amount(row)
if amount <= 0:
continue
pid = row.get("person_id")
if pid is not None:
try:
new_pid = person_id_map.get((old_bid, int(pid)))
except (TypeError, ValueError):
new_pid = None
if is_valid_mapped_id(new_pid, dry_run=self.dry_run):
person_lines.append({
"person_id": new_pid,
"amount": amount,
"description": row.get("des"),
})
tx_type, tx_extra = self._detect_transaction(row)
if tx_type:
acc_amount = amount
acc_line: Dict[str, Any] = {
"amount": acc_amount,
"transaction_type": tx_type,
"description": row.get("des"),
}
if tx_type == "bank":
ob = row.get("bank_id")
nb = bank_id_map.get((old_bid, int(ob))) if ob is not None else None
if not is_valid_mapped_id(nb, dry_run=self.dry_run):
continue
acc_line["bank_id"] = nb
elif tx_type == "cash_register":
oc = row.get("cashdesk_id")
nc = cashdesk_id_map.get((old_bid, int(oc))) if oc is not None else None
if not is_valid_mapped_id(nc, dry_run=self.dry_run):
continue
acc_line["cash_register_id"] = nc
elif tx_type == "petty_cash":
os = row.get("salary_id")
ns = petty_id_map.get((old_bid, int(os))) if os is not None else None
if not is_valid_mapped_id(ns, dry_run=self.dry_run):
continue
acc_line["petty_cash_id"] = ns
else:
acc_line.update(tx_extra)
account_lines.append(acc_line)
if not person_lines or not account_lines:
self.stats["skipped"] += 1
return
doc_date = convert_persian_date_to_date(doc.get("date"))
if not doc_date:
ts = convert_timestamp_to_datetime(doc.get("date_submit"))
doc_date = ts.date() if ts else date.today()
payload = {
"document_type": document_type,
"document_date": doc_date.isoformat(),
"currency_id": currency_id,
"person_lines": person_lines,
"account_lines": account_lines,
"description": doc.get("des") or "",
"extra_info": {
"source": "legacy_sql",
"old_document_id": old_doc_id,
"old_type": old_type,
},
}
if self.dry_run:
self.stats["migrated"] += 1
return
create_receipt_payment(
self.db,
new_bid,
new_user_id,
payload,
)
self.db.commit()
self._migrated_old_ids.add(old_doc_id)
self.stats["migrated"] += 1