"""Statistics file emitted when a production is closed. Fine produzione has to hand the measurements of the whole production over. That part does not depend on the ERP: the file is produced now, and the hand-off to GAIA plugs in beside it once the protocol is agreed (questions D-1 and D-2). Keeping it in its own module makes that seam obvious rather than buried in the close path. The CSV uses the delimiters configured in system_settings, the same ones the manual export honours, so a shop floor set to the Italian convention (';' and ',') gets files that open correctly in their Excel. """ import asyncio import csv import io import logging from datetime import datetime from src.backend.utils.tempo import iso_utc, to_local, utcnow from pathlib import Path from sqlalchemy import select, update from sqlalchemy.ext.asyncio import AsyncSession from src.backend.config import settings from src.backend.models.orm.measurement import Measurement from src.backend.models.orm.production import ProductionRun from src.backend.models.orm.recipe import Recipe from src.backend.models.orm.setting import SystemSetting from src.backend.models.orm.station import Station from src.backend.models.orm.task import RecipeSubtask logger = logging.getLogger(__name__) # Written under the upload directory: it is the volume that is already mounted and # backed up, so the files survive a container being replaced. EXPORT_SUBDIR = "statistics" # `csv.writer` requires a 1-character delimiter. PUT /api/settings/ validates this # going forward (gitea#23), but a value written before that guard existed must not # be able to make close_run fail: fall back and log instead of raising. _VALID_CSV_DELIMITERS = {",", ";", "\t", "|"} _FALLBACK_CSV_DELIMITER = ";" # A cell starting with one of these (after leading spaces) is read as a formula # by Excel/LibreOffice when the CSV is opened — CSV/formula injection (OWASP). _DANGEROUS_CSV_PREFIXES = ("=", "+", "-", "@", "\t", "\r") def neutralise_csv_cell(value: str) -> str: """Defuse a formula-injection payload in a text CSV cell. Prepends a single quote when the value, stripped of leading spaces, starts with =, +, -, @, TAB or CR: this keeps spreadsheet software from evaluating it as a formula while the value stays valid, literal CSV text. Only for free-text fields (lot, serial, descriptions, names, codes) — never call this on numeric cells, the quote would make them text. """ stripped = value.lstrip(" ") if stripped and stripped[0] in _DANGEROUS_CSV_PREFIXES: return "'" + value return value HEADER = [ "production_run_id", "station_code", "recipe_code", "version_id", "lot_number", "serial_number", "measurement_id", "subtask_id", "marker_number", "subtask_description", "nominal", "value", "deviation", "pass_fail", # The limits travel with the row: without them a pass/fail cannot be re-derived # from the file years later, which is the whole point of an audit record. "ltl", "lwl", "uwl", "utl", "unit", "input_method", "input_duration_ms", "measured_by", "measured_at", # A value outside tolerance that was allowed to stand carries the name of who # allowed it. Without these two columns the file would show the failure and # not the decision, which is the half an auditor asks about. "authorised_by", "authorised_at", ] async def _csv_format(db: AsyncSession) -> tuple[str, str]: """Delimiter and decimal separator as configured for this installation.""" delimiter_row = await db.execute( select(SystemSetting).where(SystemSetting.setting_key == "csv_delimiter") ) delimiter = delimiter_row.scalar_one_or_none() decimal_row = await db.execute( select(SystemSetting).where( SystemSetting.setting_key == "csv_decimal_separator" ) ) decimal = decimal_row.scalar_one_or_none() delimiter_value = delimiter.setting_value if delimiter else "," if delimiter_value not in _VALID_CSV_DELIMITERS: logger.warning( "csv_delimiter non valido nel DB (%r): uso %r", delimiter_value, _FALLBACK_CSV_DELIMITER, ) delimiter_value = _FALLBACK_CSV_DELIMITER return ( delimiter_value, decimal.setting_value if decimal else ".", ) async def build_statistics_csv(db: AsyncSession, run: ProductionRun) -> str: """Render every measurement of a run as CSV text.""" delimiter, decimal_separator = await _csv_format(db) station = (await db.execute( select(Station).where(Station.id == run.station_id) )).scalar_one_or_none() recipe = (await db.execute( select(Recipe).where(Recipe.id == run.recipe_id) )).scalar_one_or_none() rows = (await db.execute( select(Measurement, RecipeSubtask) .join(RecipeSubtask, RecipeSubtask.id == Measurement.subtask_id, isouter=True) .where(Measurement.production_run_id == run.id) .order_by(Measurement.measured_at, Measurement.id) )).all() def number(value) -> str: return "" if value is None else str(value).replace(".", decimal_separator) output = io.StringIO() writer = csv.writer(output, delimiter=delimiter, lineterminator="\n") writer.writerow(HEADER) for measurement, subtask in rows: writer.writerow([ run.id, neutralise_csv_cell(station.code) if station else "", neutralise_csv_cell(recipe.code) if recipe else "", measurement.version_id, neutralise_csv_cell(measurement.lot_number or ""), neutralise_csv_cell(measurement.serial_number or ""), measurement.id, measurement.subtask_id, subtask.marker_number if subtask else "", neutralise_csv_cell(subtask.description) if subtask else "", number(subtask.nominal) if subtask else "", number(measurement.value), number(measurement.deviation), measurement.pass_fail, number(subtask.ltl) if subtask else "", number(subtask.lwl) if subtask else "", number(subtask.uwl) if subtask else "", number(subtask.utl) if subtask else "", (subtask.unit or "") if subtask else "", measurement.input_method, "" if measurement.input_duration_ms is None else measurement.input_duration_ms, measurement.measured_by, iso_utc(measurement.measured_at), measurement.supervisor_id or "", iso_utc(measurement.authorised_at), ]) return output.getvalue() async def export_run_statistics(db: AsyncSession, run: ProductionRun) -> str | None: """Write the run's statistics file and mark its measurements as exported. Returns the path relative to the upload directory, or None when the run has no measurements - an empty file would only be noise in the folder. """ content = await build_statistics_csv(db, run) # Header only: nothing was measured during this production. if len(content.strip().splitlines()) <= 1: return None directory = Path(settings.upload_path) / EXPORT_SUBDIR directory.mkdir(parents=True, exist_ok=True) stamp = to_local(run.closed_at or utcnow()).strftime("%Y%m%d_%H%M%S") filename = f"production_{run.id}_{stamp}.csv" # A large run's file can take a moment to write: off the event loop, so # closing one run does not stall every other request being served. await asyncio.to_thread( (directory / filename).write_text, content, encoding="utf-8-sig" ) # utf-8-sig: the BOM is what makes Excel read accented characters correctly, # and these files are opened in Excel on the shop floor. await db.execute( update(Measurement) .where(Measurement.production_run_id == run.id) .values(synced_to_csv=True) ) return f"{EXPORT_SUBDIR}/{filename}"