213 lines
8.0 KiB
Python
213 lines
8.0 KiB
Python
"""Statistics file emitted when a production is closed.
|
|
|
|
Fine produzione has to hand the measurements of the whole production over. That part
|
|
does not depend on the ERP: the file is produced now, and the hand-off to GAIA plugs
|
|
in beside it once the protocol is agreed (questions D-1 and D-2). Keeping it in its
|
|
own module makes that seam obvious rather than buried in the close path.
|
|
|
|
The CSV uses the delimiters configured in system_settings, the same ones the manual
|
|
export honours, so a shop floor set to the Italian convention (';' and ',') gets
|
|
files that open correctly in their Excel.
|
|
"""
|
|
import asyncio
|
|
import csv
|
|
import io
|
|
import logging
|
|
from datetime import datetime
|
|
from src.backend.utils.tempo import iso_utc, to_local, utcnow
|
|
from pathlib import Path
|
|
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from src.backend.config import settings
|
|
from src.backend.models.orm.measurement import Measurement
|
|
from src.backend.models.orm.production import ProductionRun
|
|
from src.backend.models.orm.recipe import Recipe
|
|
from src.backend.models.orm.setting import SystemSetting
|
|
from src.backend.models.orm.station import Station
|
|
from src.backend.models.orm.task import RecipeSubtask
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Written under the upload directory: it is the volume that is already mounted and
|
|
# backed up, so the files survive a container being replaced.
|
|
EXPORT_SUBDIR = "statistics"
|
|
|
|
# `csv.writer` requires a 1-character delimiter. PUT /api/settings/ validates this
|
|
# going forward (gitea#23), but a value written before that guard existed must not
|
|
# be able to make close_run fail: fall back and log instead of raising.
|
|
_VALID_CSV_DELIMITERS = {",", ";", "\t", "|"}
|
|
_FALLBACK_CSV_DELIMITER = ";"
|
|
|
|
# A cell starting with one of these (after leading spaces) is read as a formula
|
|
# by Excel/LibreOffice when the CSV is opened — CSV/formula injection (OWASP).
|
|
_DANGEROUS_CSV_PREFIXES = ("=", "+", "-", "@", "\t", "\r")
|
|
|
|
|
|
def neutralise_csv_cell(value: str) -> str:
|
|
"""Defuse a formula-injection payload in a text CSV cell.
|
|
|
|
Prepends a single quote when the value, stripped of leading spaces, starts
|
|
with =, +, -, @, TAB or CR: this keeps spreadsheet software from evaluating
|
|
it as a formula while the value stays valid, literal CSV text. Only for
|
|
free-text fields (lot, serial, descriptions, names, codes) — never call
|
|
this on numeric cells, the quote would make them text.
|
|
"""
|
|
stripped = value.lstrip(" ")
|
|
if stripped and stripped[0] in _DANGEROUS_CSV_PREFIXES:
|
|
return "'" + value
|
|
return value
|
|
|
|
|
|
HEADER = [
|
|
"production_run_id",
|
|
"station_code",
|
|
"recipe_code",
|
|
"version_id",
|
|
"lot_number",
|
|
"serial_number",
|
|
"measurement_id",
|
|
"subtask_id",
|
|
"marker_number",
|
|
"subtask_description",
|
|
"nominal",
|
|
"value",
|
|
"deviation",
|
|
"pass_fail",
|
|
# The limits travel with the row: without them a pass/fail cannot be re-derived
|
|
# from the file years later, which is the whole point of an audit record.
|
|
"ltl",
|
|
"lwl",
|
|
"uwl",
|
|
"utl",
|
|
"unit",
|
|
"input_method",
|
|
"input_duration_ms",
|
|
"measured_by",
|
|
"measured_at",
|
|
# A value outside tolerance that was allowed to stand carries the name of who
|
|
# allowed it. Without these two columns the file would show the failure and
|
|
# not the decision, which is the half an auditor asks about.
|
|
"authorised_by",
|
|
"authorised_at",
|
|
]
|
|
|
|
|
|
async def _csv_format(db: AsyncSession) -> tuple[str, str]:
|
|
"""Delimiter and decimal separator as configured for this installation."""
|
|
delimiter_row = await db.execute(
|
|
select(SystemSetting).where(SystemSetting.setting_key == "csv_delimiter")
|
|
)
|
|
delimiter = delimiter_row.scalar_one_or_none()
|
|
decimal_row = await db.execute(
|
|
select(SystemSetting).where(
|
|
SystemSetting.setting_key == "csv_decimal_separator"
|
|
)
|
|
)
|
|
decimal = decimal_row.scalar_one_or_none()
|
|
|
|
delimiter_value = delimiter.setting_value if delimiter else ","
|
|
if delimiter_value not in _VALID_CSV_DELIMITERS:
|
|
logger.warning(
|
|
"csv_delimiter non valido nel DB (%r): uso %r",
|
|
delimiter_value, _FALLBACK_CSV_DELIMITER,
|
|
)
|
|
delimiter_value = _FALLBACK_CSV_DELIMITER
|
|
|
|
return (
|
|
delimiter_value,
|
|
decimal.setting_value if decimal else ".",
|
|
)
|
|
|
|
|
|
async def build_statistics_csv(db: AsyncSession, run: ProductionRun) -> str:
|
|
"""Render every measurement of a run as CSV text."""
|
|
delimiter, decimal_separator = await _csv_format(db)
|
|
|
|
station = (await db.execute(
|
|
select(Station).where(Station.id == run.station_id)
|
|
)).scalar_one_or_none()
|
|
recipe = (await db.execute(
|
|
select(Recipe).where(Recipe.id == run.recipe_id)
|
|
)).scalar_one_or_none()
|
|
|
|
rows = (await db.execute(
|
|
select(Measurement, RecipeSubtask)
|
|
.join(RecipeSubtask, RecipeSubtask.id == Measurement.subtask_id, isouter=True)
|
|
.where(Measurement.production_run_id == run.id)
|
|
.order_by(Measurement.measured_at, Measurement.id)
|
|
)).all()
|
|
|
|
def number(value) -> str:
|
|
return "" if value is None else str(value).replace(".", decimal_separator)
|
|
|
|
output = io.StringIO()
|
|
writer = csv.writer(output, delimiter=delimiter, lineterminator="\n")
|
|
writer.writerow(HEADER)
|
|
|
|
for measurement, subtask in rows:
|
|
writer.writerow([
|
|
run.id,
|
|
neutralise_csv_cell(station.code) if station else "",
|
|
neutralise_csv_cell(recipe.code) if recipe else "",
|
|
measurement.version_id,
|
|
neutralise_csv_cell(measurement.lot_number or ""),
|
|
neutralise_csv_cell(measurement.serial_number or ""),
|
|
measurement.id,
|
|
measurement.subtask_id,
|
|
subtask.marker_number if subtask else "",
|
|
neutralise_csv_cell(subtask.description) if subtask else "",
|
|
number(subtask.nominal) if subtask else "",
|
|
number(measurement.value) if measurement.value is not None else "NaN", # #118
|
|
"NaN" if measurement.value is None else number(measurement.deviation), # #124: quota NaN -> deviazione NaN
|
|
measurement.pass_fail,
|
|
number(subtask.ltl) if subtask else "",
|
|
number(subtask.lwl) if subtask else "",
|
|
number(subtask.uwl) if subtask else "",
|
|
number(subtask.utl) if subtask else "",
|
|
(subtask.unit or "") if subtask else "",
|
|
measurement.input_method,
|
|
"" if measurement.input_duration_ms is None else measurement.input_duration_ms,
|
|
measurement.measured_by,
|
|
iso_utc(measurement.measured_at),
|
|
measurement.supervisor_id or "",
|
|
iso_utc(measurement.authorised_at),
|
|
])
|
|
|
|
return output.getvalue()
|
|
|
|
|
|
async def export_run_statistics(db: AsyncSession, run: ProductionRun) -> str | None:
|
|
"""Write the run's statistics file and mark its measurements as exported.
|
|
|
|
Returns the path relative to the upload directory, or None when the run has no
|
|
measurements - an empty file would only be noise in the folder.
|
|
"""
|
|
content = await build_statistics_csv(db, run)
|
|
# Header only: nothing was measured during this production.
|
|
if len(content.strip().splitlines()) <= 1:
|
|
return None
|
|
|
|
directory = Path(settings.upload_path) / EXPORT_SUBDIR
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
|
|
stamp = to_local(run.closed_at or utcnow()).strftime("%Y%m%d_%H%M%S")
|
|
filename = f"production_{run.id}_{stamp}.csv"
|
|
# A large run's file can take a moment to write: off the event loop, so
|
|
# closing one run does not stall every other request being served.
|
|
await asyncio.to_thread(
|
|
(directory / filename).write_text, content, encoding="utf-8-sig"
|
|
)
|
|
|
|
# utf-8-sig: the BOM is what makes Excel read accented characters correctly,
|
|
# and these files are opened in Excel on the shop floor.
|
|
|
|
await db.execute(
|
|
update(Measurement)
|
|
.where(Measurement.production_run_id == run.id)
|
|
.values(synced_to_csv=True)
|
|
)
|
|
|
|
return f"{EXPORT_SUBDIR}/{filename}"
|