Coverage for binette/checkpoint.py: 100%
133 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 13:33 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 13:33 +0000
1"""
2Module : checkpoint
3Description : Track intermediate results so that a run can be safely resumed.
5Binette stores a small JSON manifest in its temporary directory. The manifest
6records the identity of the run (fingerprints of its data inputs) and, for each
7completed step, the inputs it was computed from and the files it produced
8(size + sha256). A step is only reused when its inputs match the current run
9and its output files are intact on disk.
11Copyright : (c) Jean Mainguy
12License : GPL-3.0
13"""
15import hashlib
16import json
17import logging
18import os
19from collections.abc import Iterable
20from pathlib import Path
21from typing import Any
23import binette as binette_init
25logger = logging.getLogger(__name__)
27FORMAT_VERSION = 1
28CHECKPOINT_FILENAME = "checkpoint.json"
30# Files and directories written by Binette in its output directory. Their presence
31# without a checkpoint manifest indicates a run from an older Binette version.
32BINETTE_OUTPUT_MARKERS = (
33 "final_bins_quality_reports.tsv",
34 "final_contig_to_bin.tsv",
35 "final_bins",
36 "input_bins_quality_reports",
37 "temporary_files/assembly_proteins.faa.gz",
38 "temporary_files/diamond_result.tsv.gz",
39)
41RUN_INPUT_LABELS = {
42 "contigs": "contig sequences (--contigs)",
43 "bins": "input bin sets (--bin_dirs / --contig2bin_tables)",
44 "proteins": "protein file (--proteins)",
45 "checkm2_db": "CheckM2 database (--checkm2_db)",
46}
49class OutputDirConflictError(Exception):
50 """Raised when the output directory holds results from a different run."""
53def sha256_file(path: Path, chunk_size: int = 1 << 20) -> str:
54 """Compute the sha256 hex digest of a file."""
55 digest = hashlib.sha256()
56 with open(path, "rb") as fl:
57 while chunk := fl.read(chunk_size):
58 digest.update(chunk)
59 return digest.hexdigest()
62def hash_json(obj: Any) -> str:
63 """Compute a stable sha256 hex digest of a JSON-serialisable object."""
64 return hashlib.sha256(
65 json.dumps(obj, sort_keys=True, separators=(",", ":")).encode()
66 ).hexdigest()
69def contigs_fingerprint(contig_to_seq_digest: dict[str, str]) -> str:
70 """
71 Fingerprint a set of contigs from per-contig sequence digests.
73 The fingerprint is independent of the order of contigs in the FASTA file.
75 :param contig_to_seq_digest: Mapping of contig name to the digest of its sequence.
76 """
77 digest = hashlib.sha256()
78 for name in sorted(contig_to_seq_digest):
79 digest.update(f"{name}\t{contig_to_seq_digest[name]}\n".encode())
80 return digest.hexdigest()
83def bins_fingerprint(bin_set_name_to_bins_info: dict[str, list[dict]]) -> str:
84 """
85 Fingerprint input bin sets: bin set names, bin names and their contigs.
87 :param bin_set_name_to_bins_info: Mapping of bin set name to a list of bin info dicts
88 with keys "bin_name" and "contigs".
89 """
90 normalized = {
91 set_name: sorted(
92 (str(info["bin_name"]), sorted(info["contigs"])) for info in bins_info
93 )
94 for set_name, bins_info in bin_set_name_to_bins_info.items()
95 }
96 return hash_json(normalized)
99def checkm2_db_identity(db_path: Path) -> dict[str, Any]:
100 """
101 Identify a CheckM2 diamond database by file name and size.
103 The database is too large to hash. The path is not used, so a database copied
104 to another location (e.g. node-local scratch) is still recognised.
105 """
106 return {"name": db_path.name, "size": db_path.stat().st_size}
109def partial_path(path: Path) -> Path:
110 """Temporary path used to write a file before atomically moving it into place."""
111 return path.with_name(f"partial_{path.name}")
114def atomic_replace(tmp_path: Path, final_path: Path) -> None:
115 """Move a fully written temporary file to its final location."""
116 os.replace(tmp_path, final_path)
119class Checkpoint:
120 """Manifest of the intermediate results of a Binette run."""
122 def __init__(self, manifest_path: Path, root: Path, data: dict[str, Any]):
123 """
124 :param manifest_path: Path of the JSON manifest.
125 :param root: Directory that recorded output paths are relative to.
126 :param data: Manifest content.
127 """
128 self.manifest_path = manifest_path
129 self.root = root
130 self.data = data
132 @classmethod
133 def new(
134 cls, manifest_path: Path, root: Path, run_inputs: dict[str, Any]
135 ) -> "Checkpoint":
136 return cls(
137 manifest_path,
138 root,
139 {
140 "format_version": FORMAT_VERSION,
141 "binette_version": binette_init.__version__,
142 "run_inputs": run_inputs,
143 "steps": {},
144 },
145 )
147 @classmethod
148 def load(cls, manifest_path: Path, root: Path) -> tuple["Checkpoint | None", str]:
149 """
150 Load a manifest from disk.
152 :return: The checkpoint (or None) and a reason when it could not be loaded.
153 """
154 if not manifest_path.exists():
155 return None, f"no checkpoint file '{manifest_path}'"
156 try:
157 with open(manifest_path) as fl:
158 data = json.load(fl)
159 except (OSError, json.JSONDecodeError) as exc:
160 return None, f"checkpoint file '{manifest_path}' is unreadable ({exc})"
162 if not isinstance(data, dict) or data.get("format_version") != FORMAT_VERSION:
163 return None, (
164 f"checkpoint file '{manifest_path}' has an unsupported format version"
165 )
166 data.setdefault("steps", {})
167 return cls(manifest_path, root, data), ""
169 @property
170 def run_inputs(self) -> dict[str, Any]:
171 return self.data.get("run_inputs", {})
173 def save(self) -> None:
174 """Write the manifest atomically."""
175 self.manifest_path.parent.mkdir(parents=True, exist_ok=True)
176 tmp = partial_path(self.manifest_path)
177 with open(tmp, "w") as fl:
178 json.dump(self.data, fl, indent=2, sort_keys=True)
179 atomic_replace(tmp, self.manifest_path)
181 def _relative(self, path: Path) -> str:
182 return Path(os.path.relpath(path, self.root)).as_posix()
184 def step_status(self, step: str, inputs: dict[str, Any]) -> tuple[bool, str]:
185 """
186 Check whether a recorded step can be reused.
188 :param step: Step name.
189 :param inputs: Inputs of the step in the current run.
190 :return: (is_valid, reason). The reason explains why a step is not reusable.
191 """
192 record = self.data["steps"].get(step)
193 if record is None:
194 return False, "no record of a completed previous run of this step"
196 if record.get("inputs") != inputs:
197 changed = sorted(
198 key
199 for key in set(inputs) | set(record.get("inputs", {}))
200 if inputs.get(key) != record.get("inputs", {}).get(key)
201 )
202 return False, f"inputs changed since previous run ({', '.join(changed)})"
204 for rel_path, expected in record.get("outputs", {}).items():
205 path = self.root / rel_path
206 if not path.is_file():
207 return False, f"file '{path}' is missing"
208 if path.stat().st_size != expected["size"]:
209 return False, f"file '{path}' has changed or is incomplete"
210 if sha256_file(path) != expected["sha256"]:
211 return False, f"file '{path}' has changed"
213 return True, ""
215 def can_reuse(
216 self, step: str, inputs: dict[str, Any], resume: bool, description: str
217 ) -> bool:
218 """
219 Decide whether a recorded step can be reused and log the decision.
221 :param step: Name of the step.
222 :param inputs: Inputs of the step in the current run.
223 :param resume: Whether resuming is enabled.
224 :param description: Human readable description of the step results.
225 """
226 if not resume:
227 return False
229 if step not in self.data["steps"]:
230 logger.debug(f"No {description} from a previous run to reuse")
231 return False
233 is_valid, reason = self.step_status(step, inputs)
234 if is_valid:
235 logger.info(f"Resuming: reusing {description} from a previous run")
236 else:
237 logger.info(f"Cannot reuse {description} from a previous run: {reason}")
238 return is_valid
240 def step_output_digest(self, step: str, path: Path) -> str | None:
241 """Return the recorded sha256 of an output file of a step."""
242 record = self.data["steps"].get(step, {})
243 output = record.get("outputs", {}).get(self._relative(path))
244 return None if output is None else output["sha256"]
246 def record_step(
247 self, step: str, inputs: dict[str, Any], outputs: Iterable[Path]
248 ) -> None:
249 """Record a completed step and save the manifest."""
250 self.data["steps"][step] = {
251 "inputs": inputs,
252 "outputs": {
253 self._relative(path): {
254 "size": path.stat().st_size,
255 "sha256": sha256_file(path),
256 }
257 for path in outputs
258 },
259 }
260 self.save()
262 def invalidate(self, *steps: str) -> None:
263 """Forget steps (e.g. before their outputs are rewritten) and save."""
264 changed = False
265 for step in steps:
266 if self.data["steps"].pop(step, None) is not None:
267 changed = True
268 if changed:
269 self.save()
272def find_binette_outputs(outdir: Path) -> list[Path]:
273 """List Binette output files/directories present in an output directory."""
274 return [outdir / m for m in BINETTE_OUTPUT_MARKERS if (outdir / m).exists()]
277def check_outdir(
278 outdir: Path,
279 manifest_path: Path,
280 run_inputs: dict[str, Any],
281 force: bool,
282) -> Checkpoint:
283 """
284 Make sure the output directory can be used for the current run.
286 - Empty/new output directory: a new checkpoint is created.
287 - Output directory from a run with the same inputs: its checkpoint is returned.
288 - Output directory from a run with different inputs, or from an older Binette
289 version that cannot be verified: raises OutputDirConflictError, unless
290 ``force`` is set, in which case a new checkpoint is created.
292 :param outdir: Output directory of the run.
293 :param manifest_path: Path of the checkpoint manifest.
294 :param run_inputs: Fingerprints of the data inputs of the current run.
295 :param force: Allow reusing an output directory from a different run.
296 :raises OutputDirConflictError: When the output directory belongs to another run.
297 """
298 checkpoint, reason = Checkpoint.load(manifest_path, root=outdir)
300 if checkpoint is not None:
301 if checkpoint.run_inputs == run_inputs:
302 logger.debug(f"Output directory '{outdir}' matches the current inputs")
303 return checkpoint
305 changed = [
306 RUN_INPUT_LABELS.get(key, key)
307 for key in sorted(set(run_inputs) | set(checkpoint.run_inputs))
308 if run_inputs.get(key) != checkpoint.run_inputs.get(key)
309 ]
310 problem = (
311 f"Output directory '{outdir}' contains results from a previous Binette run "
312 f"with different inputs. Changed: {'; '.join(changed)}."
313 )
314 else:
315 existing = find_binette_outputs(outdir)
316 if not existing and not manifest_path.exists():
317 checkpoint = Checkpoint.new(manifest_path, outdir, run_inputs)
318 checkpoint.save()
319 return checkpoint
320 if manifest_path.exists():
321 problem = (
322 f"Output directory '{outdir}' contains results from a previous "
323 f"Binette run that cannot be verified: {reason}."
324 )
325 else:
326 problem = (
327 f"Output directory '{outdir}' contains results from a previous Binette "
328 "run that cannot be verified (no checkpoint information, "
329 "probably produced by an older Binette version)."
330 )
332 if not force:
333 raise OutputDirConflictError(
334 f"{problem} Use a different output directory, or use --force to discard "
335 "the previous intermediate files and overwrite the results."
336 )
338 logger.warning(
339 f"{problem} --force is set: previous intermediate files are discarded "
340 "and results will be overwritten."
341 )
342 checkpoint = Checkpoint.new(manifest_path, outdir, run_inputs)
343 checkpoint.save()
344 return checkpoint