Coverage for binette/checkpoint.py: 100%

133 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-17 13:33 +0000

1""" 

2Module : checkpoint 

3Description : Track intermediate results so that a run can be safely resumed. 

4 

5Binette stores a small JSON manifest in its temporary directory. The manifest 

6records the identity of the run (fingerprints of its data inputs) and, for each 

7completed step, the inputs it was computed from and the files it produced 

8(size + sha256). A step is only reused when its inputs match the current run 

9and its output files are intact on disk. 

10 

11Copyright : (c) Jean Mainguy 

12License : GPL-3.0 

13""" 

14 

15import hashlib 

16import json 

17import logging 

18import os 

19from collections.abc import Iterable 

20from pathlib import Path 

21from typing import Any 

22 

23import binette as binette_init 

24 

25logger = logging.getLogger(__name__) 

26 

27FORMAT_VERSION = 1 

28CHECKPOINT_FILENAME = "checkpoint.json" 

29 

30# Files and directories written by Binette in its output directory. Their presence 

31# without a checkpoint manifest indicates a run from an older Binette version. 

32BINETTE_OUTPUT_MARKERS = ( 

33 "final_bins_quality_reports.tsv", 

34 "final_contig_to_bin.tsv", 

35 "final_bins", 

36 "input_bins_quality_reports", 

37 "temporary_files/assembly_proteins.faa.gz", 

38 "temporary_files/diamond_result.tsv.gz", 

39) 

40 

41RUN_INPUT_LABELS = { 

42 "contigs": "contig sequences (--contigs)", 

43 "bins": "input bin sets (--bin_dirs / --contig2bin_tables)", 

44 "proteins": "protein file (--proteins)", 

45 "checkm2_db": "CheckM2 database (--checkm2_db)", 

46} 

47 

48 

49class OutputDirConflictError(Exception): 

50 """Raised when the output directory holds results from a different run.""" 

51 

52 

53def sha256_file(path: Path, chunk_size: int = 1 << 20) -> str: 

54 """Compute the sha256 hex digest of a file.""" 

55 digest = hashlib.sha256() 

56 with open(path, "rb") as fl: 

57 while chunk := fl.read(chunk_size): 

58 digest.update(chunk) 

59 return digest.hexdigest() 

60 

61 

62def hash_json(obj: Any) -> str: 

63 """Compute a stable sha256 hex digest of a JSON-serialisable object.""" 

64 return hashlib.sha256( 

65 json.dumps(obj, sort_keys=True, separators=(",", ":")).encode() 

66 ).hexdigest() 

67 

68 

69def contigs_fingerprint(contig_to_seq_digest: dict[str, str]) -> str: 

70 """ 

71 Fingerprint a set of contigs from per-contig sequence digests. 

72 

73 The fingerprint is independent of the order of contigs in the FASTA file. 

74 

75 :param contig_to_seq_digest: Mapping of contig name to the digest of its sequence. 

76 """ 

77 digest = hashlib.sha256() 

78 for name in sorted(contig_to_seq_digest): 

79 digest.update(f"{name}\t{contig_to_seq_digest[name]}\n".encode()) 

80 return digest.hexdigest() 

81 

82 

83def bins_fingerprint(bin_set_name_to_bins_info: dict[str, list[dict]]) -> str: 

84 """ 

85 Fingerprint input bin sets: bin set names, bin names and their contigs. 

86 

87 :param bin_set_name_to_bins_info: Mapping of bin set name to a list of bin info dicts 

88 with keys "bin_name" and "contigs". 

89 """ 

90 normalized = { 

91 set_name: sorted( 

92 (str(info["bin_name"]), sorted(info["contigs"])) for info in bins_info 

93 ) 

94 for set_name, bins_info in bin_set_name_to_bins_info.items() 

95 } 

96 return hash_json(normalized) 

97 

98 

99def checkm2_db_identity(db_path: Path) -> dict[str, Any]: 

100 """ 

101 Identify a CheckM2 diamond database by file name and size. 

102 

103 The database is too large to hash. The path is not used, so a database copied 

104 to another location (e.g. node-local scratch) is still recognised. 

105 """ 

106 return {"name": db_path.name, "size": db_path.stat().st_size} 

107 

108 

109def partial_path(path: Path) -> Path: 

110 """Temporary path used to write a file before atomically moving it into place.""" 

111 return path.with_name(f"partial_{path.name}") 

112 

113 

114def atomic_replace(tmp_path: Path, final_path: Path) -> None: 

115 """Move a fully written temporary file to its final location.""" 

116 os.replace(tmp_path, final_path) 

117 

118 

119class Checkpoint: 

120 """Manifest of the intermediate results of a Binette run.""" 

121 

122 def __init__(self, manifest_path: Path, root: Path, data: dict[str, Any]): 

123 """ 

124 :param manifest_path: Path of the JSON manifest. 

125 :param root: Directory that recorded output paths are relative to. 

126 :param data: Manifest content. 

127 """ 

128 self.manifest_path = manifest_path 

129 self.root = root 

130 self.data = data 

131 

132 @classmethod 

133 def new( 

134 cls, manifest_path: Path, root: Path, run_inputs: dict[str, Any] 

135 ) -> "Checkpoint": 

136 return cls( 

137 manifest_path, 

138 root, 

139 { 

140 "format_version": FORMAT_VERSION, 

141 "binette_version": binette_init.__version__, 

142 "run_inputs": run_inputs, 

143 "steps": {}, 

144 }, 

145 ) 

146 

147 @classmethod 

148 def load(cls, manifest_path: Path, root: Path) -> tuple["Checkpoint | None", str]: 

149 """ 

150 Load a manifest from disk. 

151 

152 :return: The checkpoint (or None) and a reason when it could not be loaded. 

153 """ 

154 if not manifest_path.exists(): 

155 return None, f"no checkpoint file '{manifest_path}'" 

156 try: 

157 with open(manifest_path) as fl: 

158 data = json.load(fl) 

159 except (OSError, json.JSONDecodeError) as exc: 

160 return None, f"checkpoint file '{manifest_path}' is unreadable ({exc})" 

161 

162 if not isinstance(data, dict) or data.get("format_version") != FORMAT_VERSION: 

163 return None, ( 

164 f"checkpoint file '{manifest_path}' has an unsupported format version" 

165 ) 

166 data.setdefault("steps", {}) 

167 return cls(manifest_path, root, data), "" 

168 

169 @property 

170 def run_inputs(self) -> dict[str, Any]: 

171 return self.data.get("run_inputs", {}) 

172 

173 def save(self) -> None: 

174 """Write the manifest atomically.""" 

175 self.manifest_path.parent.mkdir(parents=True, exist_ok=True) 

176 tmp = partial_path(self.manifest_path) 

177 with open(tmp, "w") as fl: 

178 json.dump(self.data, fl, indent=2, sort_keys=True) 

179 atomic_replace(tmp, self.manifest_path) 

180 

181 def _relative(self, path: Path) -> str: 

182 return Path(os.path.relpath(path, self.root)).as_posix() 

183 

184 def step_status(self, step: str, inputs: dict[str, Any]) -> tuple[bool, str]: 

185 """ 

186 Check whether a recorded step can be reused. 

187 

188 :param step: Step name. 

189 :param inputs: Inputs of the step in the current run. 

190 :return: (is_valid, reason). The reason explains why a step is not reusable. 

191 """ 

192 record = self.data["steps"].get(step) 

193 if record is None: 

194 return False, "no record of a completed previous run of this step" 

195 

196 if record.get("inputs") != inputs: 

197 changed = sorted( 

198 key 

199 for key in set(inputs) | set(record.get("inputs", {})) 

200 if inputs.get(key) != record.get("inputs", {}).get(key) 

201 ) 

202 return False, f"inputs changed since previous run ({', '.join(changed)})" 

203 

204 for rel_path, expected in record.get("outputs", {}).items(): 

205 path = self.root / rel_path 

206 if not path.is_file(): 

207 return False, f"file '{path}' is missing" 

208 if path.stat().st_size != expected["size"]: 

209 return False, f"file '{path}' has changed or is incomplete" 

210 if sha256_file(path) != expected["sha256"]: 

211 return False, f"file '{path}' has changed" 

212 

213 return True, "" 

214 

215 def can_reuse( 

216 self, step: str, inputs: dict[str, Any], resume: bool, description: str 

217 ) -> bool: 

218 """ 

219 Decide whether a recorded step can be reused and log the decision. 

220 

221 :param step: Name of the step. 

222 :param inputs: Inputs of the step in the current run. 

223 :param resume: Whether resuming is enabled. 

224 :param description: Human readable description of the step results. 

225 """ 

226 if not resume: 

227 return False 

228 

229 if step not in self.data["steps"]: 

230 logger.debug(f"No {description} from a previous run to reuse") 

231 return False 

232 

233 is_valid, reason = self.step_status(step, inputs) 

234 if is_valid: 

235 logger.info(f"Resuming: reusing {description} from a previous run") 

236 else: 

237 logger.info(f"Cannot reuse {description} from a previous run: {reason}") 

238 return is_valid 

239 

240 def step_output_digest(self, step: str, path: Path) -> str | None: 

241 """Return the recorded sha256 of an output file of a step.""" 

242 record = self.data["steps"].get(step, {}) 

243 output = record.get("outputs", {}).get(self._relative(path)) 

244 return None if output is None else output["sha256"] 

245 

246 def record_step( 

247 self, step: str, inputs: dict[str, Any], outputs: Iterable[Path] 

248 ) -> None: 

249 """Record a completed step and save the manifest.""" 

250 self.data["steps"][step] = { 

251 "inputs": inputs, 

252 "outputs": { 

253 self._relative(path): { 

254 "size": path.stat().st_size, 

255 "sha256": sha256_file(path), 

256 } 

257 for path in outputs 

258 }, 

259 } 

260 self.save() 

261 

262 def invalidate(self, *steps: str) -> None: 

263 """Forget steps (e.g. before their outputs are rewritten) and save.""" 

264 changed = False 

265 for step in steps: 

266 if self.data["steps"].pop(step, None) is not None: 

267 changed = True 

268 if changed: 

269 self.save() 

270 

271 

272def find_binette_outputs(outdir: Path) -> list[Path]: 

273 """List Binette output files/directories present in an output directory.""" 

274 return [outdir / m for m in BINETTE_OUTPUT_MARKERS if (outdir / m).exists()] 

275 

276 

277def check_outdir( 

278 outdir: Path, 

279 manifest_path: Path, 

280 run_inputs: dict[str, Any], 

281 force: bool, 

282) -> Checkpoint: 

283 """ 

284 Make sure the output directory can be used for the current run. 

285 

286 - Empty/new output directory: a new checkpoint is created. 

287 - Output directory from a run with the same inputs: its checkpoint is returned. 

288 - Output directory from a run with different inputs, or from an older Binette 

289 version that cannot be verified: raises OutputDirConflictError, unless 

290 ``force`` is set, in which case a new checkpoint is created. 

291 

292 :param outdir: Output directory of the run. 

293 :param manifest_path: Path of the checkpoint manifest. 

294 :param run_inputs: Fingerprints of the data inputs of the current run. 

295 :param force: Allow reusing an output directory from a different run. 

296 :raises OutputDirConflictError: When the output directory belongs to another run. 

297 """ 

298 checkpoint, reason = Checkpoint.load(manifest_path, root=outdir) 

299 

300 if checkpoint is not None: 

301 if checkpoint.run_inputs == run_inputs: 

302 logger.debug(f"Output directory '{outdir}' matches the current inputs") 

303 return checkpoint 

304 

305 changed = [ 

306 RUN_INPUT_LABELS.get(key, key) 

307 for key in sorted(set(run_inputs) | set(checkpoint.run_inputs)) 

308 if run_inputs.get(key) != checkpoint.run_inputs.get(key) 

309 ] 

310 problem = ( 

311 f"Output directory '{outdir}' contains results from a previous Binette run " 

312 f"with different inputs. Changed: {'; '.join(changed)}." 

313 ) 

314 else: 

315 existing = find_binette_outputs(outdir) 

316 if not existing and not manifest_path.exists(): 

317 checkpoint = Checkpoint.new(manifest_path, outdir, run_inputs) 

318 checkpoint.save() 

319 return checkpoint 

320 if manifest_path.exists(): 

321 problem = ( 

322 f"Output directory '{outdir}' contains results from a previous " 

323 f"Binette run that cannot be verified: {reason}." 

324 ) 

325 else: 

326 problem = ( 

327 f"Output directory '{outdir}' contains results from a previous Binette " 

328 "run that cannot be verified (no checkpoint information, " 

329 "probably produced by an older Binette version)." 

330 ) 

331 

332 if not force: 

333 raise OutputDirConflictError( 

334 f"{problem} Use a different output directory, or use --force to discard " 

335 "the previous intermediate files and overwrite the results." 

336 ) 

337 

338 logger.warning( 

339 f"{problem} --force is set: previous intermediate files are discarded " 

340 "and results will be overwritten." 

341 ) 

342 checkpoint = Checkpoint.new(manifest_path, outdir, run_inputs) 

343 checkpoint.save() 

344 return checkpoint