Coverage for binette/io_manager.py: 99%

160 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-17 13:33 +0000

1import gzip 

2import logging 

3from collections import defaultdict 

4from collections.abc import Iterable 

5from pathlib import Path 

6 

7import pandas as pd 

8import pyfastx 

9 

10from binette.bin_manager import Bin 

11from binette.checkpoint import atomic_replace, partial_path 

12 

13logger = logging.getLogger(__name__) 

14 

15 

16def get_paths_common_prefix_suffix( 

17 paths: list[Path], 

18) -> tuple[list[str], list[str], list[str]]: 

19 """ 

20 Determine the common prefix parts, suffix parts, and common extensions of the last part of a list of pathlib.Path objects. 

21 

22 :param paths: List of pathlib.Path objects. 

23 :return: A tuple containing three lists: 

24 - The common prefix parts. 

25 - The common suffix parts. 

26 - The common extensions of the last part of the paths. 

27 """ 

28 # Extract parts for all paths 

29 parts = [list(path.parts) for path in paths] 

30 

31 # Find the common prefix 

32 if not parts: 

33 return [], [], [] 

34 

35 # Initialize common prefix and suffix lists 

36 common_prefix = list(parts[0]) 

37 common_suffix = list(parts[0]) 

38 # Determine common prefix 

39 for part_tuple in parts[1:]: 

40 common_prefix_length = min(len(common_prefix), len(part_tuple)) 

41 common_prefix = [ 

42 common_prefix[i] 

43 for i in range(common_prefix_length) 

44 if common_prefix[: i + 1] == part_tuple[: i + 1] 

45 ] 

46 if not common_prefix: 

47 break 

48 

49 # Determine common suffix 

50 for part_tuple in parts[1:]: 

51 common_suffix_length = min(len(common_suffix), len(part_tuple)) 

52 common_suffix = [ 

53 common_suffix[-i] 

54 for i in range(1, common_suffix_length + 1) 

55 if common_suffix[-i:] == part_tuple[-i:] 

56 ] 

57 if not common_suffix: 

58 break 

59 if len(parts) > 1: 

60 common_suffix.reverse() 

61 

62 # Determine common extensions of the last part of the paths 

63 if len(paths) == 1: 

64 common_extensions = paths[0].suffixes 

65 else: 

66 common_extensions = list(paths[0].suffixes) 

67 for path in paths[1:]: 

68 common_extension_length = min(len(common_extensions), len(path.suffixes)) 

69 common_extensions = [ 

70 common_extensions[i] 

71 for i in range(common_extension_length) 

72 if common_extensions[i] == path.suffixes[i] 

73 ] 

74 if not common_extensions: 

75 break 

76 

77 return common_prefix, common_suffix, common_extensions 

78 

79 

80def infer_bin_set_names_from_input_paths(input_bins: list[Path]) -> dict[str, Path]: 

81 """ 

82 Infer bin set names from a list of bin input directories or files. 

83 

84 :param input_bins: List of input bin directories or files. 

85 :return: Dictionary mapping inferred bin names to their corresponding directories or files. 

86 """ 

87 bin_name_to_bin_dir = {} 

88 

89 common_prefix, common_suffix, common_extensions = get_paths_common_prefix_suffix( 

90 input_bins 

91 ) 

92 

93 for path in input_bins: 

94 specific_parts = path.parts[ 

95 len(common_prefix) : len(path.parts) - len(common_suffix) 

96 ] 

97 

98 if not common_suffix and common_extensions: 

99 last_specific_part = specific_parts[-1].split(".")[ 

100 : -len(common_extensions) 

101 ] 

102 specific_parts = list(specific_parts[:-1]) + last_specific_part 

103 

104 bin_set_name = "/".join(specific_parts) 

105 if bin_set_name == "": 

106 bin_set_name = path.as_posix() 

107 

108 bin_name_to_bin_dir[bin_set_name] = path 

109 

110 logger.debug(f"Input bins: {' '.join([path.as_posix() for path in input_bins])}") 

111 logger.debug(f"Common prefix to remove: {common_prefix}") 

112 logger.debug(f"Common suffix to remove: {common_suffix}") 

113 logger.debug(f"Common extension to remove: {common_suffix}") 

114 logger.debug(f"bin_name_to_bin_dir: {bin_name_to_bin_dir}") 

115 

116 return bin_name_to_bin_dir 

117 

118 

119def write_bin_info(bins: Iterable[Bin], output: Path, add_contigs: bool = False): 

120 """ 

121 Write bin information to a TSV file. 

122 

123 :param bins: List of Bin objects. 

124 :param output: Output file path for writing the TSV. 

125 :param add_contigs: Flag indicating whether to include contig information. 

126 """ 

127 

128 # Define columns for the DataFrame 

129 columns = [ 

130 "name", 

131 "origin", 

132 "is_original", 

133 "original_name", 

134 "completeness", 

135 "contamination", 

136 "score", 

137 "checkm2_model", 

138 "size", 

139 "N50", 

140 "coding_density", 

141 "contig_count", 

142 ] 

143 if add_contigs: 

144 columns.append("contigs") 

145 

146 # Create a list of dictionaries to build the DataFrame 

147 data = [] 

148 for bin_obj in sorted( 

149 bins, key=lambda x: (-x.score, -x.N50, -x.is_original, x.contigs_key) 

150 ): 

151 original_name = bin_obj.original_name if bin_obj.original_name else bin_obj.name 

152 origins = bin_obj.origin if bin_obj.is_original else {"binette"} 

153 

154 bin_info = { 

155 "name": bin_obj.name, 

156 "origin": ";".join(origins), 

157 "is_original": bin_obj.is_original, 

158 "original_name": original_name, 

159 "completeness": bin_obj.completeness, 

160 "contamination": bin_obj.contamination, 

161 "score": round(bin_obj.score, 2), 

162 "checkm2_model": bin_obj.checkm2_model, 

163 "size": bin_obj.length, 

164 "N50": bin_obj.N50, 

165 "coding_density": round(bin_obj.coding_density, 4) 

166 if bin_obj.coding_density is not None 

167 else None, 

168 "contig_count": len(bin_obj.contigs), 

169 } 

170 

171 if add_contigs: 

172 bin_info["contigs"] = ";".join(str(c) for c in bin_obj.contigs) 

173 

174 data.append(bin_info) 

175 

176 # Create pandas DataFrame and write to TSV 

177 df = pd.DataFrame(data, columns=columns) 

178 df.to_csv(output, sep="\t", index=False) 

179 

180 

181def write_bins_fasta( 

182 selected_bins: list[Bin], 

183 contigs_fasta: Path, 

184 outdir: Path, 

185 contigs_names: list[str], 

186 max_buffer_size: int = 50_000_000, 

187): 

188 """ 

189 Write selected bins' contigs to separate FASTA files using pyfastx.Fastx (no index). 

190 Buffer entries by total character size, not just number of sequences. 

191 

192 :param selected_bins: List of Bin objects with .id and .contigs. 

193 :param contigs_fasta: Path to the input FASTA file. 

194 :param outdir: Directory to save bin FASTA files. 

195 :param max_buffer_size: Maximum total character size to buffer before flushing. 

196 """ 

197 # Map contig name to bin IDs 

198 contig_to_bin = {} 

199 for sbin in selected_bins: 

200 for contig_id in sbin.contigs: 

201 contig_name = contigs_names[contig_id] 

202 contig_to_bin[contig_name] = sbin.name 

203 

204 assert len(contig_to_bin) == sum(len(sbin.contigs) for sbin in selected_bins), ( 

205 "Some contigs are present in multiple bins but should be unique." 

206 ) 

207 

208 write_bins_fasta_from_contig_to_bin( 

209 contig_to_bin, contigs_fasta, outdir, max_buffer_size 

210 ) 

211 

212 

213def write_bins_fasta_from_contig_to_bin( 

214 contig_to_bin: dict[str, str], 

215 contigs_fasta: Path, 

216 outdir: Path, 

217 max_buffer_size: int = 50_000_000, 

218): 

219 """ 

220 Write one FASTA file per bin from a mapping of contig name to bin name. 

221 

222 :param contig_to_bin: Mapping of contig name to bin name. 

223 :param contigs_fasta: Path to the input FASTA file. 

224 :param outdir: Directory to save bin FASTA files. 

225 :param max_buffer_size: Maximum total character size to buffer before flushing. 

226 """ 

227 outdir.mkdir(parents=True, exist_ok=True) 

228 

229 # Clear existing files for the bins to write 

230 for bin_name in set(contig_to_bin.values()): 

231 out_path = outdir / f"{bin_name}.fa" 

232 if out_path.exists(): 

233 out_path.unlink() # remove the file 

234 

235 buffer = defaultdict(list) 

236 buffer_size = 0 

237 

238 def flush_buffer(): 

239 nonlocal buffer_size 

240 for bin_name, seqs in buffer.items(): 

241 if seqs: 

242 with open(outdir / f"{bin_name}.fa", "a") as f: 

243 f.writelines(seqs) 

244 buffer.clear() 

245 buffer_size = 0 

246 

247 for name, seq in pyfastx.Fastx(contigs_fasta.as_posix()): 

248 bin_name = contig_to_bin.get(name) 

249 if not bin_name: 

250 continue 

251 

252 fasta_entry = f">{name}\n{seq}\n" 

253 entry_size = len(fasta_entry) 

254 

255 buffer[bin_name].append(fasta_entry) 

256 

257 buffer_size += entry_size 

258 if buffer_size >= max_buffer_size: 

259 flush_buffer() 

260 

261 flush_buffer() 

262 

263 

264def check_contig_consistency( 

265 contigs_from_assembly: Iterable[str], 

266 contigs_from_elsewhere: Iterable[str], 

267 assembly_file: str, 

268 elsewhere_file: str, 

269): 

270 """ 

271 Check the consistency of contig names between different sources. 

272 

273 :param contigs_from_assembly: List of contig names from the assembly file. 

274 :param contigs_from_elsewhere: List of contig names from an external source. 

275 :param assembly_file: Path to the assembly file. 

276 :param elsewhere_file: Path to the file from an external source. 

277 :raises AssertionError: If inconsistencies in contig names are found. 

278 """ 

279 logger.debug("Checking contig consistency") 

280 are_contigs_consistent = len( 

281 set(contigs_from_elsewhere) | set(contigs_from_assembly) 

282 ) <= len(set(contigs_from_assembly)) 

283 

284 issue_countigs = len(set(contigs_from_elsewhere) - set(contigs_from_assembly)) 

285 

286 message = ( 

287 f"{issue_countigs} contigs found in file '{elsewhere_file}' " 

288 f"were not found in assembly_file '{assembly_file}'" 

289 ) 

290 assert are_contigs_consistent, message 

291 

292 

293def write_contig_coding_length( 

294 contig_to_coding_length: dict[str, int | None], output_file: Path 

295) -> None: 

296 """ 

297 Write the coding length of each contig to a (gzipped) TSV file. 

298 

299 The file is written atomically so an interrupted run never leaves a partial file. 

300 

301 :param contig_to_coding_length: Mapping of contig name to coding length (or None). 

302 :param output_file: Output path. Compressed with gzip if it ends with '.gz'. 

303 """ 

304 proper_open = gzip.open if output_file.name.endswith(".gz") else open 

305 tmp_file = partial_path(output_file) 

306 with proper_open(tmp_file, "wt") as fl: 

307 for contig, coding_length in contig_to_coding_length.items(): 

308 value = "NA" if coding_length is None else int(coding_length) 

309 fl.write(f"{contig}\t{value}\n") 

310 atomic_replace(tmp_file, output_file) 

311 

312 

313def read_contig_coding_length(input_file: Path) -> dict[str, int | None]: 

314 """ 

315 Read a file written by :func:`write_contig_coding_length`. 

316 

317 :param input_file: Path of the coding length file. 

318 :return: Mapping of contig name to coding length (or None). 

319 """ 

320 proper_open = gzip.open if input_file.name.endswith(".gz") else open 

321 contig_to_coding_length: dict[str, int | None] = {} 

322 with proper_open(input_file, "rt") as fl: 

323 for line in fl: 

324 contig, value = line.rstrip("\n").split("\t") 

325 contig_to_coding_length[contig] = None if value == "NA" else int(value) 

326 return contig_to_coding_length 

327 

328 

329def clear_previous_outputs(final_bins_dir: Path, original_bin_report_dir: Path) -> None: 

330 """ 

331 Remove final outputs from a previous run that may not be overwritten by the current 

332 one (e.g. bins written with another prefix). Only files written by Binette are removed. 

333 

334 :param final_bins_dir: Directory of final bin FASTA files. 

335 :param original_bin_report_dir: Directory of input bin quality reports. 

336 """ 

337 to_remove = [ 

338 *final_bins_dir.glob("*.fa"), 

339 *original_bin_report_dir.glob("input_bins_*.tsv"), 

340 ] 

341 if to_remove: 

342 logger.info(f"Removing {len(to_remove)} result files from a previous run") 

343 for path in to_remove: 

344 path.unlink() 

345 

346 

347def write_contig2bin_table( 

348 selected_bins: list[Bin], 

349 output_file: Path, 

350 contigs_names: list[str], 

351): 

352 """ 

353 Write a simple TSV file mapping contig IDs to bin IDs. 

354 

355 :param selected_bins: List of selected Bin objects. 

356 :param output_file: Path to the output TSV file. 

357 :param contigs_names: List of contig names where index corresponds to contig ID. 

358 """ 

359 logger.info(f"Writing contig2bin table to '{output_file}'") 

360 

361 # Ensure output directory exists 

362 output_file.parent.mkdir(parents=True, exist_ok=True) 

363 

364 with open(output_file, "w") as f: 

365 # Write contig to bin mappings 

366 for bin_obj in selected_bins: 

367 for contig_index in bin_obj.contigs: 

368 contig_name = contigs_names[contig_index] 

369 f.write(f"{contig_name}\t{bin_obj.name}\n") 

370 

371 total_entries = sum(len(bin_obj.contigs) for bin_obj in selected_bins) 

372 logger.debug(f"Successfully wrote contig2bin table with {total_entries} entries") 

373 

374 

375def read_contig2bin_table(input_file: Path) -> dict[str, str]: 

376 """ 

377 Read a headerless contig-to-bin TSV file written by :func:`write_contig2bin_table`. 

378 

379 :param input_file: Path to the contig2bin table. 

380 :return: Mapping of contig name to bin name. 

381 """ 

382 contig_to_bins = {} 

383 with open(input_file) as fl: 

384 for line in fl: 

385 if line.strip(): 

386 contig, bin_name = line.rstrip("\n").split("\t") 

387 contig_to_bins[contig] = bin_name 

388 return contig_to_bins 

389 

390 

391def write_original_bin_metrics(original_bins: list[Bin], original_bin_report_dir: Path): 

392 """ 

393 Write metrics of original input bins to a specified directory. 

394 

395 This function writes the metrics for each bin set to a TSV file in the specified directory. 

396 Each bin set will have its own TSV file named according to its set name. 

397 

398 :param original_bins: A set containing input bins 

399 :param original_bin_report_dir: The directory path (Path) where the bin metrics will be saved. 

400 """ 

401 

402 original_bin_report_dir.mkdir(parents=True, exist_ok=True) 

403 

404 bin_set_name_to_bins = defaultdict(list) 

405 for bin_obj in original_bins: 

406 for origin in bin_obj.origin: 

407 bin_set_name_to_bins[origin].append(bin_obj) 

408 

409 for i, (set_name, bins) in enumerate(sorted(bin_set_name_to_bins.items())): 

410 bins_metric_file = ( 

411 original_bin_report_dir 

412 / f"input_bins_{i + 1}.{set_name.replace('/', '_')}.tsv" 

413 ) 

414 

415 logger.debug( 

416 f"Writing metrics for bin set '{set_name}' to file '{bins_metric_file}'" 

417 ) 

418 write_bin_info(bins, bins_metric_file) 

419 

420 logger.debug("Completed writing all original input bin metrics")