Coverage for binette/io_manager.py: 99%
160 statements
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 13:33 +0000
« prev ^ index » next coverage.py v7.17.3, created at 2026-11-17 13:33 +0000
1import gzip
2import logging
3from collections import defaultdict
4from collections.abc import Iterable
5from pathlib import Path
7import pandas as pd
8import pyfastx
10from binette.bin_manager import Bin
11from binette.checkpoint import atomic_replace, partial_path
13logger = logging.getLogger(__name__)
16def get_paths_common_prefix_suffix(
17 paths: list[Path],
18) -> tuple[list[str], list[str], list[str]]:
19 """
20 Determine the common prefix parts, suffix parts, and common extensions of the last part of a list of pathlib.Path objects.
22 :param paths: List of pathlib.Path objects.
23 :return: A tuple containing three lists:
24 - The common prefix parts.
25 - The common suffix parts.
26 - The common extensions of the last part of the paths.
27 """
28 # Extract parts for all paths
29 parts = [list(path.parts) for path in paths]
31 # Find the common prefix
32 if not parts:
33 return [], [], []
35 # Initialize common prefix and suffix lists
36 common_prefix = list(parts[0])
37 common_suffix = list(parts[0])
38 # Determine common prefix
39 for part_tuple in parts[1:]:
40 common_prefix_length = min(len(common_prefix), len(part_tuple))
41 common_prefix = [
42 common_prefix[i]
43 for i in range(common_prefix_length)
44 if common_prefix[: i + 1] == part_tuple[: i + 1]
45 ]
46 if not common_prefix:
47 break
49 # Determine common suffix
50 for part_tuple in parts[1:]:
51 common_suffix_length = min(len(common_suffix), len(part_tuple))
52 common_suffix = [
53 common_suffix[-i]
54 for i in range(1, common_suffix_length + 1)
55 if common_suffix[-i:] == part_tuple[-i:]
56 ]
57 if not common_suffix:
58 break
59 if len(parts) > 1:
60 common_suffix.reverse()
62 # Determine common extensions of the last part of the paths
63 if len(paths) == 1:
64 common_extensions = paths[0].suffixes
65 else:
66 common_extensions = list(paths[0].suffixes)
67 for path in paths[1:]:
68 common_extension_length = min(len(common_extensions), len(path.suffixes))
69 common_extensions = [
70 common_extensions[i]
71 for i in range(common_extension_length)
72 if common_extensions[i] == path.suffixes[i]
73 ]
74 if not common_extensions:
75 break
77 return common_prefix, common_suffix, common_extensions
80def infer_bin_set_names_from_input_paths(input_bins: list[Path]) -> dict[str, Path]:
81 """
82 Infer bin set names from a list of bin input directories or files.
84 :param input_bins: List of input bin directories or files.
85 :return: Dictionary mapping inferred bin names to their corresponding directories or files.
86 """
87 bin_name_to_bin_dir = {}
89 common_prefix, common_suffix, common_extensions = get_paths_common_prefix_suffix(
90 input_bins
91 )
93 for path in input_bins:
94 specific_parts = path.parts[
95 len(common_prefix) : len(path.parts) - len(common_suffix)
96 ]
98 if not common_suffix and common_extensions:
99 last_specific_part = specific_parts[-1].split(".")[
100 : -len(common_extensions)
101 ]
102 specific_parts = list(specific_parts[:-1]) + last_specific_part
104 bin_set_name = "/".join(specific_parts)
105 if bin_set_name == "":
106 bin_set_name = path.as_posix()
108 bin_name_to_bin_dir[bin_set_name] = path
110 logger.debug(f"Input bins: {' '.join([path.as_posix() for path in input_bins])}")
111 logger.debug(f"Common prefix to remove: {common_prefix}")
112 logger.debug(f"Common suffix to remove: {common_suffix}")
113 logger.debug(f"Common extension to remove: {common_suffix}")
114 logger.debug(f"bin_name_to_bin_dir: {bin_name_to_bin_dir}")
116 return bin_name_to_bin_dir
119def write_bin_info(bins: Iterable[Bin], output: Path, add_contigs: bool = False):
120 """
121 Write bin information to a TSV file.
123 :param bins: List of Bin objects.
124 :param output: Output file path for writing the TSV.
125 :param add_contigs: Flag indicating whether to include contig information.
126 """
128 # Define columns for the DataFrame
129 columns = [
130 "name",
131 "origin",
132 "is_original",
133 "original_name",
134 "completeness",
135 "contamination",
136 "score",
137 "checkm2_model",
138 "size",
139 "N50",
140 "coding_density",
141 "contig_count",
142 ]
143 if add_contigs:
144 columns.append("contigs")
146 # Create a list of dictionaries to build the DataFrame
147 data = []
148 for bin_obj in sorted(
149 bins, key=lambda x: (-x.score, -x.N50, -x.is_original, x.contigs_key)
150 ):
151 original_name = bin_obj.original_name if bin_obj.original_name else bin_obj.name
152 origins = bin_obj.origin if bin_obj.is_original else {"binette"}
154 bin_info = {
155 "name": bin_obj.name,
156 "origin": ";".join(origins),
157 "is_original": bin_obj.is_original,
158 "original_name": original_name,
159 "completeness": bin_obj.completeness,
160 "contamination": bin_obj.contamination,
161 "score": round(bin_obj.score, 2),
162 "checkm2_model": bin_obj.checkm2_model,
163 "size": bin_obj.length,
164 "N50": bin_obj.N50,
165 "coding_density": round(bin_obj.coding_density, 4)
166 if bin_obj.coding_density is not None
167 else None,
168 "contig_count": len(bin_obj.contigs),
169 }
171 if add_contigs:
172 bin_info["contigs"] = ";".join(str(c) for c in bin_obj.contigs)
174 data.append(bin_info)
176 # Create pandas DataFrame and write to TSV
177 df = pd.DataFrame(data, columns=columns)
178 df.to_csv(output, sep="\t", index=False)
181def write_bins_fasta(
182 selected_bins: list[Bin],
183 contigs_fasta: Path,
184 outdir: Path,
185 contigs_names: list[str],
186 max_buffer_size: int = 50_000_000,
187):
188 """
189 Write selected bins' contigs to separate FASTA files using pyfastx.Fastx (no index).
190 Buffer entries by total character size, not just number of sequences.
192 :param selected_bins: List of Bin objects with .id and .contigs.
193 :param contigs_fasta: Path to the input FASTA file.
194 :param outdir: Directory to save bin FASTA files.
195 :param max_buffer_size: Maximum total character size to buffer before flushing.
196 """
197 # Map contig name to bin IDs
198 contig_to_bin = {}
199 for sbin in selected_bins:
200 for contig_id in sbin.contigs:
201 contig_name = contigs_names[contig_id]
202 contig_to_bin[contig_name] = sbin.name
204 assert len(contig_to_bin) == sum(len(sbin.contigs) for sbin in selected_bins), (
205 "Some contigs are present in multiple bins but should be unique."
206 )
208 write_bins_fasta_from_contig_to_bin(
209 contig_to_bin, contigs_fasta, outdir, max_buffer_size
210 )
213def write_bins_fasta_from_contig_to_bin(
214 contig_to_bin: dict[str, str],
215 contigs_fasta: Path,
216 outdir: Path,
217 max_buffer_size: int = 50_000_000,
218):
219 """
220 Write one FASTA file per bin from a mapping of contig name to bin name.
222 :param contig_to_bin: Mapping of contig name to bin name.
223 :param contigs_fasta: Path to the input FASTA file.
224 :param outdir: Directory to save bin FASTA files.
225 :param max_buffer_size: Maximum total character size to buffer before flushing.
226 """
227 outdir.mkdir(parents=True, exist_ok=True)
229 # Clear existing files for the bins to write
230 for bin_name in set(contig_to_bin.values()):
231 out_path = outdir / f"{bin_name}.fa"
232 if out_path.exists():
233 out_path.unlink() # remove the file
235 buffer = defaultdict(list)
236 buffer_size = 0
238 def flush_buffer():
239 nonlocal buffer_size
240 for bin_name, seqs in buffer.items():
241 if seqs:
242 with open(outdir / f"{bin_name}.fa", "a") as f:
243 f.writelines(seqs)
244 buffer.clear()
245 buffer_size = 0
247 for name, seq in pyfastx.Fastx(contigs_fasta.as_posix()):
248 bin_name = contig_to_bin.get(name)
249 if not bin_name:
250 continue
252 fasta_entry = f">{name}\n{seq}\n"
253 entry_size = len(fasta_entry)
255 buffer[bin_name].append(fasta_entry)
257 buffer_size += entry_size
258 if buffer_size >= max_buffer_size:
259 flush_buffer()
261 flush_buffer()
264def check_contig_consistency(
265 contigs_from_assembly: Iterable[str],
266 contigs_from_elsewhere: Iterable[str],
267 assembly_file: str,
268 elsewhere_file: str,
269):
270 """
271 Check the consistency of contig names between different sources.
273 :param contigs_from_assembly: List of contig names from the assembly file.
274 :param contigs_from_elsewhere: List of contig names from an external source.
275 :param assembly_file: Path to the assembly file.
276 :param elsewhere_file: Path to the file from an external source.
277 :raises AssertionError: If inconsistencies in contig names are found.
278 """
279 logger.debug("Checking contig consistency")
280 are_contigs_consistent = len(
281 set(contigs_from_elsewhere) | set(contigs_from_assembly)
282 ) <= len(set(contigs_from_assembly))
284 issue_countigs = len(set(contigs_from_elsewhere) - set(contigs_from_assembly))
286 message = (
287 f"{issue_countigs} contigs found in file '{elsewhere_file}' "
288 f"were not found in assembly_file '{assembly_file}'"
289 )
290 assert are_contigs_consistent, message
293def write_contig_coding_length(
294 contig_to_coding_length: dict[str, int | None], output_file: Path
295) -> None:
296 """
297 Write the coding length of each contig to a (gzipped) TSV file.
299 The file is written atomically so an interrupted run never leaves a partial file.
301 :param contig_to_coding_length: Mapping of contig name to coding length (or None).
302 :param output_file: Output path. Compressed with gzip if it ends with '.gz'.
303 """
304 proper_open = gzip.open if output_file.name.endswith(".gz") else open
305 tmp_file = partial_path(output_file)
306 with proper_open(tmp_file, "wt") as fl:
307 for contig, coding_length in contig_to_coding_length.items():
308 value = "NA" if coding_length is None else int(coding_length)
309 fl.write(f"{contig}\t{value}\n")
310 atomic_replace(tmp_file, output_file)
313def read_contig_coding_length(input_file: Path) -> dict[str, int | None]:
314 """
315 Read a file written by :func:`write_contig_coding_length`.
317 :param input_file: Path of the coding length file.
318 :return: Mapping of contig name to coding length (or None).
319 """
320 proper_open = gzip.open if input_file.name.endswith(".gz") else open
321 contig_to_coding_length: dict[str, int | None] = {}
322 with proper_open(input_file, "rt") as fl:
323 for line in fl:
324 contig, value = line.rstrip("\n").split("\t")
325 contig_to_coding_length[contig] = None if value == "NA" else int(value)
326 return contig_to_coding_length
329def clear_previous_outputs(final_bins_dir: Path, original_bin_report_dir: Path) -> None:
330 """
331 Remove final outputs from a previous run that may not be overwritten by the current
332 one (e.g. bins written with another prefix). Only files written by Binette are removed.
334 :param final_bins_dir: Directory of final bin FASTA files.
335 :param original_bin_report_dir: Directory of input bin quality reports.
336 """
337 to_remove = [
338 *final_bins_dir.glob("*.fa"),
339 *original_bin_report_dir.glob("input_bins_*.tsv"),
340 ]
341 if to_remove:
342 logger.info(f"Removing {len(to_remove)} result files from a previous run")
343 for path in to_remove:
344 path.unlink()
347def write_contig2bin_table(
348 selected_bins: list[Bin],
349 output_file: Path,
350 contigs_names: list[str],
351):
352 """
353 Write a simple TSV file mapping contig IDs to bin IDs.
355 :param selected_bins: List of selected Bin objects.
356 :param output_file: Path to the output TSV file.
357 :param contigs_names: List of contig names where index corresponds to contig ID.
358 """
359 logger.info(f"Writing contig2bin table to '{output_file}'")
361 # Ensure output directory exists
362 output_file.parent.mkdir(parents=True, exist_ok=True)
364 with open(output_file, "w") as f:
365 # Write contig to bin mappings
366 for bin_obj in selected_bins:
367 for contig_index in bin_obj.contigs:
368 contig_name = contigs_names[contig_index]
369 f.write(f"{contig_name}\t{bin_obj.name}\n")
371 total_entries = sum(len(bin_obj.contigs) for bin_obj in selected_bins)
372 logger.debug(f"Successfully wrote contig2bin table with {total_entries} entries")
375def read_contig2bin_table(input_file: Path) -> dict[str, str]:
376 """
377 Read a headerless contig-to-bin TSV file written by :func:`write_contig2bin_table`.
379 :param input_file: Path to the contig2bin table.
380 :return: Mapping of contig name to bin name.
381 """
382 contig_to_bins = {}
383 with open(input_file) as fl:
384 for line in fl:
385 if line.strip():
386 contig, bin_name = line.rstrip("\n").split("\t")
387 contig_to_bins[contig] = bin_name
388 return contig_to_bins
391def write_original_bin_metrics(original_bins: list[Bin], original_bin_report_dir: Path):
392 """
393 Write metrics of original input bins to a specified directory.
395 This function writes the metrics for each bin set to a TSV file in the specified directory.
396 Each bin set will have its own TSV file named according to its set name.
398 :param original_bins: A set containing input bins
399 :param original_bin_report_dir: The directory path (Path) where the bin metrics will be saved.
400 """
402 original_bin_report_dir.mkdir(parents=True, exist_ok=True)
404 bin_set_name_to_bins = defaultdict(list)
405 for bin_obj in original_bins:
406 for origin in bin_obj.origin:
407 bin_set_name_to_bins[origin].append(bin_obj)
409 for i, (set_name, bins) in enumerate(sorted(bin_set_name_to_bins.items())):
410 bins_metric_file = (
411 original_bin_report_dir
412 / f"input_bins_{i + 1}.{set_name.replace('/', '_')}.tsv"
413 )
415 logger.debug(
416 f"Writing metrics for bin set '{set_name}' to file '{bins_metric_file}'"
417 )
418 write_bin_info(bins, bins_metric_file)
420 logger.debug("Completed writing all original input bin metrics")