Coverage for PyFHD/pyfhd_tools/pyfhd_setup.py: 69%
337 statements
« prev ^ index » next coverage.py v7.9.1, created at 2025-07-01 10:58 +0800
1from pathlib import Path
2import configargparse
3import argparse
4import time
5import subprocess
6import logging
7from typing import Tuple
8import os
9from importlib.metadata import version
10from glob import glob
11import re
12import sys
13import importlib_resources
16class OrderedBooleanOptionalAction(argparse.BooleanOptionalAction):
17 """
18 OrderedBooleanOptionalAction is a custom action based on BooleanOptionalAction
19 that ensures that the long options are always first in the list of options. More specifically,
20 it ensures the the positive long form is first (i.e. --foo), and the negative (i.e. --no-foo) is second.
21 This was needed as configargparse when using checking the config file and the action set in the argparse,
22 set the arg passed into PyFHD by either option[0] or option[1] if the value in the config file was set to True or False
23 respectively. This also allows easy negation of values from the configuration file with the command line if we need to.
24 For example we can now set silent: true in the configuration file, but then set --no-silent in the command line to override the config file.
26 This is also still ensures that short switches are available in PyFHD with the BooleanOptionalAction.
27 """
29 def __init__(self, *args, **kwargs):
30 super().__init__(*args, **kwargs)
31 longs = [o for o in self.option_strings if o.startswith("--")]
32 shorts = [o for o in self.option_strings if not o.startswith("--")]
33 # put “--foo”, “--no-foo” first, then any shorts like “-s”
34 self.option_strings = longs + shorts
37def pyfhd_parser():
38 """
39 The pyfhd_parser configures the argparse for PyFHD
41 Returns
42 -------
43 configargparse.ArgumentParser
44 The parser for PyFHD which contains the help strings for the terminal and Usage section of the docs.
45 """
47 parser = configargparse.ArgumentParser(
48 prog="PyFHD",
49 description="This is the Python Fast Holographic Deconvolution package, only the observation ID (obs_id) and configuration (-c, --config) is required to start your run, but you should need to modify these arguments below to get something useful. If you don't supply a configuration file, PyFHD will use the default configuration file in the resources/config directory from the PyFHD install.",
50 config_file_parser_class=configargparse.YAMLConfigFileParser,
51 formatter_class=configargparse.RawTextHelpFormatter,
52 )
53 parser.add_argument(
54 "-c",
55 "--config",
56 default=importlib_resources.files("PyFHD.resources.config").joinpath(
57 "pyfhd.yaml"
58 ),
59 is_config_file=True,
60 help="Configuration File Path for PyFHD (By default will find PyFHD in the Python Path and use the default config file PyFHD/resources/config/pyfhd.yaml).",
61 )
62 # Add All the Groups
63 checkpoints = parser.add_argument_group(
64 "Checkpoints", "Activate checkpoints and Load up checkpoints"
65 )
66 instrument = parser.add_argument_group(
67 "Instrument", "Adjust parameters specific to your instrument"
68 )
69 calibration = parser.add_argument_group(
70 "Calibration", "Adjust Parameters for Calibration"
71 )
72 flag = parser.add_argument_group("Flagging", "Adjust Parameters for Flagging")
73 beam = parser.add_argument_group(
74 "Beam Setup", "Adjust Parameters for the Beam Setup"
75 )
76 gridding = parser.add_argument_group("Gridding", "Tune the Gridding in PyFHD")
77 # Ready for deconvolution translation
78 # deconv = parser.add_argument_group("Deconvolution", "Tune the Degridding in PyFHD")
79 export = parser.add_argument_group(
80 "Export", "Adjust the outputs of the PyFHD pipeline"
81 )
82 plotting = parser.add_argument_group(
83 "Plotting", "Adjust the plotting of the PyFHD pipeline"
84 )
85 model = parser.add_argument_group("Model", "Tune the modelling in PyFHD")
86 # Ready for simulation translation
87 # sim = parser.add_argument_group(
88 # "Simulation", "Turn On Simulation and Tune the simulation"
89 # )
90 healpix = parser.add_argument_group("HEALPIX", "Adjust the HEALPIX output")
92 # Version Argument
93 try:
94 from PyFHD.__git__ import __git_commit__, __git_branch__
96 commit = __git_commit__
97 branch = __git_branch__
98 except ImportError:
99 # If the git commit is not available, set it to unknown
100 commit = "Unknown"
101 version_string = f"""\
102 ________________________________________________________________________
103 | ooooooooo. oooooooooooo ooooo ooooo oooooooooo. |
104 | 8888 `Y88. 8888 8 8888 888 888 Y8b |
105 | 888 .d88' oooo ooo 888 888 888 888 888 |
106 | 888ooo88P' `88. .8' 888oooo8 888ooooo888 888 888 |
107 | 888 `88..8' 888 888 888 888 888 |
108 | 888 `888' 888 888 888 888 d88' |
109 | o888o .8' o888o o888o o888o o888bood8P' |
110 | .o..P' |
111 | `Y8P' |
112 |_______________________________________________________________________|
114 Python Fast Holographic Deconvolution
116 Translated from IDL to Python as a collaboration between Astronomy Data and Computing Services (ADACS) and the Epoch of Reionisation (EoR) Team.
118 Repository: https://github.com/EoRImaging/PyFHD
120 Documentation: https://pyfhd.readthedocs.io/en/latest/
122 Version: {version('PyFHD')}
124 Git Commit Hash: {commit} ({branch})
125 """
126 parser.add_argument("-v", "--version", action="version", version=version_string)
128 # General Defaults
129 parser.add_argument(
130 "obs_id",
131 help="The Observation ID as per the MWA file naming standards. Assumes the fits files for this observation is in the uvfits-path. obs_id and uvfits replace file_path_vis from FHD",
132 )
133 parser.add_argument(
134 "-i",
135 "--input-path",
136 type=Path,
137 help="Directory for the uvfits files and other inputs, by default it looks for a directory called input in the working directory",
138 default="./input/",
139 )
140 parser.add_argument(
141 "--get-sample-data",
142 action="store_true",
143 help="Copy sample data from PyFHD package directory to the current working directory. Will copy to an 'input' directory.",
144 )
145 parser.add_argument(
146 "-r",
147 "--recalculate-all",
148 action=OrderedBooleanOptionalAction,
149 help="Forces PyFHD to recalculate all values. This will ignore values set for recalculate-grid, recalculate-beam, as it will set all of them to True",
150 )
151 parser.add_argument(
152 "-s",
153 "--silent",
154 action=OrderedBooleanOptionalAction,
155 help="This PyFHD stops all output to the terminal except in the case of an error and/or exception",
156 )
157 parser.add_argument(
158 "-l",
159 "--log-file",
160 action=OrderedBooleanOptionalAction,
161 help="Logging in a log file is enabled by default, set to False in the config to disable logging to a file.",
162 )
163 parser.add_argument(
164 "--instrument",
165 type=str,
166 default="mwa",
167 choices=["mwa", "lwa", "hera", "other"],
168 help="Set the instrument used for the FHD run, currently only MWA is supported",
169 )
170 parser.add_argument(
171 "--dimension",
172 type=int,
173 default=2048,
174 help="The number of pixels in the UV plane along one axis.",
175 )
176 parser.add_argument(
177 "--elements",
178 type=int,
179 default=2048,
180 help="The number of pixels in the UV plane along the other axis.",
181 )
182 parser.add_argument(
183 "--kbinsize",
184 type=float,
185 default=0.5,
186 help="Size of UV pixels in wavelengths. Given a defined number of pixels in dimension, this sets the UV space extent. This will supersede degpix if dimension is also set.",
187 )
188 parser.add_argument(
189 "--FoV",
190 "--fov",
191 type=float,
192 default=None,
193 help="A proxy for the field of view in degrees. FoV is actually used to determine kbinsize, which will be set to !RaDeg/FoV.\nThis means that the pixel size at phase center times dimension is approximately equal to FoV, which is not equal to the actual field of view owing to larger pixel sizes away from phase center.\nIf set to 0, then kbinsize determines the UV resolution.",
194 )
195 parser.add_argument(
196 "--deproject_w_term",
197 type=float,
198 default=None,
199 help="Enables the function for simple_deproject_w_term and uses the parameter value for the direction value in the function",
200 )
201 parser.add_argument(
202 "--conserve-memory",
203 default=False,
204 action=OrderedBooleanOptionalAction,
205 help="Optionally split many loops into chunks in the case of high memory usage.",
206 )
207 parser.add_argument(
208 "--memory-threshold",
209 type=int,
210 default=1e9,
211 help="Set a memory threshold for each chunk in set in bytes. By default it is set at ~100MB",
212 )
213 parser.add_argument(
214 "--min-baseline",
215 type=float,
216 default=1.0,
217 help="The minimum baseline length in wavelengths to include in the analysis",
218 )
219 parser.add_argument(
220 "--n-pol",
221 type=int,
222 default=2,
223 choices=[0, 2, 4],
224 help="Set number of polarizations to use (XX, YY versus XX, YY, XY, YX).",
225 )
227 # Checkpoints
228 checkpoints.add_argument(
229 "--save-checkpoints",
230 default=False,
231 action=OrderedBooleanOptionalAction,
232 help="Activates PyFHD's checkpointing system and saves them into the output directory",
233 )
234 checkpoints.add_argument(
235 "--obs-checkpoint",
236 default=None,
237 type=Path,
238 help="Load the checkpoint just after creating the observation metadata dictionary, should contain the observation metadata dictionary, uncalibrated visibility parameters, array and weights. If calibrate-checkpoint has been set, then obs-checkpoint will be ignored",
239 )
240 checkpoints.add_argument(
241 "--calibrate-checkpoint",
242 default=None,
243 type=Path,
244 help="Load the checkpoint after calibration containing the observation metadata dictionary with flagged tiles and frequencies, the calibration dictionary containing the gains and the calibrated visibility parameters, model, array and weights.",
245 )
246 checkpoints.add_argument(
247 "--gridding-checkpoint",
248 default=None,
249 type=Path,
250 help="Load the checkpoint after gridding containing the gridded uv planes for the image, weights, variance and filter, with an updated observation metadata dictionary. Should be used in conjunction with the calibrate-checkpoint option",
251 )
253 # Instrument Group
254 instrument.add_argument(
255 "--override-target-phasera",
256 default=None,
257 type=float,
258 help="RA of the target phase center, which overrides the value supplied in the metafits under the header keyword RAPHASE. If the metafits doesn't exist, it ovverides the value supplied in the uvfits under the header keyword RA",
259 )
260 instrument.add_argument(
261 "--override-target-phasedec",
262 default=None,
263 type=float,
264 help="dec of the target phase center, which overrides the value supplied in the metafits under the header keyword DECPHASE. If the metafits doesn't exist, it overrides the value supplied in the uvfits under the header keyword Dec.",
265 )
267 # Calibration Group
268 calibration.add_argument(
269 "-cv",
270 "--calibrate-visibilities",
271 default=False,
272 action=OrderedBooleanOptionalAction,
273 help="Turn on the calibration of the visibilities. If turned on, calibration of the dirty, modelling, and subtraction to make a residual occurs. Otherwise, none of these occur and an uncalibrated dirty cube is output.",
274 )
275 calibration.add_argument(
276 "--transfer-calibration",
277 type=Path,
278 help="The file path of a calibration to be read-in, if you give a directory PyFHD expects there to be a file called <obs_id>_cal.hdf5 using the same observation as you plan to process.",
279 )
280 calibration.add_argument(
281 "--cal-stop",
282 default=False,
283 action=OrderedBooleanOptionalAction,
284 help="Stops the code right after calibration, and saves unflagged model visibilities along with the obs structure in a folder called cal_prerun in the PyFHD file structure.\nThis allows for post-processing calibration steps like multi-day averaging, but still has all of the needed information for minimal reprocessing to get to the calibration step.\nTo run a post-processing run, see keywords model_transfer and transfer_psf",
285 )
286 calibration.add_argument(
287 "--cal-convergence-threshold",
288 type=float,
289 default=1e-7,
290 help="Threshold at which calibration ends. Calibration convergence is quantified by the absolute value of the fractional change in the gains over the last calibration iteration. If this quantity is less than cal_convergence_threshold then calibration terminates.",
291 )
292 calibration.add_argument(
293 "--cal-adaptive-calibration-gain",
294 default=False,
295 action=OrderedBooleanOptionalAction,
296 help="Controls whether to use a Kalman Filter to adjust the gain to use for each iteration of calculating calibration.",
297 )
298 calibration.add_argument(
299 "--cal-base-gain",
300 type=float,
301 default=None,
302 help="The relative weight to give the old calibration solution when averaging with the new. Set to 1. to give equal weight, to 2. to give more weight to the old solution and slow down convergence, or to 0.5 to give greater weight to the new solution and attempt to speed up convergence. If use_adaptive_calibration_gain is set, the weight of the new calibration solutions will be calculated in the range cal_base_gain/2. to 1.0",
303 )
304 calibration.add_argument(
305 "--min-cal-baseline",
306 type=float,
307 default=50.0,
308 help="The minimum baseline length in wavelengths to be used in calibration.",
309 )
310 calibration.add_argument(
311 "--max-cal-baseline",
312 type=float,
313 default=None,
314 help="The maximum baseline length in wavelengths to be used in calibration. If max_baseline is smaller, it will be used instead.",
315 )
316 calibration.add_argument(
317 "--cable-bandpass-fit",
318 default=False,
319 action=OrderedBooleanOptionalAction,
320 help="Average the calibration solutions across tiles within a cable grouping for the particular instrument.\nDependency: instrument_config/<instrument>_cable_length.txt",
321 )
322 calibration.add_argument(
323 "--cal-bp-transfer",
324 type=Path,
325 default=None,
326 help="Use a saved bandpass for bandpass calibration. Read in the specified file with calfits format greatly preferred.",
327 )
328 calibration.add_argument(
329 "--calibration-polyfit",
330 default=False,
331 action=OrderedBooleanOptionalAction,
332 help="Calculates a polynomial fit across the frequency band for the gain, and allows a cable reflection to be fit.\nThe orders of the polynomial fit are determined by cal_phase_degree_fit and cal_amp_degree_fit.\nIf unset, no polynomial fit or cable reflection fit are used.",
333 )
334 calibration.add_argument(
335 "--cal-amp-degree-fit",
336 default=2,
337 type=int,
338 help="The nth order of the polynomial fit over the whole band to create calibration solutions for the amplitude of the gain.\nSetting it to 0 gives a 0th order polynomial fit (one number for the whole band),\n1 gives a 1st order polynomial fit (linear fit),\n2 gives a 2nd order polynomial fit (quadratic),\nn gives nth order polynomial fit.\nRequires calibration_polyfit to be enabled.",
339 )
340 calibration.add_argument(
341 "--cal-phase-degree-fit",
342 default=1,
343 type=int,
344 help="The nth order of the polynomial fit over the whole band to create calibration solutions for the phase of the gain.\nSetting it to 0 gives a 0th order polynomial fit (one number for the whole band),\n1 gives a 1st order polynomial fit (linear fit),\n2 gives a 2nd order polynomial fit (quadratic),\nn gives nth order polynomial fit.\nRequires calibration_polyfit to be enabled.",
345 )
346 calibration.add_argument(
347 "--cal-reflection-hyperresolve",
348 default=False,
349 action=OrderedBooleanOptionalAction,
350 help="Hyperresolve and fit residual gains using nominal reflection modes (calculated from cal_reflection_mode_delay or cal_reflection_mode_theory),\nproducing a finetuned mode fit, amplitude, and phase.\nWill be ignored if cal_reflection_mode_file is set because it is assumed that a file read-in contains mode/amp/phase to use.",
351 )
352 calibration.add_argument(
353 "--cal-reflection-mode-theory",
354 default=150,
355 type=float,
356 help="Calculate theoretical cable reflection modes given the velocity and length data stored in a config file named <instrument>_cable_length.txt.\nFile must have a header line and at least five columns (tile index, tile name, cable length, cable velocity factor, logic on whether to fit (1) or not (0)).\nCan set it to positive/negative cable lengths (see cal_mode_fit) to include/exclude certain cable types.",
357 )
358 calibration.add_argument(
359 "--cal-reflection-mode-delay",
360 default=False,
361 action=OrderedBooleanOptionalAction,
362 help="Calculate cable reflection modes by Fourier transforming the residual gains, removing modes contaminated by frequency flagging, and choosing the maximum mode.",
363 )
364 calibration.add_argument(
365 "--cal-reflection-mode-file",
366 default=False,
367 action=OrderedBooleanOptionalAction,
368 help="Use predetermined cable reflection parameters (mode, amplitude, and phase) in the calibration solutions from a file.\nThe specified format of the text file must have one header line and eleven columns:\ntile index\ntile name\ncable length\ncable velocity factor\nlogic on whether to fit (1) or not (0)\nmode for X\namplitude for X\nphase for X\nmode for Y\namplitude for Y\nphase for Y. The file will be instrument_config of the input directory",
369 )
370 calibration.add_argument(
371 "--calibration-auto-fit",
372 default=False,
373 action=OrderedBooleanOptionalAction,
374 help="Use the autocorrelations to calibrate. This will suffer from increased, correlated noise and bit statistic errors. However, this will save the autos as the gain in the cal structure, which can be a useful diagnostic.",
375 )
376 calibration.add_argument(
377 "--calibration-auto-initialize",
378 default=False,
379 action=OrderedBooleanOptionalAction,
380 help="initialize gain values for calibration with the autocorrelations. If unset, gains will initialize to 1 or the value supplied by cal_gain_init",
381 )
382 calibration.add_argument(
383 "--cal-gain-init",
384 default=1,
385 type=int,
386 help="Initial gain values for calibration. Selecting accurate inital calibration values speeds up calibration and can improve convergence. This keyword will not be used if calibration_auto_initialize is set.",
387 )
388 calibration.add_argument(
389 "--vis-baseline-hist",
390 default=False,
391 action=OrderedBooleanOptionalAction,
392 help="Calculates the vis_baseline_hist dictionary containing the visibility resolution ratio average and standard deviation",
393 )
394 calibration.add_argument(
395 "--bandpass-calibrate",
396 default=False,
397 action=OrderedBooleanOptionalAction,
398 help="Calculates a bandpass.\nThis is an average of tiles by frequency by polarization (default), beamformer-to-LNA cable types by frequency by polarization (see cable_bandpass_fit),\nor over the whole season by pointing by by cable type by frequency by polarization via a read-in file (see saved_run_bp).\nIf unset, no by-frequency bandpass is used",
399 )
400 calibration.add_argument(
401 "--cal-time-average",
402 default=False,
403 action=OrderedBooleanOptionalAction,
404 help="Performs a time average of the model/data visibilities over the time steps in the observation to reduce the number of equations that are used in the linear-least squares solver. This improves computation time, but will downweight longer baseline visibilities due to their faster phase variation.",
405 )
406 calibration.add_argument(
407 "--auto-ratio-calibration",
408 default=False,
409 action=OrderedBooleanOptionalAction,
410 help="Calculates the auto ratios for cable reflections and enables global bandpass",
411 )
412 calibration.add_argument(
413 "--digital-gain-jump-polyfit",
414 default=False,
415 action=OrderedBooleanOptionalAction,
416 help="Perform polynomial fitting for the amplitude separately before and after the highband digital gain jump at 187.515E6.",
417 )
418 calibration.add_argument(
419 "--cal-phase-fit-iter",
420 default=4,
421 type=int,
422 help="Set the iteration number to begin phase calibration. Before this, phase is held fixed and only amplitude is being calibrated.",
423 )
424 calibration.add_argument(
425 "--max-cal-iter",
426 default=100,
427 type=int,
428 help="Sets the maximum number of iterations allowed for the linear least-squares solver to converge during vis_calibrate_subroutine. Ideally do not set this number unless you notice some of the frequencies not reaching convergence within 100 iterations and do not set this number to 5 or below.",
429 )
431 # Flagging Group
432 flag.add_argument(
433 "-fm",
434 "--flag-model",
435 default=False,
436 action=OrderedBooleanOptionalAction,
437 help="Flag the imported model based on time offsets and the tiles. Turn off if you're dealing with an already flagged model or simulation.",
438 )
439 flag.add_argument(
440 "-fv",
441 "--flag-visibilities",
442 default=False,
443 action=OrderedBooleanOptionalAction,
444 help="Flag visibilities based on calculations in vis_flag",
445 )
446 flag.add_argument(
447 "-fc",
448 "--flag-calibration",
449 default=False,
450 action=OrderedBooleanOptionalAction,
451 help="Flags antennas based on calculations in vis_calibration_flag",
452 )
453 flag.add_argument(
454 "-fcf",
455 "--flag-calibration-frequencies",
456 default=False,
457 action=OrderedBooleanOptionalAction,
458 help="If True, flags frequencies based off 0 calibration gain, if False, ignores the calibration gain for frequencies",
459 )
460 flag.add_argument(
461 "-fb",
462 "--flag-basic",
463 default=False,
464 action=OrderedBooleanOptionalAction,
465 help="Flags Frequencies and Tiles based on your configuration, params and the visibility weights.\nThe freq_use, tile_use arrays of obs will be adjusted and the vis_weights_arr adjusted to be in line with the freq_use and tile_use arrays.\nThis should be True always, the only time you should consider turning off basic flagging is when you're dealing with a simulated visibilities and weights in PyFHD",
466 )
467 flag.add_argument(
468 "-ft",
469 "--flag-tiles",
470 default=[],
471 type=list,
472 action="append",
473 help="A list of tile names to manually flag. I repeat, a list of tile names, NOT tile indices",
474 )
475 flag.add_argument(
476 "-ff",
477 "--flag-frequencies",
478 default=False,
479 action=OrderedBooleanOptionalAction,
480 help="When set to False, PyFHD will not flag any frequencies inside of `vis_flag_basic`, `vis_weights_update` or `vis_calibration_flag`.",
481 )
482 flag.add_argument(
483 "--flag-freq-start",
484 default=None,
485 type=float,
486 help="Frequency in MHz to begin the observation. Flags frequencies less than it. Replaces freq_start from FHD",
487 )
488 flag.add_argument(
489 "--flag-freq-end",
490 default=None,
491 type=float,
492 help="Frequency in MHz to end the observation. Flags frequencies greater than it. Replaces freq_end from FHD",
493 )
494 flag.add_argument(
495 "--time-cut",
496 type=list,
497 default=None,
498 help="Seconds to cut (rounded up to next time integration step) from the beginning of the observation. Can also specify a negative time to cut off the end of the observation. Specify a vector to cut at both the start and end.",
499 )
501 # Beam Setup Group
502 beam.add_argument(
503 "-b",
504 "--beam-file-path",
505 type=Path,
506 help="The path to the file containing a sav or fits file",
507 )
508 beam.add_argument(
509 "-ll",
510 "--lazy-load-beam",
511 default=False,
512 action=OrderedBooleanOptionalAction,
513 help="PyFHD will lazy load the beam HDF5 file, allowing PyFHD to be run on much smaller systems with much less memory than FHD",
514 )
515 beam.add_argument(
516 "--recalculate-beam",
517 default=False,
518 action=OrderedBooleanOptionalAction,
519 help="Forces PyFHD to redo the beam setup using PyFHD's beam setup.",
520 )
521 beam.add_argument(
522 "--beam-nfreq-avg",
523 type=int,
524 default=16,
525 help="The number of fine frequency channels to calculate a beam for, using the average of the frequencies.\nThe beam is a function of frequency, and a calculation on the finest level is most correct (beam_nfreq_avg=1).\nHowever, this is computationally difficult for most machines.",
526 )
527 beam.add_argument(
528 "--psf-dim",
529 default=54,
530 type=int,
531 help="Controls the span of the beam in u-v space. Some defaults are 30, 54 (1e6 mask with -2) or 62 (1e7 with -2).",
532 )
533 beam.add_argument(
534 "--psf-resolution",
535 default=100,
536 type=int,
537 help="Super-resolution factor of the psf in UV space. Values greater than 1 increase the resolution of the gridding kernel.",
538 )
539 beam.add_argument(
540 "--beam-mask-threshold",
541 default=100,
542 type=int,
543 help="The factor at which to clip the beam model. For example, a factor of 100 would clip the beam model at 100x down from the maximum value. This removes extraneous and uncertain modelling at low levels.",
544 )
545 beam.add_argument(
546 "--beam-model-version",
547 type=int,
548 default=2,
549 help="A number that indicates the tile beam model calculation.\nThis is dependent on the instrument, and specific calculations are carried out in <instrument>_beam_setup_gain.\nMWA range: 0, 1 (or anything else captured in the else statement), 2\nPAPER range: 1 (or anything else captured in the else statement), 2\nHERA range: 2 (or anything else captured in the else statement)",
550 )
551 beam.add_argument(
552 "--beam-clip-floor",
553 default=False,
554 action=OrderedBooleanOptionalAction,
555 help="Set to subtract the minimum non-zero value of the beam model from all pixels.",
556 )
557 beam.add_argument(
558 "--interpolate-kernel",
559 default=False,
560 action=OrderedBooleanOptionalAction,
561 help="Use interpolation of the gridding kernel while gridding and degridding, rather than selecting the closest super-resolution kernel.",
562 )
563 beam.add_argument(
564 "--beam-per-baseline",
565 default=False,
566 action=OrderedBooleanOptionalAction,
567 help="Set to true if the beams were made with corrective phases given the baseline location, which then enables the gridding to be done per baseline",
568 )
569 beam.add_argument(
570 "--dipole-mutual-coupling-factor",
571 default=False,
572 action=OrderedBooleanOptionalAction,
573 help="Allows a modification to the beam as a result of mutual coupling between dipoles calculated in mwa_dipole_mutual_coupling (See Sutinjo 2015 for more details).",
574 )
575 beam.add_argument(
576 "--beam-offset-time",
577 type=float,
578 default=56,
579 help="Calculate the beam at a specific time within the observation. 0 seconds indicates the start of the observation, and the # of seconds in an observation indicates the end of the observation.",
580 )
582 # Gridding Group
583 gridding.add_argument(
584 "-g",
585 "--recalculate-grid",
586 default=False,
587 action=OrderedBooleanOptionalAction,
588 help="Forces PyFHD to recalculate the gridding function. Replaces grid_recalculate from FHD",
589 )
590 gridding.add_argument(
591 "--image-filter",
592 default="filter_uv_uniform",
593 type=str,
594 choices=[
595 "filter_uv_uniform",
596 "filter_uv_hanning",
597 "filter_uv_natural",
598 "filter_uv_radial",
599 "filter_uv_tapered_uniform",
600 "filter_uv_optimal",
601 ],
602 help="Weighting filter to be applied to resulting snapshot images and fits files. Replaces image_filter_fn from FHD",
603 )
604 gridding.add_argument(
605 "--mask-mirror-indices",
606 default=False,
607 action=OrderedBooleanOptionalAction,
608 help="Inside baseline_grid_location optionally exclude v-axis mirrored baselines",
609 )
610 gridding.add_argument(
611 "--grid-weights",
612 default=False,
613 action=OrderedBooleanOptionalAction,
614 help="Grid the weights for the uv plane",
615 )
616 gridding.add_argument(
617 "--grid-variance",
618 default=False,
619 action=OrderedBooleanOptionalAction,
620 help="Grid the variance for the uv plane",
621 ),
622 gridding.add_argument(
623 "--grid-uniform",
624 default=False,
625 action=OrderedBooleanOptionalAction,
626 help="Grid uniformally by applying a uniform weighted filter to all uv-planes",
627 ),
628 gridding.add_argument(
629 "--grid-spectral",
630 default=False,
631 action=OrderedBooleanOptionalAction,
632 help="Optionally use the spectral index information to scale the uv-plane in gridding",
633 )
635 # Deconvolution Group
636 # deconv.add_argument(
637 # "-d",
638 # "--deconvolve",
639 # default=False,
640 # action=OrderedBooleanOptionalAction,
641 # help="Run Fast Holographic Deconvolution",
642 # )
643 # deconv.add_argument(
644 # "--max-deconvolution-components",
645 # type=int,
646 # default=20000,
647 # help="The number of source components allowed to be found in fast holographic deconvolution.",
648 # )
649 parser.add_argument(
650 "--dft-threshold",
651 default=False,
652 action=OrderedBooleanOptionalAction,
653 help="Set to True to use the DFT approximation. When set equal to 0 the true DFT is calculated for each source.\nIt can also be explicitly set to a value that determines the accuracy of the approximation.",
654 )
655 # deconv.add_argument(
656 # "--return-decon-visibilities",
657 # default=False,
658 # action=OrderedBooleanOptionalAction,
659 # help="When activated degrid and export the visibilities formed from the deconvolution model",
660 # )
661 # deconv.add_argument(
662 # "--deconvolution-filter",
663 # default="filter_uv_uniform",
664 # type=str,
665 # choices=[
666 # "filter_uv_uniform",
667 # "filter_uv_hanning",
668 # "filter_uv_natural",
669 # "filter_uv_radial",
670 # "filter_uv_tapered_uniform",
671 # "filter_uv_optimal",
672 # ],
673 # help="Filter applied to images from deconvolution.",
674 # )
675 # deconv.add_argument(
676 # "--smooth-width",
677 # default=32,
678 # type=int,
679 # help="Integer equal to the size of the region to smooth when filtering out large-scale background fluctuations.",
680 # )
681 # deconv.add_argument(
682 # "--filter-background",
683 # default=False,
684 # action=OrderedBooleanOptionalAction,
685 # help="Filters out large-scale background fluctuations before deconvolving point sources.",
686 # )
688 # Export Group
689 export.add_argument(
690 "-o",
691 "--output-path",
692 type=Path,
693 help="Set the output path for the current run, note a directory will still be created inside the given path",
694 default="./output/",
695 )
696 export.add_argument(
697 "--description",
698 type=str,
699 default=None,
700 help="A more detailed description of the current task, will get applied to the output directory and logging where all output will be stored.\nBy default the date and time is used",
701 )
702 export.add_argument(
703 "--export-images",
704 help="Export fits files and images of the sky.",
705 action=OrderedBooleanOptionalAction,
706 default=True,
707 )
708 export.add_argument(
709 "--snapshot-healpix-export",
710 default=False,
711 action=OrderedBooleanOptionalAction,
712 help="Save model/dirty/residual/weights/variance cubes as healpix arrays, split into even and odd time samples, in preparation for epsilon.",
713 )
714 export.add_argument(
715 "--pad-uv-image",
716 type=float,
717 default=1.0,
718 help="Pad the UV image by this factor with 0's along the outside so that output images are at a higher resolution.",
719 )
720 export.add_argument(
721 "--ring-radius-multi",
722 type=float,
723 default=10,
724 help="Sets the multiplier for the size of the rings around sources in the restored images.\nRing Radius will equal pad-uv-image * ring-radius-multi.\nTo generate restored images without rings, set ring_radius = 0.",
725 )
726 export.add_argument(
727 "--save-obs",
728 default=False,
729 action=OrderedBooleanOptionalAction,
730 help="Save the obs dictionary created during PyFHD's run",
731 )
732 export.add_argument(
733 "--save-params",
734 default=False,
735 action=OrderedBooleanOptionalAction,
736 help="Save the params dictionary created during PyFHD's run",
737 )
738 export.add_argument(
739 "--save-cal",
740 default=False,
741 action=OrderedBooleanOptionalAction,
742 help="Save the calibration dictionary created during PyFHD's run",
743 )
744 export.add_argument(
745 "--save-visibilities",
746 default=False,
747 action=OrderedBooleanOptionalAction,
748 help="Save the raw visibilities, calibrated data visibilities, the model visibilities, and the gridded uv planes",
749 )
750 export.add_argument(
751 "--save-weights",
752 default=False,
753 action=OrderedBooleanOptionalAction,
754 help="Save the raw and calibrated weights from PyFHD's run",
755 )
756 export.add_argument(
757 "--save-healpix-fits",
758 default=False,
759 action=OrderedBooleanOptionalAction,
760 help="Create Healpix fits files. Healpix fits maps are in units Jy/sr. Replaces write_healpix_fits",
761 )
762 export.add_argument(
763 "--save-model",
764 default=False,
765 action=OrderedBooleanOptionalAction,
766 help="Save the model visibilities created transferred in during PyFHD's run as HDF5.",
767 )
769 # Plotting Group
770 plotting.add_argument(
771 "--calibration-plots",
772 default=False,
773 action=OrderedBooleanOptionalAction,
774 help="Turns on the plotting of calibration solutions",
775 )
776 plotting.add_argument(
777 "--gridding-plots",
778 default=False,
779 action=OrderedBooleanOptionalAction,
780 help="Turns on the plotting of the continuum gridding outputs",
781 )
782 plotting.add_argument(
783 "--image-plots",
784 default=False,
785 action=OrderedBooleanOptionalAction,
786 help="Turns on the plotting of the continuum fits images",
787 )
789 # Model Group
790 model.add_argument(
791 "-m",
792 "--model-file-type",
793 default="sav",
794 choices=["sav", "uvfits"],
795 help="Set the file type of the model, by default it looks for sav files of format <obs_id>_params.sav and <obs_id>_vis_model_<pol_name>.sav.\nIf you set uvfits you must put set path using --import-model-uvfits.\nThis argument is required as PyFHD currently cannot produce a model.",
796 )
797 model.add_argument(
798 "--model-file-path",
799 default="./input",
800 type=Path,
801 help='In the case you chose sav for model-file-type then this will be a directory containing all the <obs_id>_params and <obs_id>_vis_model_<pol_name> sav files.\nIn the case you chose uvfits, then the path is to a uvfits file, in which case make sure the phase centre of model data must match the "RA" and "DEC" values in the metafits file (NOT the "RAPHASE" and "DECPHASE").',
802 )
803 model.add_argument(
804 "--allow-sidelobe-model-sources",
805 default=False,
806 action=OrderedBooleanOptionalAction,
807 help="Allows PyFHD to model sources in the sidelobes for subtraction.\nForces the beam_threshold to 0.01 in order to go down to 1%% of the beam to capture sidelobe sources during the generation of a model calibration source catalog for the particular observation.",
808 )
810 # Simultation Group
811 # sim.add_argument(
812 # "-sim",
813 # "--run-simulation",
814 # default=False,
815 # action=OrderedBooleanOptionalAction,
816 # help="Run an in situ simulation, where model visibilities are made and input as the dirty visibilities (see Barry et. al. 2016 for more information on use-cases).\nIn the case where in-situ-sim-input is not provided visibilities will be made within the current PyFHD run.",
817 # )
818 # sim.add_argument(
819 # "--in-situ-sim-input",
820 # type=Path,
821 # default=None,
822 # help="Inputs model visibilities from a previous run, which is the preferred method since that run is independently documented.",
823 # )
824 # sim.add_argument(
825 # "--eor-vis-filepath",
826 # type=Path,
827 # default=None,
828 # help="A path to a file of EoR visibilities to include the EoR in the dirty input visibilities. in-situ-sim-input must be used in order to use this parameter. Replaces eor_savefile from FHD",
829 # )
830 # sim.add_argument(
831 # "--enhance-eor",
832 # type=float,
833 # default=1.0,
834 # help="Input a multiplicative factor to boost the signal of the EoR in the dirty input visibilities. in-situ-sim-input must be used in order to use this parameter.",
835 # )
836 # sim.add_argument(
837 # "--sim-noise",
838 # type=Path,
839 # default=None,
840 # help="Add a uncorrelated thermal noise to the input dirty visibilities from a file, or create them for the run. in-situ-sim-input must be used in order to use this parameter.",
841 # )
842 # sim.add_argument(
843 # "--tile-flag-list",
844 # type=list,
845 # help="A string array of tile names to manually flag tiles. Note that this is an array of tile names, not tile indices!",
846 # )
847 # sim.add_argument(
848 # "--remove-sim-flags",
849 # default=False,
850 # action=OrderedBooleanOptionalAction,
851 # help="Bypass main flagging for in situ simulations and remove all weighting to remove pfb effects and flagged channels.",
852 # )
853 # sim.add_argument(
854 # "--extra-vis-filepath",
855 # type=Path,
856 # default=None,
857 # help="Optionally add general visibilities to the simulation, must be a uvfits file.",
858 # )
860 # HEALPIX Group
861 healpix.add_argument(
862 "--ps-kbinsize",
863 type=float,
864 default=0.5,
865 help="UV pixel size in wavelengths to grid for Healpix cube generation. Overrides ps_fov and the kpix in the obs structure if set.",
866 )
867 healpix.add_argument(
868 "--ps-kspan",
869 type=int,
870 default=0,
871 help="UV plane dimension in wavelengths for Healpix cube generation.\nOverrides ps_dimension and ps_degpix if set.\nIf ps_kspan, ps_dimension, or ps_degpix are not set, the UV plane dimension is calculated from the FoV and the degpix from the obs structure.",
872 )
873 healpix.add_argument(
874 "--ps-beam-threshold",
875 type=float,
876 default=0,
877 help="Minimum value to which to calculate the beam out to in image space. The beam in UV space is pre-calculated and may have its own beam_threshold (see that keyword for more information), and this is only an additional cut in image space.",
878 )
879 healpix.add_argument(
880 "--ps-fov",
881 type=float,
882 default=None,
883 help="Field of view in degrees for Healpix cube generation. Overrides kpix in the obs dictionary if set.",
884 )
885 healpix.add_argument(
886 "--ps-dimension",
887 type=int,
888 default=None,
889 help="UV plane dimension in pixel number for Healpix cube generation. Overrides ps_degpix if set. If ps_kspan, ps_dimension, or ps_degpix are not set, the UV plane dimension is calculated from the FoV and the degpix from the obs dictionary.",
890 )
891 healpix.add_argument(
892 "--ps-degpix",
893 type=float,
894 default=None,
895 help="Degrees per pixel for Healpix cube generation. If ps_kspan, ps_dimension, or ps_degpix are not set, the UV plane dimension is calculated from the FoV and the degpix from the obs dictionary.",
896 )
897 healpix.add_argument(
898 "--ps-nfreq-avg",
899 type=float,
900 default=None,
901 help="A factor to average up the frequency resolution of the HEALPix cubes from the analysis frequency resolution. By default averages by a factor of 2 when this is set to None.",
902 )
903 healpix.add_argument(
904 "--ps-tile-flag-list",
905 type=list,
906 default=[],
907 action="append",
908 help="A list of tile names to manually flag in the healpix export. I repeat, a list of tile names, NOT tile indices.",
909 )
910 healpix.add_argument(
911 "--n-avg",
912 type=int,
913 default=2,
914 help="Number of frequencies to average over to smooth the frequency band.",
915 )
916 healpix.add_argument(
917 "--rephase-weights",
918 default=False,
919 action=OrderedBooleanOptionalAction,
920 help="If turned off, target phase center is the pointing center (as defined by Cotter). Setting to False overrides override_target_phasera and override_target_phasedec",
921 )
922 healpix.add_argument(
923 "--restrict-healpix-inds",
924 default=False,
925 action=OrderedBooleanOptionalAction,
926 help="Only allow gridding of the output healpix cubes to include the healpix pixels specified in a file.\nThis is useful for restricting many observations to have consistent healpix pixels during integration, and saves on memory and walltime.",
927 )
928 healpix.add_argument(
929 "--healpix-inds",
930 default=None,
931 type=Path,
932 help="In the event you want to restrict the healpix indices to a specified file, use a combination of restrict-healpix-inds and this argument to restrict the healpix indexes to your given file rather than a predetermined one from the obs dictionary.",
933 )
934 healpix.add_argument(
935 "--split-ps-export",
936 default=False,
937 action=OrderedBooleanOptionalAction,
938 help="Split up the Healpix outputs into even and odd time samples.\nThis is essential to propogating errors in εppsilon.\nRequires more than one time sample.",
939 )
941 return parser
944def _check_file_exists(config: dict, key: str) -> int:
945 """
946 Helper function to check if the key is not None and if it isn't None, then if file exists given from the config. If it does exist, replace the relative path
947 with the absolute path, so when we write out paths to a config file they are
948 transferable.
950 Parameters
951 ----------
952 config : dict
953 Should be the pyfhd_config
954 key : str
955 The keyword in the config we are checking.
957 Returns
958 -------
959 int
960 Will return 0 if there is no error, 1 if there is.
961 """
962 if config[key]:
963 # If it doesn't exist, add error message
964 if not Path(config[key]).exists(): 964 ↛ 965line 964 didn't jump to line 965 because the condition on line 964 was never true
965 logging.error(
966 "{} has been enabled with a path that doesn't exist, check the path.".format(
967 key
968 )
969 )
970 return 1
971 # If it does exist, replace with the absolute path
972 else:
973 config[key] = Path(os.path.abspath(config[key]))
974 return 0
977def write_collated_yaml_config(
978 pyfhd_config: dict, output_dir: Path, description: str = ""
979):
980 """
981 After all inputs have been validated using `PyFHD.pyfhd_tools.pyfhd_setup`,
982 write out all the arguments gather in `pyfhd_config` and write out to
983 a yaml configuration file. This yaml file can then be fed back into
984 `pyfhd` to exactly duplicate the current run.
986 Parameters
987 ----------
988 pyfhd_config : dict
989 The options from the argparse in a dictionary
990 output_dir : Path
991 Path to save the file to
993 """
995 # for group in parser._action_groups:
996 # group_dict={a.dest:getattr(args,a.dest,None) for a in group._group_actions}
997 # arg_groups[group.title]=argparse.Namespace(**group_dict)
999 with open(
1000 f"{output_dir}/{pyfhd_config['log_name']}{description}.yaml", "w"
1001 ) as outfile:
1002 outfile.write(f"# input options used for run {pyfhd_config['log_name']}\n")
1003 outfile.write("# git hash for this run: {}\n".format(pyfhd_config["commit"]))
1004 for key in pyfhd_config.keys():
1005 # These either a direct argument or are variables set internally to
1006 # each run,so should not appear in the yaml
1007 if key in [
1008 "top_level_dir",
1009 "log_name",
1010 "log_time",
1011 "commit",
1012 "obs_id",
1013 "config_file",
1014 ]:
1015 pass
1016 else:
1017 yaml_key = key.replace("_", "-")
1018 if pyfhd_config[key] == None:
1019 outfile.write(f"{yaml_key} : ~\n")
1020 elif type(pyfhd_config[key]) == float or type(pyfhd_config[key]) == int:
1021 outfile.write(f"{yaml_key} : {pyfhd_config[key]}\n")
1022 elif type(pyfhd_config[key]) == bool:
1023 outfile.write(f"{yaml_key} : {pyfhd_config[key]}\n")
1024 # If it's a list, write it out as a list of strings
1025 # (Unless it's empty)
1026 elif type(pyfhd_config[key]) == list:
1027 if len(pyfhd_config[key]) == 0: 1027 ↛ 1030line 1027 didn't jump to line 1030 because the condition on line 1027 was always true
1028 pass
1029 else:
1030 line = f"{key} : ["
1031 line += f"'{pyfhd_config[key][0]}'"
1032 for item in pyfhd_config[key][1:]:
1033 line += f", '{item}'"
1034 line += "]\n"
1035 # for item in pyfhd_config[key]:
1036 # outfile.write(f"{key} : {item}\n")
1037 outfile.write(line)
1038 else:
1039 outfile.write(f"{yaml_key} : '{pyfhd_config[key]}'\n")
1042def pyfhd_logger(pyfhd_config: dict) -> Tuple[logging.Logger, Path]:
1043 """
1044 Creates the the logger for PyFHD. If silent is True in the pyfhd_config then
1045 the StreamHandler won't be added to logger meaning there will be no terminal output
1046 even if logger is called later. If diable_log is True then the FileHandler won't be added
1047 to the logger preventing the creation fo the log file meaning subsequent calls to the logger
1048 will not add to or create a log file.
1050 Parameters
1051 ----------
1052 pyfhd_config : dict
1053 The options from the argparse in a dictionary
1055 Returns
1056 -------
1057 logger : logging.Logger
1058 The logger with the appropriate handlers added.
1059 output_dir : str
1060 Where the log and FHD outputs are being written to
1061 """
1062 # Get the time, Git commit and setup the name of the output directory
1063 run_time = time.localtime()
1064 stdout_time = time.strftime("%c", run_time)
1065 log_time = time.strftime("%Y_%m_%d_%H_%M_%S", run_time)
1067 if pyfhd_config["description"] is None: 1067 ↛ 1068line 1067 didn't jump to line 1068 because the condition on line 1067 was never true
1068 log_name = "pyfhd_" + log_time
1069 else:
1070 log_name = (
1071 "pyfhd_" + pyfhd_config["description"].replace(" ", "_") + "_" + log_time
1072 )
1074 try:
1075 from PyFHD.__git__ import __git_commit__, __git_branch__, __git_commit_date__
1077 commit = __git_commit__
1078 branch = __git_branch__
1079 except ImportError:
1080 # If the git commit is not available, set it to unknown
1081 commit = "Unknown"
1083 pyfhd_config["commit"] = commit
1084 pyfhd_config["log_name"] = log_name
1085 pyfhd_config["log_time"] = log_time
1086 # Format the starting string for logging
1087 start_string = f"""\
1088 ________________________________________________________________________
1089 | ooooooooo. oooooooooooo ooooo ooooo oooooooooo. |
1090 | 8888 `Y88. 8888 8 8888 888 888 Y8b |
1091 | 888 .d88' oooo ooo 888 888 888 888 888 |
1092 | 888ooo88P' `88. .8' 888oooo8 888ooooo888 888 888 |
1093 | 888 `88..8' 888 888 888 888 888 |
1094 | 888 `888' 888 888 888 888 d88' |
1095 | o888o .8' o888o o888o o888o o888bood8P' |
1096 | .o..P' |
1097 | `Y8P' |
1098 |_______________________________________________________________________|
1099 Python Fast Holographic Deconvolution
1101 Translated from IDL to Python as a collaboration between Astronomy Data and Computing Services (ADACS) and the Epoch of Reionisation (EoR) Team.
1103 Repository: https://github.com/EoRImaging/PyFHD
1105 Documentation: https://pyfhd.readthedocs.io/en/latest/
1107 Version: {version('PyFHD')}
1109 Git Commit Hash: {commit} ({branch})
1111 PyFHD Run Started At: {stdout_time}
1113 Observation ID: {pyfhd_config["obs_id"]}
1115 Confifuration File: {pyfhd_config["config"]}
1117 Validating your input..."""
1119 # Setup logging
1120 log_string = ""
1121 for line in start_string.split("\n"):
1122 log_string += (
1123 line.lstrip().replace("_", " ").replace("| ", "").replace("|", "") + "\n"
1124 )
1125 # Start the PyFHD run
1126 logger = logging.getLogger()
1127 logger.setLevel(logging.INFO)
1128 # Also capture Python Warnings and put it into the log as well
1129 logging.captureWarnings(True)
1130 # Create the logging for the temrinal
1131 if not pyfhd_config["silent"]: 1131 ↛ 1132line 1131 didn't jump to line 1132 because the condition on line 1131 was never true
1132 log_terminal = logging.StreamHandler()
1133 log_terminal.setFormatter(logging.Formatter("%(message)s"))
1134 logger.addHandler(log_terminal)
1136 # Create the output directory path. If the user has selected a description,
1137 # don't use the time in the name - that gets used for the log
1138 if pyfhd_config["description"] is None: 1138 ↛ 1139line 1138 didn't jump to line 1139 because the condition on line 1138 was never true
1139 dir_name = "pyfhd_" + log_time
1140 else:
1141 dir_name = "pyfhd_" + pyfhd_config["description"].replace(" ", "_")
1142 if pyfhd_config["get_sample_data"]: 1142 ↛ 1143line 1142 didn't jump to line 1143 because the condition on line 1142 was never true
1143 output_dir = Path(pyfhd_config["output_path"])
1144 else:
1145 output_dir = Path(pyfhd_config["output_path"], dir_name)
1146 if Path.is_dir(output_dir): 1146 ↛ 1149line 1146 didn't jump to line 1149 because the condition on line 1146 was always true
1147 output_dir_exists = True
1148 else:
1149 output_dir_exists = False
1150 Path.mkdir(output_dir, parents=True, exist_ok=True)
1152 # Create the logger for the file
1153 if pyfhd_config["log_file"]: 1153 ↛ 1154line 1153 didn't jump to line 1154 because the condition on line 1153 was never true
1154 log_file = logging.FileHandler(Path(output_dir, log_name + ".log"))
1155 log_file.setFormatter(logging.Formatter("%(message)s"))
1156 logger.addHandler(log_file)
1158 # Show that start message in the terminal and/or log file, unless both are turned off.
1159 logger.info(log_string)
1160 if not pyfhd_config["silent"]: 1160 ↛ 1161line 1160 didn't jump to line 1161 because the condition on line 1160 was never true
1161 log_terminal.setFormatter(
1162 logging.Formatter(
1163 "%(asctime)s - %(levelname)s:\n\t%(message)s",
1164 datefmt="%Y-%m-%d %H:%M:%S",
1165 )
1166 )
1167 if pyfhd_config["log_file"]: 1167 ↛ 1168line 1167 didn't jump to line 1168 because the condition on line 1167 was never true
1168 log_file.setFormatter(
1169 logging.Formatter(
1170 "%(asctime)s - %(levelname)s:\n\t%(message)s",
1171 datefmt="%Y-%m-%d %H:%M:%S",
1172 )
1173 )
1175 # Write out a config file based
1177 # Stick a warning in the log if running in an already existing dir
1178 if output_dir_exists: 1178 ↛ 1183line 1178 didn't jump to line 1183 because the condition on line 1178 was always true
1179 logger.warning(
1180 f"The output dir {output_dir} already exists, so any existing outputs might be overridden depending on settings."
1181 )
1183 logger.info(
1184 "Logging and configuration file created and copied to here: {}".format(
1185 Path(output_dir).resolve()
1186 )
1187 )
1189 return logger, output_dir
1192def pyfhd_setup(options: argparse.Namespace) -> Tuple[dict, logging.Logger]:
1193 """
1194 Check for any incompatibilities among the options given for starting the PyFHD pipeline as some options
1195 do conflict with each other or have dependencies on other options. This function should catch all of those
1196 potential errors and exit the program with errors once these have been found before any output. This function
1197 should also replace fhd_setup
1199 Parameters
1200 ----------
1201 options : argparse.Namespace
1202 The parsed argparse object.
1204 Returns
1205 -------
1206 pyfhd_config : dict
1207 The configuration dictionary for PyFHD containing all the options.
1208 logger : logging.Logger
1209 The logger with the appropriate handlers added.
1210 """
1211 # Keep track of the errors and warnings.
1212 errors = 0
1213 warnings = 0
1214 pyfhd_config = vars(options)
1215 # Start the logger
1216 logger, output_dir = pyfhd_logger(pyfhd_config)
1217 pyfhd_config["output_dir"] = output_dir
1218 pyfhd_config["top_level_dir"] = str(output_dir).split("/")[-1]
1219 # Check input_path exists and obs_id uvfits and metafits files exist (Error)
1220 if not pyfhd_config["input_path"].exists(): 1220 ↛ 1221line 1220 didn't jump to line 1221 because the condition on line 1220 was never true
1221 logger.error(
1222 "{} doesn't exist, please check your input path".format(options.input_path)
1223 )
1224 errors += 1
1225 obs_uvfits_path = Path(
1226 pyfhd_config["input_path"], pyfhd_config["obs_id"] + ".uvfits"
1227 )
1228 obs_metafits_path = Path(
1229 pyfhd_config["input_path"], pyfhd_config["obs_id"] + ".metafits"
1230 )
1231 if not obs_uvfits_path.exists(): 1231 ↛ 1232line 1231 didn't jump to line 1232 because the condition on line 1231 was never true
1232 logger.error(
1233 "{} doesn't exist, please check your input path".format(obs_uvfits_path)
1234 )
1235 errors += 1
1236 if not obs_metafits_path.exists(): 1236 ↛ 1237line 1236 didn't jump to line 1237 because the condition on line 1236 was never true
1237 logger.error(
1238 "{} doesn't exist, please check your input path".format(obs_metafits_path)
1239 )
1240 errors += 1
1242 # Force PyFHD to recalculate the beam, gridding and mapping functions
1243 if pyfhd_config["recalculate_all"]: 1243 ↛ 1244line 1243 didn't jump to line 1244 because the condition on line 1243 was never true
1244 pyfhd_config["recalculate_beam"] = True
1245 pyfhd_config["recalculate_grid"] = True
1246 pyfhd_config["recalculate_mapfn"] = True
1247 logger.info(
1248 "Recalculate All option has been enabled, the beam, gridding and map function will be recalculated"
1249 )
1251 # If both mapping function and healpix export are on save the visibilities (Warning)
1252 if ( 1252 ↛ 1256line 1252 didn't jump to line 1256 because the condition on line 1252 was never true
1253 pyfhd_config["snapshot_healpix_export"]
1254 and not pyfhd_config["save_visibilities"]
1255 ):
1256 pyfhd_config["save_visibilities"] = True
1257 logger.warning(
1258 "If we're exporting healpix we should also save the visibilities that created them. Setting save_visibilities to True"
1259 )
1260 warnings += 1
1262 if pyfhd_config["beam_offset_time"] < 0: 1262 ↛ 1263line 1262 didn't jump to line 1263 because the condition on line 1262 was never true
1263 pyfhd_config["beam_offset_time"] = 0
1264 logger.warning("You set the offset time to less than 0, it was reset to 0.")
1265 warnings += 1
1267 # If both beam and interp_flag leave a warning, prioritise beam_per_baseline
1268 if pyfhd_config["beam_per_baseline"] and pyfhd_config["interpolate_kernel"]: 1268 ↛ 1269line 1268 didn't jump to line 1269 because the condition on line 1268 was never true
1269 logger.warning(
1270 "Cannot have beam per baseline and interpolation at the same time, turning off interpolation"
1271 )
1272 pyfhd_config["interpolate_kernel"] = False
1274 # If the user has set a beam file, check it exists (Error)
1275 if ( 1275 ↛ 1279line 1275 didn't jump to line 1279 because the condition on line 1275 was never true
1276 pyfhd_config["beam_file_path"] is not None
1277 and not Path(pyfhd_config["beam_file_path"]).exists()
1278 ):
1279 logger.error(
1280 f"Beam file {pyfhd_config['beam_file_path']} does not exist, please check your input path"
1281 )
1282 errors += 1
1284 if pyfhd_config["beam_file_path"] is None: 1284 ↛ 1285line 1284 didn't jump to line 1285 because the condition on line 1284 was never true
1285 logger.info("No beam file was set, PyFHD will calculate the beam.")
1287 if ( 1287 ↛ 1292line 1287 didn't jump to line 1292 because the condition on line 1287 was never true
1288 pyfhd_config["instrument"] == "mwa"
1289 and pyfhd_config["beam_file_path"] is None
1290 and not pyfhd_config["dipole_mutual_coupling_factor"]
1291 ):
1292 logger.warning(
1293 "Since the instrument is MWA and we're calculating the beam, it's recommended to set the dipole mutual coupling factor to True."
1294 )
1295 pyfhd_config["dipole_mutual_coupling_factor"] = True
1296 warnings += 1
1298 # cal_bp_transfer when enabled should point to a file with a saved bandpass (Error)
1299 errors += _check_file_exists(pyfhd_config, "cal_bp_transfer")
1301 # If cal_amp_degree_fit or cal_phase_degree_fit have ben set but calibration_polyfit isn't warn the user (Warning)
1302 if ( 1302 ↛ 1308line 1302 didn't jump to line 1308 because the condition on line 1302 was never true
1303 pyfhd_config["cal_amp_degree_fit"]
1304 or pyfhd_config["cal_phase_degree_fit"]
1305 or pyfhd_config["cal_reflection_mode_theory"]
1306 or pyfhd_config["cal_reflection_mode_delay"]
1307 ) and not pyfhd_config["calibration_polyfit"]:
1308 logger.warning(
1309 "cal_amp_degree_fit and/or cal_amp_phase_fit have been set but calibration_polyfit has been disabled."
1310 )
1311 warnings += 1
1313 # cal_reflection_hyperresolve gets ignored when cal_reflection_mode_file is set (Warning)
1314 if ( 1314 ↛ 1318line 1314 didn't jump to line 1318 because the condition on line 1314 was never true
1315 pyfhd_config["cal_reflection_hyperresolve"]
1316 and pyfhd_config["cal_reflection_mode_file"]
1317 ):
1318 logging.warning(
1319 "cal_reflection_hyperresolve and cal_reflection_mode_file have both been turned on, cal_reflection_mode_file will be prioritised."
1320 )
1321 pyfhd_config["cal_reflection_hyperresolve"] = False
1322 warnings += 1
1324 # cal_reflection_mode_theory and cal_reflection_mode_delay cannot be on at the same time, prioritise mode_theory (Warning)
1325 logic_test = (
1326 1
1327 if pyfhd_config["cal_reflection_mode_file"]
1328 else (
1329 0 + 1
1330 if pyfhd_config["cal_reflection_mode_delay"]
1331 else 0 + 1 if pyfhd_config["cal_reflection_mode_theory"] else 0
1332 )
1333 )
1334 if logic_test > 1: 1334 ↛ 1335line 1334 didn't jump to line 1335 because the condition on line 1334 was never true
1335 logger.warning(
1336 "More than one nominal mode-fitting procedure specified for calibration reflection fits, prioritising cal_reflection_mode_theory"
1337 )
1338 warnings += 1
1339 pyfhd_config["cal_reflection_mode_file"] = False
1340 pyfhd_config["cal_reflection_mode_delay"] = False
1341 pyfhd_config["cal_reflection_mode_theory"] = True
1343 # cal_adaptive_calibration_gain impacts cal_base_gain if cal_base_gain isn't set
1344 if pyfhd_config["cal_base_gain"] == None: 1344 ↛ 1365line 1344 didn't jump to line 1365 because the condition on line 1344 was always true
1345 """
1346 Is set to 0.75 by default, confusingly the FHD code implies if
1347 use_adaptive_calibration_gain isn't active then base gain is 1.0
1348 However because they did this:
1350 IF N_Elements(use_adaptive_calibration_gain) EQ 0 THEN use_adaptive_calibration_gain=0
1351 IF N_Elements(calibration_base_gain) EQ 0 THEN BEGIN
1352 IF N_Elements(use_adaptive_calibration_gain) EQ 0 THEN calibration_base_gain=1. ELSE calibration_base_gain=0.75
1354 Since use_adaptive_calibration_gain is set before the line then
1355 N_ELEMENTS(use_adaptive_calibration_gain) == 1 meaning base_gain is set to 0.75
1356 This confusingly means it isn't checking if use_adaptive_calibraton_gain is actually active
1357 but whether it has been set at all, small but significant difference.
1358 """
1359 pyfhd_config["cal_base_gain"] = 0.75
1361 # calibration_catalog_file_path depends on a file (Error)
1362 # errors += _check_file_exists(pyfhd_config, "calibration_catalog_file_path")
1364 # transfer_calibration depends on a file (Error)
1365 errors += _check_file_exists(pyfhd_config, "transfer_calibration")
1367 # smooth-width depends on filter_background (Warning)
1368 # if not pyfhd_config["filter_background"] and pyfhd_config["smooth_width"]:
1369 # logger.warning(
1370 # "filter_background must be True for smooth_width to have any effect"
1371 # )
1372 # warnings += 1
1374 # allow_sidelobe_model_sources depends on model_visibilities (Error)
1375 if ( 1375 ↛ 1379line 1375 didn't jump to line 1379 because the condition on line 1375 was never true
1376 pyfhd_config["allow_sidelobe_model_sources"]
1377 and not pyfhd_config["model_visibilities"]
1378 ):
1379 logger.error(
1380 "allow_sidelobe_model_sources shouldn't be True when model_visibilities is not, check if you meant to turn on model_visibilities"
1381 )
1382 errors += 1
1384 # if importing model visiblities from a uvfits file, check that file
1385 # exists
1386 if pyfhd_config["model_file_path"]: 1386 ↛ 1389line 1386 didn't jump to line 1389 because the condition on line 1386 was always true
1387 errors += _check_file_exists(pyfhd_config, "model_file_path")
1389 if pyfhd_config["model_file_path"] == "sav": 1389 ↛ 1391line 1389 didn't jump to line 1391 because the condition on line 1389 was never true
1390 # We're expecting to find a params file, then a vis_model_XX and vis_model_YY at the very least
1391 if not Path.exists(
1392 Path(
1393 pyfhd_config["model_file_path"], f"{pyfhd_config['obs_id']}_params.sav"
1394 )
1395 ):
1396 errors += 1
1397 logger.error(
1398 "You selected the model-file-path and sav, but PyFHD can't find the sav file for the model params"
1399 )
1400 files_in_model_path = glob(f"{pyfhd_config['model_file_path']}/*")
1401 pattern = rf".*{re.escape(pyfhd_config['obs_id'])}.*\.sav$"
1402 regex = re.compile(pattern)
1403 matching_files = [
1404 file_path for file_path in files_in_model_path if regex.match(file_path)
1405 ]
1406 if len(matching_files) <= 2:
1407 errors + 1
1408 logger.error(
1409 f"You are missing some required files to read in the model visibilities from sav files, here is the list of found sav files: {matching_files}."
1410 )
1411 elif pyfhd_config["n_pol"] and len(matching_files) < pyfhd_config["n_pol"] + 1:
1412 errors += 1
1413 logger.error(
1414 f"You are missing files based on the number of polarizations you have set, you should have a params file then {pyfhd_config['n_pol']} polarization files. Here is the list of found sav files: {matching_files}."
1415 )
1416 elif pyfhd_config["n_pol"] and len(matching_files) > pyfhd_config["n_pol"] + 1:
1417 warnings += 1
1418 logger.warning(
1419 f"You have more files than expected for the number of polarizations you set, you set {pyfhd_config['n_pol']} polarizations but found {len(matching_files)- 1} polarization files. You can most likely ignore this warning. Here is the list of found sav files: {matching_files}."
1420 )
1421 elif not pyfhd_config["n_pol"]:
1422 warnings += 1
1423 logger.warning(
1424 f"Since you have told PyFHD before hand you are using 0 polarizations and letting the uvfits header set the number of polarizations PyFHD have no way to validate if the number of savs is correct, check the list of found files carefully: {matching_files}. If you're sure this is fine, ignore this warning."
1425 )
1427 # Entirety of Simulation Group depends on run-simulation (Error)
1428 # if not pyfhd_config["run_simulation"] and (
1429 # pyfhd_config["in_situ_sim_input"]
1430 # or pyfhd_config["eor_vis_filepath"]
1431 # or pyfhd_config["sim_noise"]
1432 # ):
1433 # logger.error(
1434 # "run_simulation should be True if you're planning on running any type of simulation and therefore using in_situ_sim_input, eor_vis_filepath or sim_noise shouldn't be used when run_simulation is False"
1435 # )
1436 # errors += 1
1438 # in-situ-sim-input depends on a file (Error)
1439 # errors += _check_file_exists(pyfhd_config, "in_situ_sim_input")
1441 # eor_vis_filepath depends on a file (Error)
1442 # errors += _check_file_exists(pyfhd_config, "eor_vis_filepath")
1444 # enhance_eor depends on eor_vis_filepath when its not 1
1445 # if pyfhd_config["enhance_eor"] > 1 and pyfhd_config["eor_vis_filepath"]:
1446 # logger.error(
1447 # "enhance_eor is only used when importing general visibilities for a simulation, it should stay as 1 when eor_vis_filepath is not being used"
1448 # )
1449 # errors += 1
1451 # sim_noise depends on a file (Error)
1452 # errors += _check_file_exists(pyfhd_config, "sim_noise")
1454 # restrict_healpix_inds depends on a file (Error)
1455 if ( 1455 ↛ 1459line 1455 didn't jump to line 1459 because the condition on line 1455 was never true
1456 pyfhd_config["healpix_inds"] is not None
1457 and pyfhd_config["restrict_healpix_inds"]
1458 ):
1459 errors += _check_file_exists(pyfhd_config, "healpix_inds")
1461 pyfhd_config["ring_radius"] = (
1462 pyfhd_config["pad_uv_image"] * pyfhd_config["ring_radius_multi"]
1463 )
1465 # --------------------------------------------------------------------------
1466 # Checks are finished, report any errors or warnings
1467 # --------------------------------------------------------------------------
1468 # If there are any errors exit the program.
1469 if errors: 1469 ↛ 1470line 1469 didn't jump to line 1470 because the condition on line 1469 was never true
1470 logger.error(
1471 "{} errors detected, check the log above to see the errors, stopping PyFHD now".format(
1472 errors
1473 )
1474 )
1475 # Close the handlers in the log
1476 for handler in logger.handlers:
1477 handler.close()
1478 sys.exit()
1480 if warnings: 1480 ↛ 1481line 1480 didn't jump to line 1481 because the condition on line 1480 was never true
1481 logger.warning(
1482 "{} warnings detected, check the log above, these may cause some weird behavior".format(
1483 warnings
1484 )
1485 )
1486 logger.info("Input validated, starting PyFHD run now")
1488 # Create the config directory
1489 if not pyfhd_config["get_sample_data"]: 1489 ↛ 1494line 1489 didn't jump to line 1494 because the condition on line 1489 was always true
1490 config_path = Path(output_dir, "config")
1491 config_path.mkdir(exist_ok=True)
1492 write_collated_yaml_config(pyfhd_config, config_path)
1494 return pyfhd_config, logger