From 9adcbf671b19e55218a2a8692c474d3c6e05cc12 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 15 Oct 2024 17:48:52 -0700 Subject: [PATCH 001/375] started pyFAI geometry optimization lute model --- lute/io/models/__init__.py | 1 + lute/io/models/geom_opt.py | 16 ++++++++++++++++ lute/managed_tasks.py | 3 +++ 3 files changed, 20 insertions(+) create mode 100644 lute/io/models/geom_opt.py diff --git a/lute/io/models/__init__.py b/lute/io/models/__init__.py index c063ba40..ac405332 100644 --- a/lute/io/models/__init__.py +++ b/lute/io/models/__init__.py @@ -8,3 +8,4 @@ from .smd import * from .tests import * from .mpi_tests import * +from .geom_opt import * diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py new file mode 100644 index 00000000..9c550ea5 --- /dev/null +++ b/lute/io/models/geom_opt.py @@ -0,0 +1,16 @@ +import os +from pathlib import Path +from typing import Any, Dict, Literal, Optional, Union + +from pydantic import BaseModel, Field, PositiveInt, validator, root_validator + +from .base import TaskParameters, TemplateConfig + +class OptimizePyFAIGeomParameters(TaskParameters): + """Parameters for optimizing the geometry using PyFAI. + + blablabla + """ + + class Config(TaskParameters.Config): + pass \ No newline at end of file diff --git a/lute/managed_tasks.py b/lute/managed_tasks.py index e0ac6ca9..1fd949c4 100644 --- a/lute/managed_tasks.py +++ b/lute/managed_tasks.py @@ -58,6 +58,9 @@ SmallDataXESAnalyzer: MPIExecutor = MPIExecutor("AnalyzeSmallDataXES") """Process XES results from a Small Data HDF5 file.""" +PyFAIGeomOptimizer: MPIExecutor = MPIExecutor("OptimizePyFAIGeom") +"""Optimize the geometry using PyFAI.""" + # SFX ##### CCTBXIndexer: Executor = Executor("IndexCCTBXXFEL") From c4008341b9bc969954c55cfdd170a3987167c893 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 23 Oct 2024 14:07:53 -0700 Subject: [PATCH 002/375] [ENH] Implemented OptimizePyFAIGeom Task, ready for testing --- lute/io/models/geom_opt.py | 164 +++++++++++- lute/managed_tasks.py | 2 +- lute/tasks/__init__.py | 5 + lute/tasks/geom_opt.py | 528 +++++++++++++++++++++++++++++++++++++ 4 files changed, 691 insertions(+), 8 deletions(-) create mode 100644 lute/tasks/geom_opt.py diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 9c550ea5..6db67c5a 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -1,16 +1,166 @@ import os from pathlib import Path -from typing import Any, Dict, Literal, Optional, Union +from typing import Any, Dict, Literal, Optional, Union, Tuple -from pydantic import BaseModel, Field, PositiveInt, validator, root_validator +from pydantic import BaseModel, Field -from .base import TaskParameters, TemplateConfig +from .base import TaskParameters, validator +from ..db import read_latest_db_entry + +from PSCalib.CalibFileFinder import CalibFileFinder class OptimizePyFAIGeomParameters(TaskParameters): - """Parameters for optimizing the geometry using PyFAI. + """Parameters for optimizing detector geometry using PyFAI and Bayesian optimization. - blablabla + The Bayesian Optimization has default hyperparameters that can be overriden by the user. """ - class Config(TaskParameters.Config): - pass \ No newline at end of file + set_result: bool = True + """Whether the Executor should mark a specified parameter as a result.""" + + exp : str = Field( + "", + description="Experiment name.", + ) + + run : int = Field( + None, + description="Run number.", + ) + + det_type : str = Field( + "", + description="Detector type. Currently supported: 'ePix10k2M', 'ePix10kaQuad', 'Rayonix', 'Rayonix2', 'Jungfrau1M', 'Jungfrau4M'", + ) + + date : str = Field( + "", + description="Start date of analysis", + ) + + work_dir : str = Field( + "", + description="Main working directory for LUTE.", + ) + + in_file: str = Field( + "", + description="Path to the input .data file containing the detector geometry info to be calibrated.", + ) + + powder: str = Field( + "", + description="Powder diffraction pattern to be used for the calibration.", + ) + + calibrant: str = Field( + "", + description="Calibrant used for the calibration supported by pyFAI: https://github.com/silx-kit/pyFAI/tree/main/src/pyFAI/resources/calibration" + ) + + out_file: str = Field( + "", + description="Path to the output .data file containing the optimized detector geometry.", + is_result=True, + ) + + class BayesianOptParameters(BaseModel): + """Bayesian optimization hyperparameters.""" + + bounds: Dict[str, Tuple[float, float]] = Field( + { + "dist": (0.0, 0.0), + "poni1": (0.0, 0.0), + "poni2": (0.0, 0.0), + }, + description="Bounds defining the parameter search space for the Bayesian optimization.", + ) + + res: float = Field( + None, + description="Resolution of the grid used to discretize the parameter search space.", + ) + + n_samples: Optional[int] = Field( + 50, + description="Number of random starts to initialize the Bayesian optimization.", + ) + + n_iterations: Optional[int] = Field( + 50, + description="Number of iterations to run the Bayesian optimization.", + ) + + prior: Optional[bool] = Field( + True, + description="Whether to use a gaussian prior centered on the search space for the Bayesian optimization or randomly pick samples.", + ) + + af: Optional[str] = Field( + "ucb", + description="Acquisition function to be used by the Bayesian optimization.", + ) + + hyperparam : Optional[Dict[str, float]] = Field( + { + "beta": 1.96, + "epsilon": 0.01, + }, + description="Hyperparameters for the acquisition function.", + ) + + seed : Optional[int] = Field( + None, + description="Seed for the random number generator for potential reproducibility.", + ) + + @validator("exp", always=True) + def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: + if exp == "": + exp: str = values["lute_config"].experiment + return exp + + @validator("run", always=True) + def validate_run(cls, run: int, values: Dict[str, Any]) -> Union[str, int]: + if run is None: + run: Union[str, int] = values["lute_config"].run + return run + + @validator("date", always=True) + def validate_date(cls, date: str, values: Dict[str, Any]) -> str: + if date == "": + date: str = values["lute_config"].date + return date + + @validator("work_dir", always=True) + def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: + if work_dir == "": + work_dir: str = values["lute_config"].work_dir + return work_dir + + @validator("in_file", always=True) + def validate_in_file(cls, in_file: str) -> str: + if in_file == "": + exp = cls.exp + run = cls.run + det_type = cls.det_type + cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' + src = 'MfxEndstation.0:Epix10ka2M.0' + type = 'geometry' + cff = CalibFileFinder(cdir) + in_file = cff.findCalibFile(src, type, run) + return in_file + + @validator("powder", always=True) + def validate_powder(cls, powder: str, work_dir: str) -> str: + if powder == "": + powder: str = read_latest_db_entry( + f"{work_dir}/powder", "ComputePowder", "out_file" + ) + return powder + + @validator("out_file", always=True) + def validate_out_file(cls, out_file: str, run: Union[str, int], in_file: str) -> str: + if out_file == "": + out_file: str = in_file.replace("0-end.data", f"{run}-end.data") + return out_file \ No newline at end of file diff --git a/lute/managed_tasks.py b/lute/managed_tasks.py index 1fd949c4..0b97ca97 100644 --- a/lute/managed_tasks.py +++ b/lute/managed_tasks.py @@ -59,7 +59,7 @@ """Process XES results from a Small Data HDF5 file.""" PyFAIGeomOptimizer: MPIExecutor = MPIExecutor("OptimizePyFAIGeom") -"""Optimize the geometry using PyFAI.""" +"""Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" # SFX ##### diff --git a/lute/tasks/__init__.py b/lute/tasks/__init__.py index 386cbd48..3201b533 100644 --- a/lute/tasks/__init__.py +++ b/lute/tasks/__init__.py @@ -85,4 +85,9 @@ def import_task(task_name: str) -> Type[Task]: from .mpi_test import TestMultiNodeCommunication return TestMultiNodeCommunication + + if task_name == "OptimizePyFAIGeom": + from .geom_opt import OptimizePyFAIGeom + + return OptimizePyFAIGeom raise TaskNotFoundError diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py new file mode 100644 index 00000000..c4a19ce5 --- /dev/null +++ b/lute/tasks/geom_opt.py @@ -0,0 +1,528 @@ +""" +Classes for geometry optimization tasks. + +Classes: + OptimizePyFAIGeom: optimize detector geometry using PyFAI coupled with Bayesian Optimization + +""" + +__all__ = ["OptimizePyFAIGeom"] +__author__ = "Louis Conreux" + +import sys +from pathlib import Path +from typing import Any, Dict, List, Literal, TextIO, Tuple, Optional + +from lute.execution.ipc import Message +from lute.io.models.geom_opt import OptimizePyFAIGeomParameters +from lute.tasks.task import Task + +sys.path.append('/sdf/home/l/lconreux/LCLSGeom') +from LCLSGeom.swap_geom import PsanaToPyFAI, PyFAIToCrystFEL, CrystFELToPsana + +import numpy as np +import matplotlib.pyplot as plt +from pyFAI.geometry import Geometry +from pyFAI.goniometer import SingleGeometry +from pyFAI.azimuthalIntegrator import AzimuthalIntegrator +from pyFAI.calibrant import CALIBRANT_FACTORY +from mpi4py import MPI +from sklearn.gaussian_process import GaussianProcessRegressor +from sklearn.gaussian_process.kernels import RBF, ConstantKernel, WhiteKernel +from sklearn.utils._testing import ignore_warnings +from sklearn.exceptions import ConvergenceWarning +from scipy.stats import norm + +class OptimizePyFAIGeom(Task): + """Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" + + def __init__(self, *, params: OptimizePyFAIGeomParameters, use_mpi: bool = True) -> None: + super().__init__(params=params, use_mpi=use_mpi) + + def _run(self) -> None: + detector = self.build_pyFAI_detector() + optimizer = self.BayesGeomOpt( + exp=self.params.exp, + run=self.params.run, + det_type=self.params.det_type, + detector=detector, + calibrant=self.params.calibrant, + ) + optimizer.bayes_opt_geom( + powder=self.params.powder, + bounds=self.params.bounds, + res=self.params.res, + Imin=self.params.Imin, + n_samples=self.params.n_samples, + num_iterations=self.params.num_iterations, + af=self.params.af, + hyperparam=self.params.hyperparam, + prior=self.params.prior, + seed=self.params.seed, + ) + if optimizer.rank == 0: + detector = self.update_geometry(optimizer) + plot = f'{self.params.work_dir}/figs/bayes_opt/bayes_opt_geom_r{optimizer.run:04}.png' + optimizer.visualize_results( + powder=optimizer.powder_img, + bo_history=optimizer.bo_history, + detector=detector, + params=optimizer.params, + plot=plot, + ) + + def build_pyFAI_detector(self): + """ + Fetch the geometry data and build a pyFAI detector object. + """ + in_file = self.params.in_file + det_type = self.params.det_type + psana_to_pyfai = PsanaToPyFAI(in_file=in_file, det_type=det_type) + detector = psana_to_pyfai.detector + return detector + + def update_geometry(self, optimizer): + """ + Update the geometry and write a new .geom file and .data file + """ + PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self.params.in_file, out_file=self.params.out_file.replace("0-end.data", f"r{optimizer.run:04}.geom")) + CrystFELToPsana(in_file=self.params.out_file.replace("0-end.data", f"r{optimizer.run:04}.geom"), det_type=optimizer.det_type, out_file=self.params.out_file) + psana_to_pyfai = PsanaToPyFAI(in_file=self.in_file.replace("0-end.data", f"{optimizer.run}-end.data"), det_type=optimizer.det_type) + detector = psana_to_pyfai.detector + return detector + + class BayesGeomOpt: + """ + Class to perform Geometry Optimization using Bayesian Optimization on pyFAI + + Parameters + ---------- + exp : str + Experiment name + run : int + Run number + det_type : str + Detector type + detector : PyFAI(Detector) + PyFAI detector object + calibrant : str + Calibrant name + """ + + def __init__( + self, + exp, + run, + det_type, + detector, + calibrant, + ): + self.exp = exp + self.run = run + self.det_type = det_type.lower() + self.comm = MPI.COMM_WORLD + self.rank = self.comm.Get_rank() + self.size = self.comm.Get_size() + self.detector = detector + self.calibrant = calibrant + self.order = ["dist", "poni1", "poni2", "rot1", "rot2", "rot3"] + self.space = ["poni1", "poni2"] + self.values = {'dist': 0.1,'poni1':0, 'poni2':0, 'rot1':0, 'rot2':0, 'rot3':0} + + @staticmethod + def expected_improvement(X, gp_model, best_y, epsilon=0): + y_pred, y_std = gp_model.predict(X, return_std=True) + z = (y_pred - best_y + epsilon) / y_std + ei = y_pred - best_y * norm.cdf(z) + y_std * norm.pdf(z) + return ei + + @staticmethod + def upper_confidence_bound(X, gp_model, best_y=None, beta=1.96): + y_pred, y_std = gp_model.predict(X, return_std=True) + ucb = y_pred + beta * y_std + return ucb + + @staticmethod + def probability_of_improvement(X, gp_model, best_y, epsilon=0): + y_pred, y_std = gp_model.predict(X, return_std=True) + z = (y_pred - best_y + epsilon) / y_std + pi = norm.cdf(z) + return pi + + @staticmethod + def contextual_improvement(X, gp_model, best_y, hyperparam=None): + y_pred, y_std = gp_model.predict(X, return_std=True) + cv = np.mean(y_std**2) / best_y + z = (y_pred - best_y + cv) / y_std + ci = y_pred - best_y * norm.cdf(z) + y_std * norm.pdf(z) + return ci + + def build_calibrant(self, wavelength): + """ + Define calibrant for optimization + + Parameters + ---------- + wavelength : float + Wavelength of the experiment + """ + self.calibrant_name = self.calibrant + calibrant = CALIBRANT_FACTORY(self.calibrant) + photon_energy = 1.23984197386209e-09 / wavelength + self.photon_energy = photon_energy + calibrant.wavelength = wavelength + self.calibrant = calibrant + + def min_intensity(self, Imin): + """ + Define minimal intensity for control point extraction + Note: this is a heuristic that has been found to work well but may need some tuning. + + Parameters + ---------- + Imin : int or str + Minimum intensity to use for control point extraction based on photon energy or max intensity + """ + if type(Imin) == str: + if 'rayonix' not in self.det_type: + Imin = np.max(self.powder_img) * 0.01 + else: + self.powder_img = self.powder_img[self.powder_img > 1e3] + Imin = np.max(self.powder_img) * 0.01 + else: + Imin = Imin * self.photon_energy + self.Imin = Imin + + def distribute_scan(self, scan): + """ + Distribute the scan across all ranks. + + Parameters + ---------- + scan : list of distances + parameter dist for initial estimates + """ + return scan[self.rank] + + @ignore_warnings(category=ConvergenceWarning) + def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, num_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): + """ + Perform Bayesian Optimization on PONI center parameters, for a fixed distance + + Parameters + ---------- + powder_img : np.ndarray + Powder image + dist : float + Fixed distance + bounds : dict + Dictionary of bounds for each parameter + res : float + Resolution of the grid used to discretize the parameter search space + values : dict + Dictionary of values for fixed parameters + n_samples : int + Number of samples to initialize the GP model + num_iterations : int + Number of iterations for optimization + af : str + Acquisition function to use for optimization + hyperparam : dict + Dictionary of hyperparameters for the acquisition function + prior : bool + Use prior information for optimization + seed : int + Random seed for reproducibility + """ + if seed is not None: + np.random.seed(seed) + + self.values['dist'] = dist + inputs = {} + norm_inputs = {} + for p in self.order: + if p in self.space: + inputs[p] = np.arange(bounds[p][0], bounds[p][1]+bounds[p][2], bounds[p][2]) + norm_inputs[p] = inputs[p] + else: + inputs[p] = np.array([self.values[p]]) + X = np.array(np.meshgrid(*[inputs[p] for p in self.order])).T.reshape(-1, len(self.order)) + X_space = np.array(np.meshgrid(*[norm_inputs[p] for p in self.space])).T.reshape(-1, len(self.space)) + X_norm = (X_space - np.mean(X_space, axis=0)) / (np.max(X_space, axis=0) - np.min(X_space, axis=0)) + if prior: + means = np.mean(X_space, axis=0) + cov = np.diag([((bounds[param][1] - bounds[param][0]) / 5)**2 for param in self.space]) + X_samples = np.random.multivariate_normal(means, cov, n_samples) + X_norm_samples = (X_samples - np.mean(X_space, axis=0)) / (np.max(X_space, axis=0) - np.min(X_space, axis=0)) + for p in self.order: + if p not in self.space: + idx = self.order.index(p) + X_samples = np.insert(X_samples, idx, self.values[p], axis=1) + else: + idx_samples = np.random.choice(X.shape[0], n_samples) + X_samples = X[idx_samples] + X_norm_samples = X_norm[idx_samples] + + bo_history = {} + y = np.zeros((n_samples)) + for i in range(n_samples): + dist, poni1, poni2, rot1, rot2, rot3 = X_samples[i] + geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) + sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + y[i] = len(sg.geometry_refinement.data) + bo_history[f'init_sample_{i+1}'] = {'param':X_samples[i], 'score': y[i]} + + y_norm = (y - np.mean(y)) / np.std(y) + best_score = np.max(y_norm) + + kernel = RBF(length_scale=0.3, length_scale_bounds=(0.2, 0.4)) \ + * ConstantKernel(constant_value=1.0, constant_value_bounds=(0.5, 1.5)) \ + + WhiteKernel(noise_level=0.001, noise_level_bounds = 'fixed') + gp_model = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, random_state=42) + gp_model.fit(X_norm_samples, y_norm) + visited_idx = list([]) + + if af == "ucb": + if hyperparam is None: + hyperparam = {'beta': 1.96} + hyperparam = hyperparam['beta'] + af = self.upper_confidence_bound + elif af == "ei": + if hyperparam is None: + hyperparam = {'epsilon': 0} + hyperparam = hyperparam['epsilon'] + af = self.expected_improvement + elif af == "pi": + if hyperparam is None: + hyperparam = {'epsilon': 0} + hyperparam = hyperparam['epsilon'] + af = self.probability_of_improvement + elif af == "ci": + af = self.contextual_improvement + + for i in range(num_iterations): + # 1. Generate the Acquisition Function values using the Gaussian Process Regressor + af_values = af(X_norm, gp_model, best_score, hyperparam) + af_values[visited_idx] = -np.inf + + # 2. Select the next set of parameters based on the Acquisition Function + new_idx = np.argmax(af_values) + new_input = X[new_idx] + visited_idx.append(new_idx) + + # 3. Compute the score of the new set of parameters + dist, poni1, poni2, rot1, rot2, rot3 = new_input + geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) + sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + score = len(sg.geometry_refinement.data) + y = np.append(y, [score], axis=0) + ypred = gp_model.predict(X_norm, return_std=False) + bo_history[f'iteration_{i+1}'] = {'param':X[new_idx], 'score': score, 'pred': ypred, 'af': af_values} + X_samples = np.append(X_samples, [X[new_idx]], axis=0) + X_norm_samples = np.append(X_norm_samples, [X_norm[new_idx]], axis=0) + y_norm = (y - np.mean(y)) / np.std(y) + best_score = np.max(y_norm) + # 4. Update the Gaussian Process Regressor + gp_model.fit(X_norm_samples, y_norm) + + best_idx = np.argmax(y_norm) + best_param = X_samples[best_idx] + dist, poni1, poni2, rot1, rot2, rot3 = best_param + geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) + sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + self.sg = sg + residuals = sg.geometry_refinement.refine3(fix=['wavelength']) + params = sg.geometry_refinement.param + result = {'bo_history': bo_history, 'params': params, 'residuals': residuals, 'best_idx': best_idx} + return result + + def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, num_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): + """ + From guessed initial geometry, optimize the geometry using Bayesian Optimization on pyFAI package + + Parameters + ---------- + powder : str + Path to powder image to use for calibration + bounds : dict + Dictionary of bounds and resolution for search parameters + res : float + Resolution of the grid used to discretize the parameter search space + Imin : int or str + Minimum intensity to use for control point extraction based on photon energy or max intensity + values : dict + Dictionary of values for fixed parameters + n_samples : int + Number of samples to initialize the GP model + num_iterations : int + Number of iterations for optimization + af : str + Acquisition function to use for optimization + hyperparam : dict + Dictionary of hyperparameters for the acquisition function + prior : bool + Use prior information for optimization + seed : int + Random seed for reproducibility + """ + if seed is not None: + np.random.seed(seed) + + powder = np.load(powder) + + self.make_calibrant() + + self.minimal_intensity(Imin) + + if self.rank == 0: + distances = np.linspace(bounds['dist'][0], bounds['dist'][1], self.size) + else: + distances = None + + dist = self.comm.scatter(distances, root=0) + print(f"Rank {self.rank} is working on distance {dist}") + + results = self.bayes_opt_center(powder, dist, bounds, n_samples, num_iterations, af, hyperparam, prior, seed) + self.comm.Barrier() + + self.scan = {} + self.scan['bo_history'] = self.comm.gather(results['bo_history'], root=0) + self.scan['params'] = self.comm.gather(results['params'], root=0) + self.scan['residuals'] = self.comm.gather(results['residuals'], root=0) + self.scan['best_idx'] = self.comm.gather(results['best_idx'], root=0) + self.finalize() + + def finalize(self): + if self.rank == 0: + for key in self.scan.keys(): + self.scan[key] = np.array([item for item in self.scan[key]]) + index = np.argmin(self.scan['residuals']) + self.bo_history = self.scan['bo_history'][index] + self.params = self.scan['params'][index] + self.residuals = self.scan['residuals'][index] + self.best_idx = self.scan['best_idx'][index] + + def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): + """ + Display an image with the control points and the calibrated rings + + Parameters + ---------- + powder : np.ndarray + """ + if ax is None: + _fig, ax = plt.subplots() + if sg is not None: + if powder is None: + powder = sg.image + if cp is None: + cp = sg.control_points + if ai is None: + ai = sg.geometry_refinement + if label is None: + label = sg.label + ax.imshow(powder.T, + origin="lower", + cmap="viridis", + vmax=self.Imin) + ax.set_title(label) + if ai is not None and cp.calibrant is not None: + tth = cp.calibrant.get_2th() + ttha = ai.twoThetaArray() + ax.contour(ttha.T, levels=tth, cmap="autumn", linewidths=0.5, linestyles="dashed") + return ax + + def radial_integration(self, result, calibrant=None, label=None, ax=None): + """ + Display the powder diffraction pattern + + Parameters + ---------- + result : np.ndarray + Powder diffraction pattern + calibrant : Calibrant + Calibrant object + label : str + Name of the curve + ax : plt.Axes + Matplotlib axes + """ + from matplotlib import lines + + if ax is None: + _fig, ax = plt.subplots() + + try: + unit = result.unit + except: + unit = None + if len(result) == 3: + ax.errorbar(*result, label=label) + else: + ax.plot(*result, label=label) + + if label: + ax.legend() + if calibrant and unit: + x_values = calibrant.get_peaks(unit) + if x_values is not None: + for x in x_values: + line = lines.Line2D([x, x], ax.axis()[2:4], + color='red', linestyle='--', linewidth=0.5) + ax.add_line(line) + + ax.set_title("Radial Profile") + if unit: + ax.set_xlabel(unit.label) + ax.set_ylabel("Intensity") + + def visualize_results(self, powder, bo_history, detector, params, plot=''): + """ + Visualize fit, plotting (1) the BO convergence, (2) the radial profile and (3) the powder image. + + Parameters + ---------- + powder : np.ndarray + Powder image + bo_history : dict + Dictionary containing the history of optimization + detector : PyFAI(Detector) + PyFAI detector object + params : list + List of parameters for the best fit + plot : str + Path to save plot + """ + fig = plt.figure(figsize=(8,8),dpi=120) + nrow,ncol=2,2 + irow,icol=0,0 + + # Plotting BO convergence + ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) + scores = [bo_history[key]['score'] for key in bo_history.keys()] + ax1.plot(np.maximum.accumulate(scores)) + ax1.set_xticks(np.arange(len(scores), step=20)) + ax1.set_xlabel('Iteration') + ax1.set_ylabel('Best score so far') + ax1.set_title('Convergence Plot') + icol += 1 + + # Plotting radial profiles with peaks + ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol-icol) + ai = AzimuthalIntegrator(dist=params[0], detector=detector, wavelength=self.calibrant.wavelength) + res = ai.integrate1d(powder, 1000) + self.radial_integration(res, calibrant=self.calibrant, ax=ax2) + irow += 1 + + # Plotting stacked powder + geometry = Geometry(dist=params[0]) + sg = SingleGeometry(f'Max {self.calibrant_name}', powder, calibrant=self.calibrant, detector=detector, geometry=geometry) + sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + ax3 = plt.subplot2grid((nrow, ncol), (irow, 0), rowspan=nrow-irow, colspan=ncol) + self.display(sg=sg, ax=ax3) + + if plot != '': + fig.savefig(plot, dpi=300) From df14cf2ec1f5492cdb0cfde9991105e2844303b3 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 23 Oct 2024 16:07:46 -0700 Subject: [PATCH 003/375] [edit] Manually add wavelength as argument for further testing and debugging since no access to PsanaInterface --- lute/io/models/geom_opt.py | 5 +++++ lute/tasks/geom_opt.py | 13 +++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 6db67c5a..8e08e75d 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -58,6 +58,11 @@ class Config(TaskParameters.Config): description="Calibrant used for the calibration supported by pyFAI: https://github.com/silx-kit/pyFAI/tree/main/src/pyFAI/resources/calibration" ) + wavelength: float = Field( + 1e-10, + description="Wavelength of the X-ray beam in meters.", + ) + out_file: str = Field( "", description="Path to the output .data file containing the optimized detector geometry.", diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index c4a19ce5..49e2adad 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -47,6 +47,7 @@ def _run(self) -> None: det_type=self.params.det_type, detector=detector, calibrant=self.params.calibrant, + wavelength=self.params.wavelength, ) optimizer.bayes_opt_geom( powder=self.params.powder, @@ -62,7 +63,7 @@ def _run(self) -> None: ) if optimizer.rank == 0: detector = self.update_geometry(optimizer) - plot = f'{self.params.work_dir}/figs/bayes_opt/bayes_opt_geom_r{optimizer.run:04}.png' + plot = f'{self.params.work_dir}/figs/bayes_opt_geom_r{optimizer.run:04}.png' optimizer.visualize_results( powder=optimizer.powder_img, bo_history=optimizer.bo_history, @@ -107,6 +108,8 @@ class BayesGeomOpt: PyFAI detector object calibrant : str Calibrant name + wavelength : float + Wavelength of the experiment """ def __init__( @@ -116,6 +119,7 @@ def __init__( det_type, detector, calibrant, + wavelength, ): self.exp = exp self.run = run @@ -125,6 +129,7 @@ def __init__( self.size = self.comm.Get_size() self.detector = detector self.calibrant = calibrant + self.wavelength = wavelength self.order = ["dist", "poni1", "poni2", "rot1", "rot2", "rot3"] self.space = ["poni1", "poni2"] self.values = {'dist': 0.1,'poni1':0, 'poni2':0, 'rot1':0, 'rot2':0, 'rot3':0} @@ -157,7 +162,7 @@ def contextual_improvement(X, gp_model, best_y, hyperparam=None): ci = y_pred - best_y * norm.cdf(z) + y_std * norm.pdf(z) return ci - def build_calibrant(self, wavelength): + def build_calibrant(self): """ Define calibrant for optimization @@ -168,9 +173,9 @@ def build_calibrant(self, wavelength): """ self.calibrant_name = self.calibrant calibrant = CALIBRANT_FACTORY(self.calibrant) - photon_energy = 1.23984197386209e-09 / wavelength + photon_energy = 1.23984197386209e-09 / self.wavelength self.photon_energy = photon_energy - calibrant.wavelength = wavelength + calibrant.wavelength = self.wavelength self.calibrant = calibrant def min_intensity(self, Imin): From 73e32d491be4ab3552c73bf45584fd2cba1085f4 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 23 Oct 2024 17:40:23 -0700 Subject: [PATCH 004/375] [edit] Modfified model to match YAML configuration --- lute/io/models/geom_opt.py | 2 +- lute/tasks/geom_opt.py | 23 ++++++++--------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 8e08e75d..e69ab83b 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -69,7 +69,7 @@ class Config(TaskParameters.Config): is_result=True, ) - class BayesianOptParameters(BaseModel): + class BayesGeomOptParameters(BaseModel): """Bayesian optimization hyperparameters.""" bounds: Dict[str, Tuple[float, float]] = Field( diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 49e2adad..089c654d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -198,17 +198,6 @@ def min_intensity(self, Imin): Imin = Imin * self.photon_energy self.Imin = Imin - def distribute_scan(self, scan): - """ - Distribute the scan across all ranks. - - Parameters - ---------- - scan : list of distances - parameter dist for initial estimates - """ - return scan[self.rank] - @ignore_warnings(category=ConvergenceWarning) def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, num_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): """ @@ -243,11 +232,15 @@ def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, num_iter np.random.seed(seed) self.values['dist'] = dist + + if res is None: + res = self.detector.pixel_size + inputs = {} norm_inputs = {} for p in self.order: if p in self.space: - inputs[p] = np.arange(bounds[p][0], bounds[p][1]+bounds[p][2], bounds[p][2]) + inputs[p] = np.arange(bounds[p][0], bounds[p][1]+res, res) norm_inputs[p] = inputs[p] else: inputs[p] = np.array([self.values[p]]) @@ -378,9 +371,9 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, num_iter powder = np.load(powder) - self.make_calibrant() + self.build_calibrant() - self.minimal_intensity(Imin) + self.min_intensity(Imin) if self.rank == 0: distances = np.linspace(bounds['dist'][0], bounds['dist'][1], self.size) @@ -390,7 +383,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, num_iter dist = self.comm.scatter(distances, root=0) print(f"Rank {self.rank} is working on distance {dist}") - results = self.bayes_opt_center(powder, dist, bounds, n_samples, num_iterations, af, hyperparam, prior, seed) + results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, num_iterations, af, hyperparam, prior, seed) self.comm.Barrier() self.scan = {} From dbf6ed1470d19dac566aca8cd15a41f3ff9ffb8f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 12:44:30 -0700 Subject: [PATCH 005/375] [edit] Change name to OptimizePyFAIGeometry to match AgBh one + correct pydantic Parameter def logic --- lute/io/models/geom_opt.py | 105 +++-- lute/managed_tasks.py | 2 +- lute/tasks/__init__.py | 6 +- lute/tasks/geom_opt.py | 918 ++++++++++++++++++------------------- 4 files changed, 518 insertions(+), 513 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index e69ab83b..afe691ce 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -9,7 +9,57 @@ from PSCalib.CalibFileFinder import CalibFileFinder -class OptimizePyFAIGeomParameters(TaskParameters): +class BayesGeomOptParameters(BaseModel): + """Bayesian optimization hyperparameters.""" + + bounds: Dict[str, Tuple[float, float]] = Field( + { + "dist": (0.0, 0.0), + "poni1": (0.0, 0.0), + "poni2": (0.0, 0.0), + }, + description="Bounds defining the parameter search space for the Bayesian optimization.", + ) + + res: float = Field( + None, + description="Resolution of the grid used to discretize the parameter search space.", + ) + + n_samples: Optional[int] = Field( + 50, + description="Number of random starts to initialize the Bayesian optimization.", + ) + + n_iterations: Optional[int] = Field( + 50, + description="Number of iterations to run the Bayesian optimization.", + ) + + prior: Optional[bool] = Field( + True, + description="Whether to use a gaussian prior centered on the search space for the Bayesian optimization or randomly pick samples.", + ) + + af: Optional[str] = Field( + "ucb", + description="Acquisition function to be used by the Bayesian optimization.", + ) + + hyperparams : Optional[Dict[str, float]] = Field( + { + "beta": 1.96, + "epsilon": 0.01, + }, + description="Hyperparameters for the acquisition function.", + ) + + seed : Optional[int] = Field( + None, + description="Seed for the random number generator for potential reproducibility.", + ) + +class OptimizePyFAIGeometryParameters(TaskParameters): """Parameters for optimizing detector geometry using PyFAI and Bayesian optimization. The Bayesian Optimization has default hyperparameters that can be overriden by the user. @@ -69,55 +119,10 @@ class Config(TaskParameters.Config): is_result=True, ) - class BayesGeomOptParameters(BaseModel): - """Bayesian optimization hyperparameters.""" - - bounds: Dict[str, Tuple[float, float]] = Field( - { - "dist": (0.0, 0.0), - "poni1": (0.0, 0.0), - "poni2": (0.0, 0.0), - }, - description="Bounds defining the parameter search space for the Bayesian optimization.", - ) - - res: float = Field( - None, - description="Resolution of the grid used to discretize the parameter search space.", - ) - - n_samples: Optional[int] = Field( - 50, - description="Number of random starts to initialize the Bayesian optimization.", - ) - - n_iterations: Optional[int] = Field( - 50, - description="Number of iterations to run the Bayesian optimization.", - ) - - prior: Optional[bool] = Field( - True, - description="Whether to use a gaussian prior centered on the search space for the Bayesian optimization or randomly pick samples.", - ) - - af: Optional[str] = Field( - "ucb", - description="Acquisition function to be used by the Bayesian optimization.", - ) - - hyperparam : Optional[Dict[str, float]] = Field( - { - "beta": 1.96, - "epsilon": 0.01, - }, - description="Hyperparameters for the acquisition function.", - ) - - seed : Optional[int] = Field( - None, - description="Seed for the random number generator for potential reproducibility.", - ) + bo_params: BayesGeomOptParameters = Field( + BayesGeomOptParameters(), + description="Bayesian optimization parameters containing bounds and resolution for defining space search and hyperparameters.", + ) @validator("exp", always=True) def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: diff --git a/lute/managed_tasks.py b/lute/managed_tasks.py index 0b97ca97..bba87ebd 100644 --- a/lute/managed_tasks.py +++ b/lute/managed_tasks.py @@ -58,7 +58,7 @@ SmallDataXESAnalyzer: MPIExecutor = MPIExecutor("AnalyzeSmallDataXES") """Process XES results from a Small Data HDF5 file.""" -PyFAIGeomOptimizer: MPIExecutor = MPIExecutor("OptimizePyFAIGeom") +PyFAIGeometryOptimizer: MPIExecutor = MPIExecutor("OptimizePyFAIGeometry") """Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" # SFX diff --git a/lute/tasks/__init__.py b/lute/tasks/__init__.py index 3201b533..c7c7b030 100644 --- a/lute/tasks/__init__.py +++ b/lute/tasks/__init__.py @@ -86,8 +86,8 @@ def import_task(task_name: str) -> Type[Task]: return TestMultiNodeCommunication - if task_name == "OptimizePyFAIGeom": - from .geom_opt import OptimizePyFAIGeom + if task_name == "OptimizePyFAIGeometry": + from .geom_opt import OptimizePyFAIGeometry - return OptimizePyFAIGeom + return OptimizePyFAIGeometry raise TaskNotFoundError diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 089c654d..753f19bd 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -6,7 +6,7 @@ """ -__all__ = ["OptimizePyFAIGeom"] +__all__ = ["OptimizePyFAIGeometry"] __author__ = "Louis Conreux" import sys @@ -14,7 +14,7 @@ from typing import Any, Dict, List, Literal, TextIO, Tuple, Optional from lute.execution.ipc import Message -from lute.io.models.geom_opt import OptimizePyFAIGeomParameters +from lute.io.models.geom_opt import OptimizePyFAIGeometryParameters from lute.tasks.task import Task sys.path.append('/sdf/home/l/lconreux/LCLSGeom') @@ -33,37 +33,470 @@ from sklearn.exceptions import ConvergenceWarning from scipy.stats import norm -class OptimizePyFAIGeom(Task): +class BayesGeomOpt: + """ + Class to perform Geometry Optimization using Bayesian Optimization on pyFAI + + Parameters + ---------- + exp : str + Experiment name + run : int + Run number + det_type : str + Detector type + detector : PyFAI(Detector) + PyFAI detector object + calibrant : str + Calibrant name + wavelength : float + Wavelength of the experiment + """ + + def __init__( + self, + exp, + run, + det_type, + detector, + calibrant, + wavelength, + ): + self.exp = exp + self.run = run + self.det_type = det_type.lower() + self.comm = MPI.COMM_WORLD + self.rank = self.comm.Get_rank() + self.size = self.comm.Get_size() + self.detector = detector + self.calibrant = calibrant + self.wavelength = wavelength + self.order = ["dist", "poni1", "poni2", "rot1", "rot2", "rot3"] + self.space = ["poni1", "poni2"] + self.values = {'dist': 0.1,'poni1':0, 'poni2':0, 'rot1':0, 'rot2':0, 'rot3':0} + + @staticmethod + def expected_improvement(X, gp_model, best_y, epsilon=0): + y_pred, y_std = gp_model.predict(X, return_std=True) + z = (y_pred - best_y + epsilon) / y_std + ei = y_pred - best_y * norm.cdf(z) + y_std * norm.pdf(z) + return ei + + @staticmethod + def upper_confidence_bound(X, gp_model, best_y=None, beta=1.96): + y_pred, y_std = gp_model.predict(X, return_std=True) + ucb = y_pred + beta * y_std + return ucb + + @staticmethod + def probability_of_improvement(X, gp_model, best_y, epsilon=0): + y_pred, y_std = gp_model.predict(X, return_std=True) + z = (y_pred - best_y + epsilon) / y_std + pi = norm.cdf(z) + return pi + + @staticmethod + def contextual_improvement(X, gp_model, best_y, hyperparam=None): + y_pred, y_std = gp_model.predict(X, return_std=True) + cv = np.mean(y_std**2) / best_y + z = (y_pred - best_y + cv) / y_std + ci = y_pred - best_y * norm.cdf(z) + y_std * norm.pdf(z) + return ci + + def build_calibrant(self): + """ + Define calibrant for optimization + + Parameters + ---------- + wavelength : float + Wavelength of the experiment + """ + self.calibrant_name = self.calibrant + calibrant = CALIBRANT_FACTORY(self.calibrant) + photon_energy = 1.23984197386209e-09 / self.wavelength + self.photon_energy = photon_energy + calibrant.wavelength = self.wavelength + self.calibrant = calibrant + + def min_intensity(self, Imin): + """ + Define minimal intensity for control point extraction + Note: this is a heuristic that has been found to work well but may need some tuning. + + Parameters + ---------- + Imin : int or str + Minimum intensity to use for control point extraction based on photon energy or max intensity + """ + if type(Imin) == str: + if 'rayonix' not in self.det_type: + Imin = np.max(self.powder_img) * 0.01 + else: + self.powder_img = self.powder_img[self.powder_img > 1e3] + Imin = np.max(self.powder_img) * 0.01 + else: + Imin = Imin * self.photon_energy + self.Imin = Imin + + @ignore_warnings(category=ConvergenceWarning) + def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, num_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): + """ + Perform Bayesian Optimization on PONI center parameters, for a fixed distance + + Parameters + ---------- + powder_img : np.ndarray + Powder image + dist : float + Fixed distance + bounds : dict + Dictionary of bounds for each parameter + res : float + Resolution of the grid used to discretize the parameter search space + values : dict + Dictionary of values for fixed parameters + n_samples : int + Number of samples to initialize the GP model + num_iterations : int + Number of iterations for optimization + af : str + Acquisition function to use for optimization + hyperparam : dict + Dictionary of hyperparameters for the acquisition function + prior : bool + Use prior information for optimization + seed : int + Random seed for reproducibility + """ + if seed is not None: + np.random.seed(seed) + + self.values['dist'] = dist + + if res is None: + res = self.detector.pixel_size + + inputs = {} + norm_inputs = {} + for p in self.order: + if p in self.space: + inputs[p] = np.arange(bounds[p][0], bounds[p][1]+res, res) + norm_inputs[p] = inputs[p] + else: + inputs[p] = np.array([self.values[p]]) + X = np.array(np.meshgrid(*[inputs[p] for p in self.order])).T.reshape(-1, len(self.order)) + X_space = np.array(np.meshgrid(*[norm_inputs[p] for p in self.space])).T.reshape(-1, len(self.space)) + X_norm = (X_space - np.mean(X_space, axis=0)) / (np.max(X_space, axis=0) - np.min(X_space, axis=0)) + if prior: + means = np.mean(X_space, axis=0) + cov = np.diag([((bounds[param][1] - bounds[param][0]) / 5)**2 for param in self.space]) + X_samples = np.random.multivariate_normal(means, cov, n_samples) + X_norm_samples = (X_samples - np.mean(X_space, axis=0)) / (np.max(X_space, axis=0) - np.min(X_space, axis=0)) + for p in self.order: + if p not in self.space: + idx = self.order.index(p) + X_samples = np.insert(X_samples, idx, self.values[p], axis=1) + else: + idx_samples = np.random.choice(X.shape[0], n_samples) + X_samples = X[idx_samples] + X_norm_samples = X_norm[idx_samples] + + bo_history = {} + y = np.zeros((n_samples)) + for i in range(n_samples): + dist, poni1, poni2, rot1, rot2, rot3 = X_samples[i] + geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) + sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + y[i] = len(sg.geometry_refinement.data) + bo_history[f'init_sample_{i+1}'] = {'param':X_samples[i], 'score': y[i]} + + y_norm = (y - np.mean(y)) / np.std(y) + best_score = np.max(y_norm) + + kernel = RBF(length_scale=0.3, length_scale_bounds=(0.2, 0.4)) \ + * ConstantKernel(constant_value=1.0, constant_value_bounds=(0.5, 1.5)) \ + + WhiteKernel(noise_level=0.001, noise_level_bounds = 'fixed') + gp_model = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, random_state=42) + gp_model.fit(X_norm_samples, y_norm) + visited_idx = list([]) + + if af == "ucb": + if hyperparam is None: + hyperparam = {'beta': 1.96} + hyperparam = hyperparam['beta'] + af = self.upper_confidence_bound + elif af == "ei": + if hyperparam is None: + hyperparam = {'epsilon': 0} + hyperparam = hyperparam['epsilon'] + af = self.expected_improvement + elif af == "pi": + if hyperparam is None: + hyperparam = {'epsilon': 0} + hyperparam = hyperparam['epsilon'] + af = self.probability_of_improvement + elif af == "ci": + af = self.contextual_improvement + + for i in range(num_iterations): + # 1. Generate the Acquisition Function values using the Gaussian Process Regressor + af_values = af(X_norm, gp_model, best_score, hyperparam) + af_values[visited_idx] = -np.inf + + # 2. Select the next set of parameters based on the Acquisition Function + new_idx = np.argmax(af_values) + new_input = X[new_idx] + visited_idx.append(new_idx) + + # 3. Compute the score of the new set of parameters + dist, poni1, poni2, rot1, rot2, rot3 = new_input + geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) + sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + score = len(sg.geometry_refinement.data) + y = np.append(y, [score], axis=0) + ypred = gp_model.predict(X_norm, return_std=False) + bo_history[f'iteration_{i+1}'] = {'param':X[new_idx], 'score': score, 'pred': ypred, 'af': af_values} + X_samples = np.append(X_samples, [X[new_idx]], axis=0) + X_norm_samples = np.append(X_norm_samples, [X_norm[new_idx]], axis=0) + y_norm = (y - np.mean(y)) / np.std(y) + best_score = np.max(y_norm) + # 4. Update the Gaussian Process Regressor + gp_model.fit(X_norm_samples, y_norm) + + best_idx = np.argmax(y_norm) + best_param = X_samples[best_idx] + dist, poni1, poni2, rot1, rot2, rot3 = best_param + geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) + sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + self.sg = sg + residuals = sg.geometry_refinement.refine3(fix=['wavelength']) + params = sg.geometry_refinement.param + result = {'bo_history': bo_history, 'params': params, 'residuals': residuals, 'best_idx': best_idx} + return result + + def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, num_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): + """ + From guessed initial geometry, optimize the geometry using Bayesian Optimization on pyFAI package + + Parameters + ---------- + powder : str + Path to powder image to use for calibration + bounds : dict + Dictionary of bounds and resolution for search parameters + res : float + Resolution of the grid used to discretize the parameter search space + Imin : int or str + Minimum intensity to use for control point extraction based on photon energy or max intensity + values : dict + Dictionary of values for fixed parameters + n_samples : int + Number of samples to initialize the GP model + num_iterations : int + Number of iterations for optimization + af : str + Acquisition function to use for optimization + hyperparam : dict + Dictionary of hyperparameters for the acquisition function + prior : bool + Use prior information for optimization + seed : int + Random seed for reproducibility + """ + if seed is not None: + np.random.seed(seed) + + powder = np.load(powder) + + self.build_calibrant() + + self.min_intensity(Imin) + + if self.rank == 0: + distances = np.linspace(bounds['dist'][0], bounds['dist'][1], self.size) + else: + distances = None + + dist = self.comm.scatter(distances, root=0) + print(f"Rank {self.rank} is working on distance {dist}") + + results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, num_iterations, af, hyperparam, prior, seed) + self.comm.Barrier() + + self.scan = {} + self.scan['bo_history'] = self.comm.gather(results['bo_history'], root=0) + self.scan['params'] = self.comm.gather(results['params'], root=0) + self.scan['residuals'] = self.comm.gather(results['residuals'], root=0) + self.scan['best_idx'] = self.comm.gather(results['best_idx'], root=0) + self.finalize() + + def finalize(self): + if self.rank == 0: + for key in self.scan.keys(): + self.scan[key] = np.array([item for item in self.scan[key]]) + index = np.argmin(self.scan['residuals']) + self.bo_history = self.scan['bo_history'][index] + self.params = self.scan['params'][index] + self.residuals = self.scan['residuals'][index] + self.best_idx = self.scan['best_idx'][index] + + def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): + """ + Display an image with the control points and the calibrated rings + + Parameters + ---------- + powder : np.ndarray + """ + if ax is None: + _fig, ax = plt.subplots() + if sg is not None: + if powder is None: + powder = sg.image + if cp is None: + cp = sg.control_points + if ai is None: + ai = sg.geometry_refinement + if label is None: + label = sg.label + ax.imshow(powder.T, + origin="lower", + cmap="viridis", + vmax=self.Imin) + ax.set_title(label) + if ai is not None and cp.calibrant is not None: + tth = cp.calibrant.get_2th() + ttha = ai.twoThetaArray() + ax.contour(ttha.T, levels=tth, cmap="autumn", linewidths=0.5, linestyles="dashed") + return ax + + def radial_integration(self, result, calibrant=None, label=None, ax=None): + """ + Display the powder diffraction pattern + + Parameters + ---------- + result : np.ndarray + Powder diffraction pattern + calibrant : Calibrant + Calibrant object + label : str + Name of the curve + ax : plt.Axes + Matplotlib axes + """ + from matplotlib import lines + + if ax is None: + fig, ax = plt.subplots() + + try: + unit = result.unit + except: + unit = None + if len(result) == 3: + ax.errorbar(*result, label=label) + else: + ax.plot(*result, label=label) + + if label: + ax.legend() + if calibrant and unit: + x_values = calibrant.get_peaks(unit) + if x_values is not None: + for x in x_values: + line = lines.Line2D([x, x], ax.axis()[2:4], + color='red', linestyle='--', linewidth=0.5) + ax.add_line(line) + + ax.set_title("Radial Profile") + if unit: + ax.set_xlabel(unit.label) + ax.set_ylabel("Intensity") + + def visualize_results(self, powder, bo_history, detector, params, plot=''): + """ + Visualize fit, plotting (1) the BO convergence, (2) the radial profile and (3) the powder image. + + Parameters + ---------- + powder : np.ndarray + Powder image + bo_history : dict + Dictionary containing the history of optimization + detector : PyFAI(Detector) + PyFAI detector object + params : list + List of parameters for the best fit + plot : str + Path to save plot + """ + fig = plt.figure(figsize=(8,8),dpi=120) + nrow,ncol=2,2 + irow,icol=0,0 + + # Plotting BO convergence + ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) + scores = [bo_history[key]['score'] for key in bo_history.keys()] + ax1.plot(np.maximum.accumulate(scores)) + ax1.set_xticks(np.arange(len(scores), step=20)) + ax1.set_xlabel('Iteration') + ax1.set_ylabel('Best score so far') + ax1.set_title('Convergence Plot') + icol += 1 + + # Plotting radial profiles with peaks + ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol-icol) + ai = AzimuthalIntegrator(dist=params[0], detector=detector, wavelength=self.calibrant.wavelength) + res = ai.integrate1d(powder, 1000) + self.radial_integration(res, calibrant=self.calibrant, ax=ax2) + irow += 1 + + # Plotting stacked powder + geometry = Geometry(dist=params[0]) + sg = SingleGeometry(f'Max {self.calibrant_name}', powder, calibrant=self.calibrant, detector=detector, geometry=geometry) + sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + ax3 = plt.subplot2grid((nrow, ncol), (irow, 0), rowspan=nrow-irow, colspan=ncol) + self.display(sg=sg, ax=ax3) + + if plot != '': + fig.savefig(plot, dpi=300) + +class OptimizePyFAIGeometry(Task): """Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" - def __init__(self, *, params: OptimizePyFAIGeomParameters, use_mpi: bool = True) -> None: + def __init__(self, *, params: OptimizePyFAIGeometryParameters, use_mpi: bool = True) -> None: super().__init__(params=params, use_mpi=use_mpi) def _run(self) -> None: detector = self.build_pyFAI_detector() - optimizer = self.BayesGeomOpt( - exp=self.params.exp, - run=self.params.run, - det_type=self.params.det_type, + optimizer = BayesGeomOpt( + exp=self._task_parameters.exp, + run=self._task_parameters.run, + det_type=self._task_parameters.det_type, detector=detector, - calibrant=self.params.calibrant, - wavelength=self.params.wavelength, + calibrant=self._task_parameters.calibrant, + wavelength=self._task_parameters.wavelength, ) optimizer.bayes_opt_geom( - powder=self.params.powder, - bounds=self.params.bounds, - res=self.params.res, - Imin=self.params.Imin, - n_samples=self.params.n_samples, - num_iterations=self.params.num_iterations, - af=self.params.af, - hyperparam=self.params.hyperparam, - prior=self.params.prior, - seed=self.params.seed, + powder=self._task_parameters.powder, + bounds=self._task_parameters.bo_params.bounds, + res=self._task_parameters.bo_params.res, + Imin=self._task_parameters.bo_params.Imin, + n_samples=self._task_parameters.bo_params.n_samples, + num_iterations=self._task_parameters.bo_params.n_iterations, + af=self._task_parameters.bo_params.af, + hyperparam=self._task_parameters.bo_params.hyperparams, + prior=self._task_parameters.bo_params.prior, + seed=self._task_parameters.bo_params.seed, ) if optimizer.rank == 0: detector = self.update_geometry(optimizer) - plot = f'{self.params.work_dir}/figs/bayes_opt_geom_r{optimizer.run:04}.png' + plot = f'{self._task_parameters.work_dir}/figs/bayes_opt_geom_r{optimizer.run:04}.png' optimizer.visualize_results( powder=optimizer.powder_img, bo_history=optimizer.bo_history, @@ -76,8 +509,8 @@ def build_pyFAI_detector(self): """ Fetch the geometry data and build a pyFAI detector object. """ - in_file = self.params.in_file - det_type = self.params.det_type + in_file = self._task_parameters.in_file + det_type = self._task_parameters.det_type psana_to_pyfai = PsanaToPyFAI(in_file=in_file, det_type=det_type) detector = psana_to_pyfai.detector return detector @@ -86,441 +519,8 @@ def update_geometry(self, optimizer): """ Update the geometry and write a new .geom file and .data file """ - PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self.params.in_file, out_file=self.params.out_file.replace("0-end.data", f"r{optimizer.run:04}.geom")) - CrystFELToPsana(in_file=self.params.out_file.replace("0-end.data", f"r{optimizer.run:04}.geom"), det_type=optimizer.det_type, out_file=self.params.out_file) + PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self._task_parameters.in_file, out_file=self._task_parameters.out_file.replace("0-end.data", f"r{optimizer.run:04}.geom")) + CrystFELToPsana(in_file=self._task_parameters.out_file.replace("0-end.data", f"r{optimizer.run:04}.geom"), det_type=optimizer.det_type, out_file=self._task_parameters.out_file) psana_to_pyfai = PsanaToPyFAI(in_file=self.in_file.replace("0-end.data", f"{optimizer.run}-end.data"), det_type=optimizer.det_type) detector = psana_to_pyfai.detector - return detector - - class BayesGeomOpt: - """ - Class to perform Geometry Optimization using Bayesian Optimization on pyFAI - - Parameters - ---------- - exp : str - Experiment name - run : int - Run number - det_type : str - Detector type - detector : PyFAI(Detector) - PyFAI detector object - calibrant : str - Calibrant name - wavelength : float - Wavelength of the experiment - """ - - def __init__( - self, - exp, - run, - det_type, - detector, - calibrant, - wavelength, - ): - self.exp = exp - self.run = run - self.det_type = det_type.lower() - self.comm = MPI.COMM_WORLD - self.rank = self.comm.Get_rank() - self.size = self.comm.Get_size() - self.detector = detector - self.calibrant = calibrant - self.wavelength = wavelength - self.order = ["dist", "poni1", "poni2", "rot1", "rot2", "rot3"] - self.space = ["poni1", "poni2"] - self.values = {'dist': 0.1,'poni1':0, 'poni2':0, 'rot1':0, 'rot2':0, 'rot3':0} - - @staticmethod - def expected_improvement(X, gp_model, best_y, epsilon=0): - y_pred, y_std = gp_model.predict(X, return_std=True) - z = (y_pred - best_y + epsilon) / y_std - ei = y_pred - best_y * norm.cdf(z) + y_std * norm.pdf(z) - return ei - - @staticmethod - def upper_confidence_bound(X, gp_model, best_y=None, beta=1.96): - y_pred, y_std = gp_model.predict(X, return_std=True) - ucb = y_pred + beta * y_std - return ucb - - @staticmethod - def probability_of_improvement(X, gp_model, best_y, epsilon=0): - y_pred, y_std = gp_model.predict(X, return_std=True) - z = (y_pred - best_y + epsilon) / y_std - pi = norm.cdf(z) - return pi - - @staticmethod - def contextual_improvement(X, gp_model, best_y, hyperparam=None): - y_pred, y_std = gp_model.predict(X, return_std=True) - cv = np.mean(y_std**2) / best_y - z = (y_pred - best_y + cv) / y_std - ci = y_pred - best_y * norm.cdf(z) + y_std * norm.pdf(z) - return ci - - def build_calibrant(self): - """ - Define calibrant for optimization - - Parameters - ---------- - wavelength : float - Wavelength of the experiment - """ - self.calibrant_name = self.calibrant - calibrant = CALIBRANT_FACTORY(self.calibrant) - photon_energy = 1.23984197386209e-09 / self.wavelength - self.photon_energy = photon_energy - calibrant.wavelength = self.wavelength - self.calibrant = calibrant - - def min_intensity(self, Imin): - """ - Define minimal intensity for control point extraction - Note: this is a heuristic that has been found to work well but may need some tuning. - - Parameters - ---------- - Imin : int or str - Minimum intensity to use for control point extraction based on photon energy or max intensity - """ - if type(Imin) == str: - if 'rayonix' not in self.det_type: - Imin = np.max(self.powder_img) * 0.01 - else: - self.powder_img = self.powder_img[self.powder_img > 1e3] - Imin = np.max(self.powder_img) * 0.01 - else: - Imin = Imin * self.photon_energy - self.Imin = Imin - - @ignore_warnings(category=ConvergenceWarning) - def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, num_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): - """ - Perform Bayesian Optimization on PONI center parameters, for a fixed distance - - Parameters - ---------- - powder_img : np.ndarray - Powder image - dist : float - Fixed distance - bounds : dict - Dictionary of bounds for each parameter - res : float - Resolution of the grid used to discretize the parameter search space - values : dict - Dictionary of values for fixed parameters - n_samples : int - Number of samples to initialize the GP model - num_iterations : int - Number of iterations for optimization - af : str - Acquisition function to use for optimization - hyperparam : dict - Dictionary of hyperparameters for the acquisition function - prior : bool - Use prior information for optimization - seed : int - Random seed for reproducibility - """ - if seed is not None: - np.random.seed(seed) - - self.values['dist'] = dist - - if res is None: - res = self.detector.pixel_size - - inputs = {} - norm_inputs = {} - for p in self.order: - if p in self.space: - inputs[p] = np.arange(bounds[p][0], bounds[p][1]+res, res) - norm_inputs[p] = inputs[p] - else: - inputs[p] = np.array([self.values[p]]) - X = np.array(np.meshgrid(*[inputs[p] for p in self.order])).T.reshape(-1, len(self.order)) - X_space = np.array(np.meshgrid(*[norm_inputs[p] for p in self.space])).T.reshape(-1, len(self.space)) - X_norm = (X_space - np.mean(X_space, axis=0)) / (np.max(X_space, axis=0) - np.min(X_space, axis=0)) - if prior: - means = np.mean(X_space, axis=0) - cov = np.diag([((bounds[param][1] - bounds[param][0]) / 5)**2 for param in self.space]) - X_samples = np.random.multivariate_normal(means, cov, n_samples) - X_norm_samples = (X_samples - np.mean(X_space, axis=0)) / (np.max(X_space, axis=0) - np.min(X_space, axis=0)) - for p in self.order: - if p not in self.space: - idx = self.order.index(p) - X_samples = np.insert(X_samples, idx, self.values[p], axis=1) - else: - idx_samples = np.random.choice(X.shape[0], n_samples) - X_samples = X[idx_samples] - X_norm_samples = X_norm[idx_samples] - - bo_history = {} - y = np.zeros((n_samples)) - for i in range(n_samples): - dist, poni1, poni2, rot1, rot2, rot3 = X_samples[i] - geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) - sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) - sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) - y[i] = len(sg.geometry_refinement.data) - bo_history[f'init_sample_{i+1}'] = {'param':X_samples[i], 'score': y[i]} - - y_norm = (y - np.mean(y)) / np.std(y) - best_score = np.max(y_norm) - - kernel = RBF(length_scale=0.3, length_scale_bounds=(0.2, 0.4)) \ - * ConstantKernel(constant_value=1.0, constant_value_bounds=(0.5, 1.5)) \ - + WhiteKernel(noise_level=0.001, noise_level_bounds = 'fixed') - gp_model = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, random_state=42) - gp_model.fit(X_norm_samples, y_norm) - visited_idx = list([]) - - if af == "ucb": - if hyperparam is None: - hyperparam = {'beta': 1.96} - hyperparam = hyperparam['beta'] - af = self.upper_confidence_bound - elif af == "ei": - if hyperparam is None: - hyperparam = {'epsilon': 0} - hyperparam = hyperparam['epsilon'] - af = self.expected_improvement - elif af == "pi": - if hyperparam is None: - hyperparam = {'epsilon': 0} - hyperparam = hyperparam['epsilon'] - af = self.probability_of_improvement - elif af == "ci": - af = self.contextual_improvement - - for i in range(num_iterations): - # 1. Generate the Acquisition Function values using the Gaussian Process Regressor - af_values = af(X_norm, gp_model, best_score, hyperparam) - af_values[visited_idx] = -np.inf - - # 2. Select the next set of parameters based on the Acquisition Function - new_idx = np.argmax(af_values) - new_input = X[new_idx] - visited_idx.append(new_idx) - - # 3. Compute the score of the new set of parameters - dist, poni1, poni2, rot1, rot2, rot3 = new_input - geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) - sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) - sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) - score = len(sg.geometry_refinement.data) - y = np.append(y, [score], axis=0) - ypred = gp_model.predict(X_norm, return_std=False) - bo_history[f'iteration_{i+1}'] = {'param':X[new_idx], 'score': score, 'pred': ypred, 'af': af_values} - X_samples = np.append(X_samples, [X[new_idx]], axis=0) - X_norm_samples = np.append(X_norm_samples, [X_norm[new_idx]], axis=0) - y_norm = (y - np.mean(y)) / np.std(y) - best_score = np.max(y_norm) - # 4. Update the Gaussian Process Regressor - gp_model.fit(X_norm_samples, y_norm) - - best_idx = np.argmax(y_norm) - best_param = X_samples[best_idx] - dist, poni1, poni2, rot1, rot2, rot3 = best_param - geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) - sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) - sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) - self.sg = sg - residuals = sg.geometry_refinement.refine3(fix=['wavelength']) - params = sg.geometry_refinement.param - result = {'bo_history': bo_history, 'params': params, 'residuals': residuals, 'best_idx': best_idx} - return result - - def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, num_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): - """ - From guessed initial geometry, optimize the geometry using Bayesian Optimization on pyFAI package - - Parameters - ---------- - powder : str - Path to powder image to use for calibration - bounds : dict - Dictionary of bounds and resolution for search parameters - res : float - Resolution of the grid used to discretize the parameter search space - Imin : int or str - Minimum intensity to use for control point extraction based on photon energy or max intensity - values : dict - Dictionary of values for fixed parameters - n_samples : int - Number of samples to initialize the GP model - num_iterations : int - Number of iterations for optimization - af : str - Acquisition function to use for optimization - hyperparam : dict - Dictionary of hyperparameters for the acquisition function - prior : bool - Use prior information for optimization - seed : int - Random seed for reproducibility - """ - if seed is not None: - np.random.seed(seed) - - powder = np.load(powder) - - self.build_calibrant() - - self.min_intensity(Imin) - - if self.rank == 0: - distances = np.linspace(bounds['dist'][0], bounds['dist'][1], self.size) - else: - distances = None - - dist = self.comm.scatter(distances, root=0) - print(f"Rank {self.rank} is working on distance {dist}") - - results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, num_iterations, af, hyperparam, prior, seed) - self.comm.Barrier() - - self.scan = {} - self.scan['bo_history'] = self.comm.gather(results['bo_history'], root=0) - self.scan['params'] = self.comm.gather(results['params'], root=0) - self.scan['residuals'] = self.comm.gather(results['residuals'], root=0) - self.scan['best_idx'] = self.comm.gather(results['best_idx'], root=0) - self.finalize() - - def finalize(self): - if self.rank == 0: - for key in self.scan.keys(): - self.scan[key] = np.array([item for item in self.scan[key]]) - index = np.argmin(self.scan['residuals']) - self.bo_history = self.scan['bo_history'][index] - self.params = self.scan['params'][index] - self.residuals = self.scan['residuals'][index] - self.best_idx = self.scan['best_idx'][index] - - def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): - """ - Display an image with the control points and the calibrated rings - - Parameters - ---------- - powder : np.ndarray - """ - if ax is None: - _fig, ax = plt.subplots() - if sg is not None: - if powder is None: - powder = sg.image - if cp is None: - cp = sg.control_points - if ai is None: - ai = sg.geometry_refinement - if label is None: - label = sg.label - ax.imshow(powder.T, - origin="lower", - cmap="viridis", - vmax=self.Imin) - ax.set_title(label) - if ai is not None and cp.calibrant is not None: - tth = cp.calibrant.get_2th() - ttha = ai.twoThetaArray() - ax.contour(ttha.T, levels=tth, cmap="autumn", linewidths=0.5, linestyles="dashed") - return ax - - def radial_integration(self, result, calibrant=None, label=None, ax=None): - """ - Display the powder diffraction pattern - - Parameters - ---------- - result : np.ndarray - Powder diffraction pattern - calibrant : Calibrant - Calibrant object - label : str - Name of the curve - ax : plt.Axes - Matplotlib axes - """ - from matplotlib import lines - - if ax is None: - _fig, ax = plt.subplots() - - try: - unit = result.unit - except: - unit = None - if len(result) == 3: - ax.errorbar(*result, label=label) - else: - ax.plot(*result, label=label) - - if label: - ax.legend() - if calibrant and unit: - x_values = calibrant.get_peaks(unit) - if x_values is not None: - for x in x_values: - line = lines.Line2D([x, x], ax.axis()[2:4], - color='red', linestyle='--', linewidth=0.5) - ax.add_line(line) - - ax.set_title("Radial Profile") - if unit: - ax.set_xlabel(unit.label) - ax.set_ylabel("Intensity") - - def visualize_results(self, powder, bo_history, detector, params, plot=''): - """ - Visualize fit, plotting (1) the BO convergence, (2) the radial profile and (3) the powder image. - - Parameters - ---------- - powder : np.ndarray - Powder image - bo_history : dict - Dictionary containing the history of optimization - detector : PyFAI(Detector) - PyFAI detector object - params : list - List of parameters for the best fit - plot : str - Path to save plot - """ - fig = plt.figure(figsize=(8,8),dpi=120) - nrow,ncol=2,2 - irow,icol=0,0 - - # Plotting BO convergence - ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) - scores = [bo_history[key]['score'] for key in bo_history.keys()] - ax1.plot(np.maximum.accumulate(scores)) - ax1.set_xticks(np.arange(len(scores), step=20)) - ax1.set_xlabel('Iteration') - ax1.set_ylabel('Best score so far') - ax1.set_title('Convergence Plot') - icol += 1 - - # Plotting radial profiles with peaks - ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol-icol) - ai = AzimuthalIntegrator(dist=params[0], detector=detector, wavelength=self.calibrant.wavelength) - res = ai.integrate1d(powder, 1000) - self.radial_integration(res, calibrant=self.calibrant, ax=ax2) - irow += 1 - - # Plotting stacked powder - geometry = Geometry(dist=params[0]) - sg = SingleGeometry(f'Max {self.calibrant_name}', powder, calibrant=self.calibrant, detector=detector, geometry=geometry) - sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) - ax3 = plt.subplot2grid((nrow, ncol), (irow, 0), rowspan=nrow-irow, colspan=ncol) - self.display(sg=sg, ax=ax3) - - if plot != '': - fig.savefig(plot, dpi=300) + return detector \ No newline at end of file From 26f8b829c2fd67dcc1b9741476369fa4d1f994ce Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 12:47:01 -0700 Subject: [PATCH 006/375] [edit] Set up # PyFAI subpart in managed_tasks.py --- lute/managed_tasks.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lute/managed_tasks.py b/lute/managed_tasks.py index bba87ebd..36c07cfb 100644 --- a/lute/managed_tasks.py +++ b/lute/managed_tasks.py @@ -58,9 +58,6 @@ SmallDataXESAnalyzer: MPIExecutor = MPIExecutor("AnalyzeSmallDataXES") """Process XES results from a Small Data HDF5 file.""" -PyFAIGeometryOptimizer: MPIExecutor = MPIExecutor("OptimizePyFAIGeometry") -"""Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" - # SFX ##### CCTBXIndexer: Executor = Executor("IndexCCTBXXFEL") @@ -129,3 +126,8 @@ PeakFinderPsocake: Executor = Executor("FindPeaksPsocake") """Performs Bragg peak finding using psocake - *DEPRECATED*.""" + +# PyFAI +####### +PyFAIGeometryOptimizer: MPIExecutor = MPIExecutor("OptimizePyFAIGeometry") +"""Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" \ No newline at end of file From ec38b62f0a0b3fc44d41c369ff54584d2b5e2dcf Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 13:23:08 -0700 Subject: [PATCH 007/375] [edit] Fix logic in validator for powder, made a mistake --- lute/io/models/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index afe691ce..e1d7da47 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -162,10 +162,10 @@ def validate_in_file(cls, in_file: str) -> str: return in_file @validator("powder", always=True) - def validate_powder(cls, powder: str, work_dir: str) -> str: + def validate_powder(cls, powder: str, values: Dict[str, Any]) -> str: if powder == "": powder: str = read_latest_db_entry( - f"{work_dir}/powder", "ComputePowder", "out_file" + f"{values["lute_config"].work_dir}/powder", "ComputePowder", "out_file" ) return powder From eeb0a6a922fdeb60c53bb54e98307881ba8dc214 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 13:29:16 -0700 Subject: [PATCH 008/375] [edit] Fix " bug in f-string in validate_powder --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index e1d7da47..b700a00f 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -165,7 +165,7 @@ def validate_in_file(cls, in_file: str) -> str: def validate_powder(cls, powder: str, values: Dict[str, Any]) -> str: if powder == "": powder: str = read_latest_db_entry( - f"{values["lute_config"].work_dir}/powder", "ComputePowder", "out_file" + f"{values['lute_config'].work_dir}/powder", "ComputePowder", "out_file" ) return powder From 6cf8aa0744d57f87cb644ed3eb8abf9c7ce3236b Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 13:41:47 -0700 Subject: [PATCH 009/375] [edit] Fix validator logic for checking out_file and in_file --- lute/io/models/geom_opt.py | 14 ++++++++------ lute/tasks/geom_opt.py | 8 ++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index b700a00f..aa704132 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -149,11 +149,10 @@ def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: return work_dir @validator("in_file", always=True) - def validate_in_file(cls, in_file: str) -> str: + def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: if in_file == "": - exp = cls.exp - run = cls.run - det_type = cls.det_type + exp = values["exp"] + run = values["run"] cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' src = 'MfxEndstation.0:Epix10ka2M.0' type = 'geometry' @@ -164,13 +163,16 @@ def validate_in_file(cls, in_file: str) -> str: @validator("powder", always=True) def validate_powder(cls, powder: str, values: Dict[str, Any]) -> str: if powder == "": + work_dir = values["work_dir"] powder: str = read_latest_db_entry( - f"{values['lute_config'].work_dir}/powder", "ComputePowder", "out_file" + f"{work_dir}/powder", "ComputePowder", "out_file" ) return powder @validator("out_file", always=True) - def validate_out_file(cls, out_file: str, run: Union[str, int], in_file: str) -> str: + def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: if out_file == "": + in_file = values["in_file"] + run = values["run"] out_file: str = in_file.replace("0-end.data", f"{run}-end.data") return out_file \ No newline at end of file diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 753f19bd..f421e1e7 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -429,7 +429,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=''): bo_history : dict Dictionary containing the history of optimization detector : PyFAI(Detector) - PyFAI detector object + Corrected PyFAI detector object params : list List of parameters for the best fit plot : str @@ -519,8 +519,8 @@ def update_geometry(self, optimizer): """ Update the geometry and write a new .geom file and .data file """ - PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self._task_parameters.in_file, out_file=self._task_parameters.out_file.replace("0-end.data", f"r{optimizer.run:04}.geom")) - CrystFELToPsana(in_file=self._task_parameters.out_file.replace("0-end.data", f"r{optimizer.run:04}.geom"), det_type=optimizer.det_type, out_file=self._task_parameters.out_file) - psana_to_pyfai = PsanaToPyFAI(in_file=self.in_file.replace("0-end.data", f"{optimizer.run}-end.data"), det_type=optimizer.det_type) + PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self._task_parameters.in_file, out_file=self._task_parameters.in_file.replace("0-end.data", f"r{optimizer.run:04}.geom")) + CrystFELToPsana(in_file=self._task_parameters.in_file.replace("0-end.data", f"r{optimizer.run:04}.geom"), det_type=optimizer.det_type, out_file=self._task_parameters.out_file) + psana_to_pyfai = PsanaToPyFAI(in_file=self._task_parameters.out_file, det_type=self._task_parameters.det_type) detector = psana_to_pyfai.detector return detector \ No newline at end of file From 9d40d2b882b5ad7dcfb7b65d3fe3a546236f40ae Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 13:56:53 -0700 Subject: [PATCH 010/375] [edit] Forgot to include task parameter models in __all__ statement --- lute/io/models/geom_opt.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index aa704132..67b92606 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -1,3 +1,17 @@ +"""Models for optimizing detector geometry using PyFAI and Bayesian optimization. + +Classes: + - BayesGeomOptParameters(BaseModel): Bayesian optimization hyperparameters. + - OptimizePyFAIGeometryParameters(TaskParameters): + Parameters for optimizing detector geometry using PyFAI and Bayesian optimization. +""" + +__all__ = [ + "BayesGeomOptParameters", + "OptimizePyFAIGeometryParameters", +] +__author__ = "Louis Conreux" + import os from pathlib import Path from typing import Any, Dict, Literal, Optional, Union, Tuple From 4f62dda9c89f7c767cc24e45293dd3e70bdde0b0 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 14:16:31 -0700 Subject: [PATCH 011/375] [edit] Add prints + add final plot to result --- lute/tasks/geom_opt.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index f421e1e7..ad94ad03 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -9,14 +9,11 @@ __all__ = ["OptimizePyFAIGeometry"] __author__ = "Louis Conreux" -import sys -from pathlib import Path -from typing import Any, Dict, List, Literal, TextIO, Tuple, Optional - from lute.execution.ipc import Message -from lute.io.models.geom_opt import OptimizePyFAIGeometryParameters -from lute.tasks.task import Task +from lute.io.models.geom_opt import * +from lute.tasks.task import * +import sys sys.path.append('/sdf/home/l/lconreux/LCLSGeom') from LCLSGeom.swap_geom import PsanaToPyFAI, PyFAIToCrystFEL, CrystFELToPsana @@ -473,7 +470,13 @@ def __init__(self, *, params: OptimizePyFAIGeometryParameters, use_mpi: bool = T super().__init__(params=params, use_mpi=use_mpi) def _run(self) -> None: + msg = Message(contents="Starting PyFAI geometry optimization", signal="") + self._report_to_executor(msg) + msg = Message(contents="Building PyFAI detector", signal="") + self._report_to_executor(msg) detector = self.build_pyFAI_detector() + msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type}", signal="") + self._report_to_executor(msg) optimizer = BayesGeomOpt( exp=self._task_parameters.exp, run=self._task_parameters.run, @@ -482,6 +485,8 @@ def _run(self) -> None: calibrant=self._task_parameters.calibrant, wavelength=self._task_parameters.wavelength, ) + msg = Message(contents="Running Bayesian Optimization", signal="") + self._report_to_executor(msg) optimizer.bayes_opt_geom( powder=self._task_parameters.powder, bounds=self._task_parameters.bo_params.bounds, @@ -495,6 +500,16 @@ def _run(self) -> None: seed=self._task_parameters.bo_params.seed, ) if optimizer.rank == 0: + msg = Message(contents="Optimization complete", signal="") + self._report_to_executor(msg) + msg = Message(contents=f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") + self._report_to_executor(msg) + msg = Message(contents=f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})", signal="") + self._report_to_executor(msg) + msg = Message(contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", signal="") + self._report_to_executor(msg) + msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") + self._report_to_executor(msg) detector = self.update_geometry(optimizer) plot = f'{self._task_parameters.work_dir}/figs/bayes_opt_geom_r{optimizer.run:04}.png' optimizer.visualize_results( @@ -504,6 +519,8 @@ def _run(self) -> None: params=optimizer.params, plot=plot, ) + self._result.payload = plot + self._result.task_status = TaskStatus.COMPLETED def build_pyFAI_detector(self): """ From 6639e54211f3362539000c899dcdbfaf98d64cde Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 14:53:35 -0700 Subject: [PATCH 012/375] [edit] Fix BO_param model --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 67b92606..d519be51 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -134,7 +134,7 @@ class Config(TaskParameters.Config): ) bo_params: BayesGeomOptParameters = Field( - BayesGeomOptParameters(), + None, description="Bayesian optimization parameters containing bounds and resolution for defining space search and hyperparameters.", ) From f09a6404dd46d7dacb1caa39313bf880e84169ed Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 15:23:06 -0700 Subject: [PATCH 013/375] [edit] Try fixing bad parsing of config file --- lute/io/models/geom_opt.py | 108 ++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index d519be51..bcdce4e5 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -14,7 +14,7 @@ import os from pathlib import Path -from typing import Any, Dict, Literal, Optional, Union, Tuple +from typing import Any, Dict, Optional, Union, Tuple from pydantic import BaseModel, Field @@ -23,56 +23,6 @@ from PSCalib.CalibFileFinder import CalibFileFinder -class BayesGeomOptParameters(BaseModel): - """Bayesian optimization hyperparameters.""" - - bounds: Dict[str, Tuple[float, float]] = Field( - { - "dist": (0.0, 0.0), - "poni1": (0.0, 0.0), - "poni2": (0.0, 0.0), - }, - description="Bounds defining the parameter search space for the Bayesian optimization.", - ) - - res: float = Field( - None, - description="Resolution of the grid used to discretize the parameter search space.", - ) - - n_samples: Optional[int] = Field( - 50, - description="Number of random starts to initialize the Bayesian optimization.", - ) - - n_iterations: Optional[int] = Field( - 50, - description="Number of iterations to run the Bayesian optimization.", - ) - - prior: Optional[bool] = Field( - True, - description="Whether to use a gaussian prior centered on the search space for the Bayesian optimization or randomly pick samples.", - ) - - af: Optional[str] = Field( - "ucb", - description="Acquisition function to be used by the Bayesian optimization.", - ) - - hyperparams : Optional[Dict[str, float]] = Field( - { - "beta": 1.96, - "epsilon": 0.01, - }, - description="Hyperparameters for the acquisition function.", - ) - - seed : Optional[int] = Field( - None, - description="Seed for the random number generator for potential reproducibility.", - ) - class OptimizePyFAIGeometryParameters(TaskParameters): """Parameters for optimizing detector geometry using PyFAI and Bayesian optimization. @@ -82,12 +32,62 @@ class Config(TaskParameters.Config): set_result: bool = True """Whether the Executor should mark a specified parameter as a result.""" + class BayesGeomOptParameters(BaseModel): + """Bayesian optimization hyperparameters.""" + + bounds: Dict[str, Tuple[float, float]] = Field( + { + "dist": (0.05, 0.5), + "poni1": (-0.01, 0.01), + "poni2": (-0.01, 0.01), + }, + description="Bounds defining the parameter search space for the Bayesian optimization.", + ) + + res: float = Field( + None, + description="Resolution of the grid used to discretize the parameter search space.", + ) + + n_samples: Optional[int] = Field( + 50, + description="Number of random starts to initialize the Bayesian optimization.", + ) + + n_iterations: Optional[int] = Field( + 50, + description="Number of iterations to run the Bayesian optimization.", + ) + + prior: Optional[bool] = Field( + True, + description="Whether to use a gaussian prior centered on the search space for the Bayesian optimization or randomly pick samples.", + ) + + af: Optional[str] = Field( + "ucb", + description="Acquisition function to be used by the Bayesian optimization.", + ) + + hyperparams : Optional[Dict[str, float]] = Field( + { + "beta": 1.96, + "epsilon": 0.01, + }, + description="Hyperparameters for the acquisition function.", + ) + + seed : Optional[int] = Field( + None, + description="Seed for the random number generator for potential reproducibility.", + ) + exp : str = Field( "", description="Experiment name.", ) - run : int = Field( + run : Union[str, int] = Field( None, description="Run number.", ) @@ -134,7 +134,7 @@ class Config(TaskParameters.Config): ) bo_params: BayesGeomOptParameters = Field( - None, + BayesGeomOptParameters(), description="Bayesian optimization parameters containing bounds and resolution for defining space search and hyperparameters.", ) @@ -145,7 +145,7 @@ def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: return exp @validator("run", always=True) - def validate_run(cls, run: int, values: Dict[str, Any]) -> Union[str, int]: + def validate_run(cls, run: Union[str, int], values: Dict[str, Any]) -> Union[str, int]: if run is None: run: Union[str, int] = values["lute_config"].run return run From 0c7ac35bedb518f1b34d0efaa5e9db5392542c87 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 15:26:47 -0700 Subject: [PATCH 014/375] [edit] Try fixing bad parsing of config file --- lute/io/models/geom_opt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index bcdce4e5..b62633bd 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -7,7 +7,6 @@ """ __all__ = [ - "BayesGeomOptParameters", "OptimizePyFAIGeometryParameters", ] __author__ = "Louis Conreux" From 29ec9a50ebc19358458cb54bd68f82415807e1d9 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 24 Oct 2024 15:44:32 -0700 Subject: [PATCH 015/375] [edit] Minor change in __all__ --- lute/io/models/geom_opt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index b62633bd..6c10fbdf 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -6,9 +6,7 @@ Parameters for optimizing detector geometry using PyFAI and Bayesian optimization. """ -__all__ = [ - "OptimizePyFAIGeometryParameters", -] +__all__ = ["OptimizePyFAIGeometryParameters"] __author__ = "Louis Conreux" import os From a32e3251614e05c9ac08faae13ed79eb70cbd7be Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 28 Oct 2024 14:06:09 -0700 Subject: [PATCH 016/375] [edit] Change description and structure in classes definition --- lute/io/models/geom_opt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 6c10fbdf..68013931 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -1,7 +1,6 @@ """Models for optimizing detector geometry using PyFAI and Bayesian optimization. Classes: - - BayesGeomOptParameters(BaseModel): Bayesian optimization hyperparameters. - OptimizePyFAIGeometryParameters(TaskParameters): Parameters for optimizing detector geometry using PyFAI and Bayesian optimization. """ From 75e7ac47c8ba9ea9bbf82f5189a0e3d722d2625d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 29 Oct 2024 11:49:40 -0700 Subject: [PATCH 017/375] [edit] Fix omission of Imin parameter for BO parameters --- lute/io/models/geom_opt.py | 5 +++++ lute/tasks/geom_opt.py | 14 +++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 68013931..01c678da 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -55,6 +55,11 @@ class BayesGeomOptParameters(BaseModel): description="Number of iterations to run the Bayesian optimization.", ) + Imin: Optional[Union[str, int]] = Field( + 'max', + description="Minimal Intensity for extracting key control points, either based on heuristics on the maximal intensity, or a multiple of the photon energy" + ) + prior: Optional[bool] = Field( True, description="Whether to use a gaussian prior centered on the search space for the Bayesian optimization or randomly pick samples.", diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index ad94ad03..c88b70e9 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -137,7 +137,7 @@ def min_intensity(self, Imin): self.Imin = Imin @ignore_warnings(category=ConvergenceWarning) - def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, num_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): + def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, n_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): """ Perform Bayesian Optimization on PONI center parameters, for a fixed distance @@ -155,7 +155,7 @@ def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, num_iter Dictionary of values for fixed parameters n_samples : int Number of samples to initialize the GP model - num_iterations : int + n_iterations : int Number of iterations for optimization af : str Acquisition function to use for optimization @@ -237,7 +237,7 @@ def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, num_iter elif af == "ci": af = self.contextual_improvement - for i in range(num_iterations): + for i in range(n_iterations): # 1. Generate the Acquisition Function values using the Gaussian Process Regressor af_values = af(X_norm, gp_model, best_score, hyperparam) af_values[visited_idx] = -np.inf @@ -275,7 +275,7 @@ def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, num_iter result = {'bo_history': bo_history, 'params': params, 'residuals': residuals, 'best_idx': best_idx} return result - def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, num_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): + def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): """ From guessed initial geometry, optimize the geometry using Bayesian Optimization on pyFAI package @@ -293,7 +293,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, num_iter Dictionary of values for fixed parameters n_samples : int Number of samples to initialize the GP model - num_iterations : int + n_iterations : int Number of iterations for optimization af : str Acquisition function to use for optimization @@ -321,7 +321,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, num_iter dist = self.comm.scatter(distances, root=0) print(f"Rank {self.rank} is working on distance {dist}") - results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, num_iterations, af, hyperparam, prior, seed) + results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, n_iterations, af, hyperparam, prior, seed) self.comm.Barrier() self.scan = {} @@ -493,7 +493,7 @@ def _run(self) -> None: res=self._task_parameters.bo_params.res, Imin=self._task_parameters.bo_params.Imin, n_samples=self._task_parameters.bo_params.n_samples, - num_iterations=self._task_parameters.bo_params.n_iterations, + n_iterations=self._task_parameters.bo_params.n_iterations, af=self._task_parameters.bo_params.af, hyperparam=self._task_parameters.bo_params.hyperparams, prior=self._task_parameters.bo_params.prior, From 383439dee2a93d9026d8f95866d6f69f35df2141 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 29 Oct 2024 11:59:25 -0700 Subject: [PATCH 018/375] [debug] Adapt code logic for lute implementation --- lute/tasks/geom_opt.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index c88b70e9..856032fd 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -116,7 +116,7 @@ def build_calibrant(self): calibrant.wavelength = self.wavelength self.calibrant = calibrant - def min_intensity(self, Imin): + def min_intensity(self, Imin, powder): """ Define minimal intensity for control point extraction Note: this is a heuristic that has been found to work well but may need some tuning. @@ -127,23 +127,25 @@ def min_intensity(self, Imin): Minimum intensity to use for control point extraction based on photon energy or max intensity """ if type(Imin) == str: - if 'rayonix' not in self.det_type: - Imin = np.max(self.powder_img) * 0.01 + if 'rayonix' in self.det_type: + powder = powder[powder > 1e3] + Imin = np.max(powder) * 0.01 else: - self.powder_img = self.powder_img[self.powder_img > 1e3] - Imin = np.max(self.powder_img) * 0.01 + Imin = np.max(powder) * 0.01 else: Imin = Imin * self.photon_energy self.Imin = Imin + self.powder = powder + return powder @ignore_warnings(category=ConvergenceWarning) - def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, n_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): + def bayes_opt_center(self, powder, dist, bounds, res, n_samples=50, n_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): """ Perform Bayesian Optimization on PONI center parameters, for a fixed distance Parameters ---------- - powder_img : np.ndarray + powder : np.ndarray Powder image dist : float Fixed distance @@ -204,7 +206,7 @@ def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, n_iterat for i in range(n_samples): dist, poni1, poni2, rot1, rot2, rot3 = X_samples[i] geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) - sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + sg = SingleGeometry("extract_cp", powder, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) y[i] = len(sg.geometry_refinement.data) bo_history[f'init_sample_{i+1}'] = {'param':X_samples[i], 'score': y[i]} @@ -250,7 +252,7 @@ def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, n_iterat # 3. Compute the score of the new set of parameters dist, poni1, poni2, rot1, rot2, rot3 = new_input geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) - sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + sg = SingleGeometry("extract_cp", powder, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) score = len(sg.geometry_refinement.data) y = np.append(y, [score], axis=0) @@ -267,7 +269,7 @@ def bayes_opt_center(self, powder_img, dist, bounds, res, n_samples=50, n_iterat best_param = X_samples[best_idx] dist, poni1, poni2, rot1, rot2, rot3 = best_param geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) - sg = SingleGeometry("extract_cp", powder_img, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + sg = SingleGeometry("extract_cp", powder, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) self.sg = sg residuals = sg.geometry_refinement.refine3(fix=['wavelength']) @@ -311,7 +313,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat self.build_calibrant() - self.min_intensity(Imin) + powder = self.min_intensity(Imin, powder) if self.rank == 0: distances = np.linspace(bounds['dist'][0], bounds['dist'][1], self.size) @@ -513,7 +515,7 @@ def _run(self) -> None: detector = self.update_geometry(optimizer) plot = f'{self._task_parameters.work_dir}/figs/bayes_opt_geom_r{optimizer.run:04}.png' optimizer.visualize_results( - powder=optimizer.powder_img, + powder=optimizer.powder, bo_history=optimizer.bo_history, detector=detector, params=optimizer.params, From 8651de0491346619cd0c2167a67153ff8d82c0ad Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 29 Oct 2024 13:07:56 -0700 Subject: [PATCH 019/375] [debug] Fix update_geometry scripts --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 856032fd..79fffe36 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -538,8 +538,8 @@ def update_geometry(self, optimizer): """ Update the geometry and write a new .geom file and .data file """ - PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self._task_parameters.in_file, out_file=self._task_parameters.in_file.replace("0-end.data", f"r{optimizer.run:04}.geom")) - CrystFELToPsana(in_file=self._task_parameters.in_file.replace("0-end.data", f"r{optimizer.run:04}.geom"), det_type=optimizer.det_type, out_file=self._task_parameters.out_file) + PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self._task_parameters.in_file, out_file=self._task_parameters.in_file.replace("0-end.data", f"r{self._task_parameters.run:04}.geom")) + CrystFELToPsana(in_file=self._task_parameters.in_file.replace("0-end.data", f"r{self._task_parameters.run:04}.geom"), det_type=optimizer.det_type, out_file=self._task_parameters.out_file) psana_to_pyfai = PsanaToPyFAI(in_file=self._task_parameters.out_file, det_type=self._task_parameters.det_type) detector = psana_to_pyfai.detector return detector \ No newline at end of file From 2acab380c73eb58a55bf40820ce65a167a3f218b Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 29 Oct 2024 13:39:26 -0700 Subject: [PATCH 020/375] [debug] Fix update_geometry how to change geom name based on run --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 79fffe36..f497dae0 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -538,8 +538,8 @@ def update_geometry(self, optimizer): """ Update the geometry and write a new .geom file and .data file """ - PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self._task_parameters.in_file, out_file=self._task_parameters.in_file.replace("0-end.data", f"r{self._task_parameters.run:04}.geom")) - CrystFELToPsana(in_file=self._task_parameters.in_file.replace("0-end.data", f"r{self._task_parameters.run:04}.geom"), det_type=optimizer.det_type, out_file=self._task_parameters.out_file) + PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self._task_parameters.in_file, out_file=self._task_parameters.in_file.replace("0-end.data", f"r{self._task_parameters.run:0>4}.geom")) + CrystFELToPsana(in_file=self._task_parameters.in_file.replace("0-end.data", f"r{self._task_parameters.run:0>4}.geom"), det_type=optimizer.det_type, out_file=self._task_parameters.out_file) psana_to_pyfai = PsanaToPyFAI(in_file=self._task_parameters.out_file, det_type=self._task_parameters.det_type) detector = psana_to_pyfai.detector return detector \ No newline at end of file From f0ed5359bf81b096e475fbe2d0420b96a4311e12 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 29 Oct 2024 14:20:29 -0700 Subject: [PATCH 021/375] [debug] Fix plots how to change geom name based on run in the same way --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index f497dae0..5c41a45b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -513,7 +513,7 @@ def _run(self) -> None: msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") self._report_to_executor(msg) detector = self.update_geometry(optimizer) - plot = f'{self._task_parameters.work_dir}/figs/bayes_opt_geom_r{optimizer.run:04}.png' + plot = f'{self._task_parameters.work_dir}/figs/bayes_opt_geom_r{optimizer.run:0>4}.png' optimizer.visualize_results( powder=optimizer.powder, bo_history=optimizer.bo_history, From 1332dbb578ecfe838513695e4331239407f21319 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 11:54:45 -0700 Subject: [PATCH 022/375] [edit] Add rank printing to MPI debug --- lute/tasks/geom_opt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 5c41a45b..0eef939c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -477,7 +477,8 @@ def _run(self) -> None: msg = Message(contents="Building PyFAI detector", signal="") self._report_to_executor(msg) detector = self.build_pyFAI_detector() - msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type}", signal="") + rank = MPI.COMM_WORLD.Get_rank() + msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}", signal="") self._report_to_executor(msg) optimizer = BayesGeomOpt( exp=self._task_parameters.exp, @@ -513,7 +514,7 @@ def _run(self) -> None: msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") self._report_to_executor(msg) detector = self.update_geometry(optimizer) - plot = f'{self._task_parameters.work_dir}/figs/bayes_opt_geom_r{optimizer.run:0>4}.png' + plot = f'{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png' optimizer.visualize_results( powder=optimizer.powder, bo_history=optimizer.bo_history, From 66e551a24db72134972b6111bd0621fe0b432c93 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 11:59:16 -0700 Subject: [PATCH 023/375] [edit] Omit pyFAI INFO related messages for clarity --- lute/execution/logging.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lute/execution/logging.py b/lute/execution/logging.py index 21e850aa..2ad5da5b 100644 --- a/lute/execution/logging.py +++ b/lute/execution/logging.py @@ -56,6 +56,10 @@ def get_logger(name: str) -> logging.Logger: other_logger, logging.PlaceHolder ): other_logger.disabled = True + elif "pyFAI" in other_name and not isinstance( + other_logger, logging.PlaceHolder + ): + other_logger.disabled = True elif "numpy" in other_name and not isinstance( other_logger, logging.PlaceHolder ): From 03564c0d4c21b25ca38e9a73e92e1a51fc845332 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 12:08:19 -0700 Subject: [PATCH 024/375] [edit] Omit pyFAI INFO related messages for clarity --- lute/execution/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/execution/logging.py b/lute/execution/logging.py index 2ad5da5b..45959fdc 100644 --- a/lute/execution/logging.py +++ b/lute/execution/logging.py @@ -59,7 +59,7 @@ def get_logger(name: str) -> logging.Logger: elif "pyFAI" in other_name and not isinstance( other_logger, logging.PlaceHolder ): - other_logger.disabled = True + other_logger.setLevel(logging.CRITICAL) elif "numpy" in other_name and not isinstance( other_logger, logging.PlaceHolder ): From a714fe609626943d64e492c5de26d0709f5988b9 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 12:20:18 -0700 Subject: [PATCH 025/375] [edit] Omit pyFAI INFO related messages for clarity --- lute/tasks/geom_opt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0eef939c..ed8426c3 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -12,6 +12,10 @@ from lute.execution.ipc import Message from lute.io.models.geom_opt import * from lute.tasks.task import * +from lute.execution.logging import get_logger + +import logging +logger: logging.Logger = get_logger(__name__, is_task=True) import sys sys.path.append('/sdf/home/l/lconreux/LCLSGeom') From ae52b6c8b84bf1ebaaf5b4eb54deb536df77f3b2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 12:22:16 -0700 Subject: [PATCH 026/375] [edit] Omit pyFAI INFO related messages for clarity --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index ed8426c3..2dbc33d2 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -15,7 +15,7 @@ from lute.execution.logging import get_logger import logging -logger: logging.Logger = get_logger(__name__, is_task=True) +logger: logging.Logger = get_logger(__name__) import sys sys.path.append('/sdf/home/l/lconreux/LCLSGeom') From 9957e9311c7aaa8a72ef953899ce8844ca3ce7aa Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 12:25:09 -0700 Subject: [PATCH 027/375] [edit] Omit pyFAI INFO related messages for clarity --- lute/execution/logging.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lute/execution/logging.py b/lute/execution/logging.py index 45959fdc..37f453ac 100644 --- a/lute/execution/logging.py +++ b/lute/execution/logging.py @@ -59,7 +59,11 @@ def get_logger(name: str) -> logging.Logger: elif "pyFAI" in other_name and not isinstance( other_logger, logging.PlaceHolder ): - other_logger.setLevel(logging.CRITICAL) + other_logger.disabled = True + other_logger.propagate = False + + for handler in other_logger.handlers: + handler.disabled = True elif "numpy" in other_name and not isinstance( other_logger, logging.PlaceHolder ): From 5859d7c27052561536a2305ffa811352709d67c1 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 12:28:56 -0700 Subject: [PATCH 028/375] [edit] Omit pyFAI INFO related messages for clarity --- lute/execution/logging.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lute/execution/logging.py b/lute/execution/logging.py index 37f453ac..2ad5da5b 100644 --- a/lute/execution/logging.py +++ b/lute/execution/logging.py @@ -60,10 +60,6 @@ def get_logger(name: str) -> logging.Logger: other_logger, logging.PlaceHolder ): other_logger.disabled = True - other_logger.propagate = False - - for handler in other_logger.handlers: - handler.disabled = True elif "numpy" in other_name and not isinstance( other_logger, logging.PlaceHolder ): From fbe8807426ab932d504d073a8a5722e830fe9149 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 12:34:53 -0700 Subject: [PATCH 029/375] [edit] Omit pyFAI INFO related messages for clarity --- lute/execution/executor.py | 3 ++- lute/tasks/geom_opt.py | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/lute/execution/executor.py b/lute/execution/executor.py index 52793768..649ce7e0 100644 --- a/lute/execution/executor.py +++ b/lute/execution/executor.py @@ -43,6 +43,7 @@ from lute.io.models.base import TaskParameters, TemplateParameters from lute.io.db import record_analysis_db from lute.io.elog import post_elog_run_status, post_elog_run_table +from lute.execution.logging import get_logger if __debug__: warnings.simplefilter("default") @@ -54,7 +55,7 @@ warnings.simplefilter("ignore") os.environ["PYTHONWARNINGS"] = "ignore" -logger: logging.Logger = logging.getLogger(__name__) +logger: logging.Logger = get_logger(__name__) class TaskletDict(TypedDict): diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 2dbc33d2..0eef939c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -12,10 +12,6 @@ from lute.execution.ipc import Message from lute.io.models.geom_opt import * from lute.tasks.task import * -from lute.execution.logging import get_logger - -import logging -logger: logging.Logger = get_logger(__name__) import sys sys.path.append('/sdf/home/l/lconreux/LCLSGeom') From 6a0e12c8840e18f0bfe1f28891a1493755ba424e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 13:14:36 -0700 Subject: [PATCH 030/375] [edit] Trying to fix MPI Parallelization --- lute/execution/executor.py | 3 +-- lute/tasks/geom_opt.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lute/execution/executor.py b/lute/execution/executor.py index 649ce7e0..52793768 100644 --- a/lute/execution/executor.py +++ b/lute/execution/executor.py @@ -43,7 +43,6 @@ from lute.io.models.base import TaskParameters, TemplateParameters from lute.io.db import record_analysis_db from lute.io.elog import post_elog_run_status, post_elog_run_table -from lute.execution.logging import get_logger if __debug__: warnings.simplefilter("default") @@ -55,7 +54,7 @@ warnings.simplefilter("ignore") os.environ["PYTHONWARNINGS"] = "ignore" -logger: logging.Logger = get_logger(__name__) +logger: logging.Logger = logging.getLogger(__name__) class TaskletDict(TypedDict): diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0eef939c..4927fa4a 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -23,13 +23,17 @@ from pyFAI.goniometer import SingleGeometry from pyFAI.azimuthalIntegrator import AzimuthalIntegrator from pyFAI.calibrant import CALIBRANT_FACTORY -from mpi4py import MPI from sklearn.gaussian_process import GaussianProcessRegressor from sklearn.gaussian_process.kernels import RBF, ConstantKernel, WhiteKernel from sklearn.utils._testing import ignore_warnings from sklearn.exceptions import ConvergenceWarning from scipy.stats import norm +from mpi4py import MPI +COMM = MPI.COMM_WORLD +RANK = COMM.Get_rank() +SIZE = COMM.Get_size() + class BayesGeomOpt: """ Class to perform Geometry Optimization using Bayesian Optimization on pyFAI @@ -62,9 +66,9 @@ def __init__( self.exp = exp self.run = run self.det_type = det_type.lower() - self.comm = MPI.COMM_WORLD - self.rank = self.comm.Get_rank() - self.size = self.comm.Get_size() + self.comm = COMM + self.rank = RANK + self.size = SIZE self.detector = detector self.calibrant = calibrant self.wavelength = wavelength @@ -477,8 +481,7 @@ def _run(self) -> None: msg = Message(contents="Building PyFAI detector", signal="") self._report_to_executor(msg) detector = self.build_pyFAI_detector() - rank = MPI.COMM_WORLD.Get_rank() - msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}", signal="") + msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {RANK}", signal="") self._report_to_executor(msg) optimizer = BayesGeomOpt( exp=self._task_parameters.exp, From 193d294851f01d15bc82090440fa59f3dc072194 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 13:16:33 -0700 Subject: [PATCH 031/375] [edit] Trying to fix MPI Parallelization --- lute/tasks/geom_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 4927fa4a..329c846e 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -320,6 +320,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat powder = self.min_intensity(Imin, powder) if self.rank == 0: + print(f"Number of distances to scan: {self.size}") distances = np.linspace(bounds['dist'][0], bounds['dist'][1], self.size) else: distances = None From 7e39daaa15d7fa0e00e75fbffea72ac2f6e36827 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 13:31:02 -0700 Subject: [PATCH 032/375] [edit] Set NUMEXPR_MAX_THREADS=1 --- lute/tasks/geom_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 329c846e..4c5ba807 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -33,6 +33,7 @@ COMM = MPI.COMM_WORLD RANK = COMM.Get_rank() SIZE = COMM.Get_size() +NUMEXPR_MAX_THREADS = 1 class BayesGeomOpt: """ From 9268ca1ef0c03dee84b39118340092c93f020fa6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 14:52:59 -0700 Subject: [PATCH 033/375] [edit] Add custom logger prints --- lute/tasks/geom_opt.py | 63 +++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 4c5ba807..9af4ab92 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -17,6 +17,10 @@ sys.path.append('/sdf/home/l/lconreux/LCLSGeom') from LCLSGeom.swap_geom import PsanaToPyFAI, PyFAIToCrystFEL, CrystFELToPsana +import logging +from lute.execution.logging import get_logger +logger = get_logger(__name__) + import numpy as np import matplotlib.pyplot as plt from pyFAI.geometry import Geometry @@ -28,12 +32,7 @@ from sklearn.utils._testing import ignore_warnings from sklearn.exceptions import ConvergenceWarning from scipy.stats import norm - from mpi4py import MPI -COMM = MPI.COMM_WORLD -RANK = COMM.Get_rank() -SIZE = COMM.Get_size() -NUMEXPR_MAX_THREADS = 1 class BayesGeomOpt: """ @@ -67,9 +66,9 @@ def __init__( self.exp = exp self.run = run self.det_type = det_type.lower() - self.comm = COMM - self.rank = RANK - self.size = SIZE + self.comm = MPI.COMM_WORLD + self.rank = self.comm.Get_rank() + self.size = self.comm.Get_size() self.detector = detector self.calibrant = calibrant self.wavelength = wavelength @@ -173,6 +172,7 @@ def bayes_opt_center(self, powder, dist, bounds, res, n_samples=50, n_iterations seed : int Random seed for reproducibility """ + if seed is not None: np.random.seed(seed) @@ -321,13 +321,13 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat powder = self.min_intensity(Imin, powder) if self.rank == 0: - print(f"Number of distances to scan: {self.size}") + logger.info(f"Number of distances to scan: {self.size}") distances = np.linspace(bounds['dist'][0], bounds['dist'][1], self.size) else: distances = None dist = self.comm.scatter(distances, root=0) - print(f"Rank {self.rank} is working on distance {dist}") + logger.info(f"Rank {self.rank} is working on distance {dist}") results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, n_iterations, af, hyperparam, prior, seed) self.comm.Barrier() @@ -478,13 +478,15 @@ def __init__(self, *, params: OptimizePyFAIGeometryParameters, use_mpi: bool = T super().__init__(params=params, use_mpi=use_mpi) def _run(self) -> None: - msg = Message(contents="Starting PyFAI geometry optimization", signal="") - self._report_to_executor(msg) - msg = Message(contents="Building PyFAI detector", signal="") - self._report_to_executor(msg) + # msg = Message(contents="Starting PyFAI geometry optimization", signal="") + # self._report_to_executor(msg) + # msg = Message(contents="Building PyFAI detector", signal="") + # self._report_to_executor(msg) detector = self.build_pyFAI_detector() - msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {RANK}", signal="") - self._report_to_executor(msg) + # msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {RANK}", signal="") + # self._report_to_executor(msg) + rank = MPI.COMM_WORLD.Get_rank() + logger.info(f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}") optimizer = BayesGeomOpt( exp=self._task_parameters.exp, run=self._task_parameters.run, @@ -493,8 +495,8 @@ def _run(self) -> None: calibrant=self._task_parameters.calibrant, wavelength=self._task_parameters.wavelength, ) - msg = Message(contents="Running Bayesian Optimization", signal="") - self._report_to_executor(msg) + # msg = Message(contents="Running Bayesian Optimization", signal="") + # self._report_to_executor(msg) optimizer.bayes_opt_geom( powder=self._task_parameters.powder, bounds=self._task_parameters.bo_params.bounds, @@ -508,16 +510,21 @@ def _run(self) -> None: seed=self._task_parameters.bo_params.seed, ) if optimizer.rank == 0: - msg = Message(contents="Optimization complete", signal="") - self._report_to_executor(msg) - msg = Message(contents=f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") - self._report_to_executor(msg) - msg = Message(contents=f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})", signal="") - self._report_to_executor(msg) - msg = Message(contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", signal="") - self._report_to_executor(msg) - msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") - self._report_to_executor(msg) + # msg = Message(contents="Optimization complete", signal="") + # self._report_to_executor(msg) + # msg = Message(contents=f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") + # self._report_to_executor(msg) + # msg = Message(contents=f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})", signal="") + # self._report_to_executor(msg) + # msg = Message(contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", signal="") + # self._report_to_executor(msg) + # msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") + # self._report_to_executor(msg) + logger.info(f"Optimization complete") + logger.info(f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") + logger.info(f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})") + logger.info(f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e}") + logger.info(f"Final Residuals: {optimizer.residuals:.2e}") detector = self.update_geometry(optimizer) plot = f'{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png' optimizer.visualize_results( From 0d21f406dbf14d15461915180f0faff5b1b58049 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 14:58:57 -0700 Subject: [PATCH 034/375] [edit] Add custom logger prints --- lute/execution/logging.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lute/execution/logging.py b/lute/execution/logging.py index 2ad5da5b..29e490a2 100644 --- a/lute/execution/logging.py +++ b/lute/execution/logging.py @@ -60,6 +60,10 @@ def get_logger(name: str) -> logging.Logger: other_logger, logging.PlaceHolder ): other_logger.disabled = True + elif "goniometer" in other_name and not isinstance( + other_logger, logging.PlaceHolder + ): + other_logger.disabled = True elif "numpy" in other_name and not isinstance( other_logger, logging.PlaceHolder ): From 6895134392fae538301823c634d5437a164774bc Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 15:00:44 -0700 Subject: [PATCH 035/375] [edit] Add custom logger prints --- lute/tasks/geom_opt.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 9af4ab92..363722d2 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -478,13 +478,13 @@ def __init__(self, *, params: OptimizePyFAIGeometryParameters, use_mpi: bool = T super().__init__(params=params, use_mpi=use_mpi) def _run(self) -> None: - # msg = Message(contents="Starting PyFAI geometry optimization", signal="") - # self._report_to_executor(msg) - # msg = Message(contents="Building PyFAI detector", signal="") - # self._report_to_executor(msg) + msg = Message(contents="Starting PyFAI geometry optimization", signal="") + self._report_to_executor(msg) + msg = Message(contents="Building PyFAI detector", signal="") + self._report_to_executor(msg) detector = self.build_pyFAI_detector() - # msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {RANK}", signal="") - # self._report_to_executor(msg) + msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {RANK}", signal="") + self._report_to_executor(msg) rank = MPI.COMM_WORLD.Get_rank() logger.info(f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}") optimizer = BayesGeomOpt( @@ -495,8 +495,8 @@ def _run(self) -> None: calibrant=self._task_parameters.calibrant, wavelength=self._task_parameters.wavelength, ) - # msg = Message(contents="Running Bayesian Optimization", signal="") - # self._report_to_executor(msg) + msg = Message(contents="Running Bayesian Optimization", signal="") + self._report_to_executor(msg) optimizer.bayes_opt_geom( powder=self._task_parameters.powder, bounds=self._task_parameters.bo_params.bounds, @@ -510,16 +510,16 @@ def _run(self) -> None: seed=self._task_parameters.bo_params.seed, ) if optimizer.rank == 0: - # msg = Message(contents="Optimization complete", signal="") - # self._report_to_executor(msg) - # msg = Message(contents=f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") - # self._report_to_executor(msg) - # msg = Message(contents=f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})", signal="") - # self._report_to_executor(msg) - # msg = Message(contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", signal="") - # self._report_to_executor(msg) - # msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") - # self._report_to_executor(msg) + msg = Message(contents="Optimization complete", signal="") + self._report_to_executor(msg) + msg = Message(contents=f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") + self._report_to_executor(msg) + msg = Message(contents=f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})", signal="") + self._report_to_executor(msg) + msg = Message(contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", signal="") + self._report_to_executor(msg) + msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") + self._report_to_executor(msg) logger.info(f"Optimization complete") logger.info(f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") logger.info(f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})") From e44ef5070f5761e29aa6f94a6e7aa79e2d533131 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 15:16:35 -0700 Subject: [PATCH 036/375] [edit] Writing txt files with time per rank for debugging lute MPI parallelization --- lute/tasks/geom_opt.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 363722d2..d41a81c8 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -311,6 +311,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat seed : int Random seed for reproducibility """ + from time import time if seed is not None: np.random.seed(seed) @@ -321,15 +322,19 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat powder = self.min_intensity(Imin, powder) if self.rank == 0: - logger.info(f"Number of distances to scan: {self.size}") + print(f"Number of distances to scan: {self.size}") distances = np.linspace(bounds['dist'][0], bounds['dist'][1], self.size) else: distances = None dist = self.comm.scatter(distances, root=0) - logger.info(f"Rank {self.rank} is working on distance {dist}") + print(f"Rank {self.rank} is working on distance {dist}") + t0 = time() results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, n_iterations, af, hyperparam, prior, seed) + t1 = time() + with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_center_rank_{self.rank}.txt', 'w') as f: + f.write(f"Bayesian Optimization on Center on Rank {self.rank} took {t1-t0:.2f} seconds") self.comm.Barrier() self.scan = {} @@ -478,15 +483,16 @@ def __init__(self, *, params: OptimizePyFAIGeometryParameters, use_mpi: bool = T super().__init__(params=params, use_mpi=use_mpi) def _run(self) -> None: + from time import time msg = Message(contents="Starting PyFAI geometry optimization", signal="") self._report_to_executor(msg) msg = Message(contents="Building PyFAI detector", signal="") self._report_to_executor(msg) detector = self.build_pyFAI_detector() - msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {RANK}", signal="") - self._report_to_executor(msg) rank = MPI.COMM_WORLD.Get_rank() - logger.info(f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}") + msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}", signal="") + self._report_to_executor(msg) + t0 = time() optimizer = BayesGeomOpt( exp=self._task_parameters.exp, run=self._task_parameters.run, @@ -509,6 +515,9 @@ def _run(self) -> None: prior=self._task_parameters.bo_params.prior, seed=self._task_parameters.bo_params.seed, ) + t1 = time() + with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_{rank}.txt', 'w') as f: + f.write(f"Bayesian Optimization Geometry took {t1-t0:.2f} seconds") if optimizer.rank == 0: msg = Message(contents="Optimization complete", signal="") self._report_to_executor(msg) @@ -520,12 +529,6 @@ def _run(self) -> None: self._report_to_executor(msg) msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") self._report_to_executor(msg) - logger.info(f"Optimization complete") - logger.info(f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") - logger.info(f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})") - logger.info(f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e}") - logger.info(f"Final Residuals: {optimizer.residuals:.2e}") - detector = self.update_geometry(optimizer) plot = f'{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png' optimizer.visualize_results( powder=optimizer.powder, From 6fc6736c2184a87253ca8a510da867074b887754 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 15:34:31 -0700 Subject: [PATCH 037/375] [edit] Writing txt files with time per rank and per distance/residuals for debugging lute MPI parallelization --- lute/tasks/geom_opt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d41a81c8..a836da18 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -334,7 +334,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, n_iterations, af, hyperparam, prior, seed) t1 = time() with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_center_rank_{self.rank}.txt', 'w') as f: - f.write(f"Bayesian Optimization on Center on Rank {self.rank} took {t1-t0:.2f} seconds") + f.write(f"Bayesian Optimization on Center with distance {dist:.2f} on Rank {self.rank} took {t1-t0:.2f} seconds") self.comm.Barrier() self.scan = {} @@ -492,7 +492,6 @@ def _run(self) -> None: rank = MPI.COMM_WORLD.Get_rank() msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}", signal="") self._report_to_executor(msg) - t0 = time() optimizer = BayesGeomOpt( exp=self._task_parameters.exp, run=self._task_parameters.run, @@ -503,6 +502,7 @@ def _run(self) -> None: ) msg = Message(contents="Running Bayesian Optimization", signal="") self._report_to_executor(msg) + t0 = time() optimizer.bayes_opt_geom( powder=self._task_parameters.powder, bounds=self._task_parameters.bo_params.bounds, @@ -516,8 +516,8 @@ def _run(self) -> None: seed=self._task_parameters.bo_params.seed, ) t1 = time() - with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_{rank}.txt', 'w') as f: - f.write(f"Bayesian Optimization Geometry took {t1-t0:.2f} seconds") + with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt', 'w') as f: + f.write(f"Bayesian Optimization Geometry gave residuals={optimizer.residuals:.2e} and took {t1-t0:.2f} seconds") if optimizer.rank == 0: msg = Message(contents="Optimization complete", signal="") self._report_to_executor(msg) From a6d43b41680a34febb0b15aefade0651496e107f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 15:50:28 -0700 Subject: [PATCH 038/375] [edit] Writing txt files with time per rank and per distance/residuals for debugging lute MPI parallelization --- lute/tasks/geom_opt.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index a836da18..8b984a0c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -312,6 +312,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat Random seed for reproducibility """ from time import time + from datetime import datetime if seed is not None: np.random.seed(seed) @@ -330,11 +331,12 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat dist = self.comm.scatter(distances, root=0) print(f"Rank {self.rank} is working on distance {dist}") + start = datetime.now() t0 = time() results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, n_iterations, af, hyperparam, prior, seed) t1 = time() with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_center_rank_{self.rank}.txt', 'w') as f: - f.write(f"Bayesian Optimization on Center with distance {dist:.2f} on Rank {self.rank} took {t1-t0:.2f} seconds") + f.write(f"Bayesian Optimization on Center with distance {dist:.2f} on Rank {self.rank} started at {start} and took {t1-t0:.2f} seconds") self.comm.Barrier() self.scan = {} @@ -484,6 +486,7 @@ def __init__(self, *, params: OptimizePyFAIGeometryParameters, use_mpi: bool = T def _run(self) -> None: from time import time + from datetime import datetime msg = Message(contents="Starting PyFAI geometry optimization", signal="") self._report_to_executor(msg) msg = Message(contents="Building PyFAI detector", signal="") @@ -502,6 +505,7 @@ def _run(self) -> None: ) msg = Message(contents="Running Bayesian Optimization", signal="") self._report_to_executor(msg) + start = datetime.now() t0 = time() optimizer.bayes_opt_geom( powder=self._task_parameters.powder, @@ -517,7 +521,7 @@ def _run(self) -> None: ) t1 = time() with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt', 'w') as f: - f.write(f"Bayesian Optimization Geometry gave residuals={optimizer.residuals:.2e} and took {t1-t0:.2f} seconds") + f.write(f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds") if optimizer.rank == 0: msg = Message(contents="Optimization complete", signal="") self._report_to_executor(msg) From d6412537baad8dc7b5406e260a1278490a1470d7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 16:49:03 -0700 Subject: [PATCH 039/375] [edit] Writing txt files with time per rank and per distance/residuals for debugging lute MPI parallelization --- lute/tasks/geom_opt.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8b984a0c..de28dc2d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -520,8 +520,6 @@ def _run(self) -> None: seed=self._task_parameters.bo_params.seed, ) t1 = time() - with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt', 'w') as f: - f.write(f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds") if optimizer.rank == 0: msg = Message(contents="Optimization complete", signal="") self._report_to_executor(msg) @@ -534,6 +532,12 @@ def _run(self) -> None: msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") self._report_to_executor(msg) plot = f'{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png' + with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt', 'w') as f: + f.write(f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds") + f.write(f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") + f.write(f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})") + f.write(f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})") + f.write(f"Final Residuals: {optimizer.residuals:.2e}") optimizer.visualize_results( powder=optimizer.powder, bo_history=optimizer.bo_history, From ca86369c445858ae0d3b34dffb79c8361c5e7cc2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 17:03:20 -0700 Subject: [PATCH 040/375] [edit] Forgot to update geometry before visualizing results --- lute/tasks/geom_opt.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index de28dc2d..23f4ee25 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -533,11 +533,12 @@ def _run(self) -> None: self._report_to_executor(msg) plot = f'{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png' with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt', 'w') as f: - f.write(f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds") - f.write(f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") - f.write(f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})") - f.write(f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})") + f.write(f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n") + f.write(f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e} \n") + f.write(f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e}) \n") + f.write(f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e}) \n") f.write(f"Final Residuals: {optimizer.residuals:.2e}") + detector = self.update_geometry(optimizer) optimizer.visualize_results( powder=optimizer.powder, bo_history=optimizer.bo_history, From f7a60f6c19f7586c5d4fb342492f2e8059b6947f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 31 Oct 2024 17:19:40 -0700 Subject: [PATCH 041/375] [edit] Correct typo in writing txt fiel --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 23f4ee25..46912db5 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -536,7 +536,7 @@ def _run(self) -> None: f.write(f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n") f.write(f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e} \n") f.write(f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e}) \n") - f.write(f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e}) \n") + f.write(f"Rotations: \u03B8x = {optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e} \n") f.write(f"Final Residuals: {optimizer.residuals:.2e}") detector = self.update_geometry(optimizer) optimizer.visualize_results( From e8574c552671a22aff87cb570c2e8c4acfe816c2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 5 Nov 2024 10:56:38 -0800 Subject: [PATCH 042/375] [test] Add NUMEXPR_MAX_THREADS=1 env variable for testing --- launch_scripts/submit_slurm.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/launch_scripts/submit_slurm.sh b/launch_scripts/submit_slurm.sh index bc2f217f..82e06ff0 100755 --- a/launch_scripts/submit_slurm.sh +++ b/launch_scripts/submit_slurm.sh @@ -125,6 +125,7 @@ fi source /sdf/group/lcls/ds/ana/sw/conda1/manage/bin/psconda.sh export LUTE_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd | sed s/launch_scripts//g )" +export NUMEXPR_MAX_THREADS=1 EXECUTABLE="${LUTE_PATH}run_task.py" if [[ ${DEBUG} ]]; then From 638b5d3e4137d741ff978908513b945c54dcf9ca Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 5 Nov 2024 12:10:23 -0800 Subject: [PATCH 043/375] [test] Removed NUMEXPR_MAX_THREADS env variable --- launch_scripts/submit_slurm.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/launch_scripts/submit_slurm.sh b/launch_scripts/submit_slurm.sh index 82e06ff0..bc2f217f 100755 --- a/launch_scripts/submit_slurm.sh +++ b/launch_scripts/submit_slurm.sh @@ -125,7 +125,6 @@ fi source /sdf/group/lcls/ds/ana/sw/conda1/manage/bin/psconda.sh export LUTE_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd | sed s/launch_scripts//g )" -export NUMEXPR_MAX_THREADS=1 EXECUTABLE="${LUTE_PATH}run_task.py" if [[ ${DEBUG} ]]; then From d2e2e427e9091333ebefbb4727c63709eac5d880 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 5 Nov 2024 13:16:22 -0800 Subject: [PATCH 044/375] [test] Set nprocs=4 for testing --- lute/execution/executor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lute/execution/executor.py b/lute/execution/executor.py index 52793768..2f9f558f 100644 --- a/lute/execution/executor.py +++ b/lute/execution/executor.py @@ -1035,9 +1035,10 @@ def _submit_cmd(self, executable_path: str, params: str) -> str: cmd (str): Appropriately formatted command for this Executor. """ py_cmd: str = "" - nprocs: int = max( - int(os.environ.get("SLURM_NPROCS", len(os.sched_getaffinity(0)))) - 1, 1 - ) + #nprocs: int = max( + # int(os.environ.get("SLURM_NPROCS", len(os.sched_getaffinity(0)))) - 1, 1 + #) + nprocs: int = 4 mpi_cmd: str = f"mpirun -np {nprocs}" if __debug__: py_cmd = f"python -B -u -m mpi4py.run {executable_path} {params}" From 11a846baa08ae9241de838e4324faa48020c9ca2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 8 Nov 2024 10:21:15 -0800 Subject: [PATCH 045/375] [test] Rerun time tests with mpirun -np nprocs --bind-to none first --- lute/execution/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/execution/executor.py b/lute/execution/executor.py index 4535aa5e..3bf9fc26 100644 --- a/lute/execution/executor.py +++ b/lute/execution/executor.py @@ -1151,7 +1151,7 @@ def _submit_cmd(self, executable_path: str, params: str) -> str: nprocs: int = max( int(os.environ.get("SLURM_NPROCS", len(os.sched_getaffinity(0)))) - 1, 1 ) - mpi_cmd: str = f"mpirun -np {nprocs}" + mpi_cmd: str = f"mpirun -np {nprocs} --bind-to none" if __debug__: py_cmd = f"python -B -u -m mpi4py.run {executable_path} {params}" else: From 109a94d71113bccb51b43211d9cefcf7ae7366b6 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 8 Nov 2024 18:21:43 +0000 Subject: [PATCH 046/375] Auto-commit black formatting --- lute/io/models/geom_opt.py | 52 +++--- lute/managed_tasks.py | 2 +- lute/tasks/__init__.py | 2 +- lute/tasks/geom_opt.py | 363 +++++++++++++++++++++++++++---------- 4 files changed, 300 insertions(+), 119 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 01c678da..e31db638 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -19,15 +19,17 @@ from PSCalib.CalibFileFinder import CalibFileFinder + class OptimizePyFAIGeometryParameters(TaskParameters): """Parameters for optimizing detector geometry using PyFAI and Bayesian optimization. - + The Bayesian Optimization has default hyperparameters that can be overriden by the user. """ + class Config(TaskParameters.Config): - set_result: bool = True - """Whether the Executor should mark a specified parameter as a result.""" - + set_result: bool = True + """Whether the Executor should mark a specified parameter as a result.""" + class BayesGeomOptParameters(BaseModel): """Bayesian optimization hyperparameters.""" @@ -56,8 +58,8 @@ class BayesGeomOptParameters(BaseModel): ) Imin: Optional[Union[str, int]] = Field( - 'max', - description="Minimal Intensity for extracting key control points, either based on heuristics on the maximal intensity, or a multiple of the photon energy" + "max", + description="Minimal Intensity for extracting key control points, either based on heuristics on the maximal intensity, or a multiple of the photon energy", ) prior: Optional[bool] = Field( @@ -70,7 +72,7 @@ class BayesGeomOptParameters(BaseModel): description="Acquisition function to be used by the Bayesian optimization.", ) - hyperparams : Optional[Dict[str, float]] = Field( + hyperparams: Optional[Dict[str, float]] = Field( { "beta": 1.96, "epsilon": 0.01, @@ -78,32 +80,32 @@ class BayesGeomOptParameters(BaseModel): description="Hyperparameters for the acquisition function.", ) - seed : Optional[int] = Field( + seed: Optional[int] = Field( None, description="Seed for the random number generator for potential reproducibility.", ) - exp : str = Field( + exp: str = Field( "", description="Experiment name.", ) - run : Union[str, int] = Field( + run: Union[str, int] = Field( None, description="Run number.", ) - det_type : str = Field( + det_type: str = Field( "", description="Detector type. Currently supported: 'ePix10k2M', 'ePix10kaQuad', 'Rayonix', 'Rayonix2', 'Jungfrau1M', 'Jungfrau4M'", ) - date : str = Field( + date: str = Field( "", description="Start date of analysis", ) - work_dir : str = Field( + work_dir: str = Field( "", description="Main working directory for LUTE.", ) @@ -120,7 +122,7 @@ class BayesGeomOptParameters(BaseModel): calibrant: str = Field( "", - description="Calibrant used for the calibration supported by pyFAI: https://github.com/silx-kit/pyFAI/tree/main/src/pyFAI/resources/calibration" + description="Calibrant used for the calibration supported by pyFAI: https://github.com/silx-kit/pyFAI/tree/main/src/pyFAI/resources/calibration", ) wavelength: float = Field( @@ -144,33 +146,35 @@ def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: if exp == "": exp: str = values["lute_config"].experiment return exp - + @validator("run", always=True) - def validate_run(cls, run: Union[str, int], values: Dict[str, Any]) -> Union[str, int]: + def validate_run( + cls, run: Union[str, int], values: Dict[str, Any] + ) -> Union[str, int]: if run is None: run: Union[str, int] = values["lute_config"].run return run - + @validator("date", always=True) def validate_date(cls, date: str, values: Dict[str, Any]) -> str: if date == "": date: str = values["lute_config"].date return date - + @validator("work_dir", always=True) def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: if work_dir == "": work_dir: str = values["lute_config"].work_dir return work_dir - + @validator("in_file", always=True) def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: if in_file == "": exp = values["exp"] run = values["run"] - cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' - src = 'MfxEndstation.0:Epix10ka2M.0' - type = 'geometry' + cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" + src = "MfxEndstation.0:Epix10ka2M.0" + type = "geometry" cff = CalibFileFinder(cdir) in_file = cff.findCalibFile(src, type, run) return in_file @@ -183,11 +187,11 @@ def validate_powder(cls, powder: str, values: Dict[str, Any]) -> str: f"{work_dir}/powder", "ComputePowder", "out_file" ) return powder - + @validator("out_file", always=True) def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: if out_file == "": in_file = values["in_file"] run = values["run"] out_file: str = in_file.replace("0-end.data", f"{run}-end.data") - return out_file \ No newline at end of file + return out_file diff --git a/lute/managed_tasks.py b/lute/managed_tasks.py index 0495f519..e44c5ec4 100644 --- a/lute/managed_tasks.py +++ b/lute/managed_tasks.py @@ -134,4 +134,4 @@ # PyFAI ####### PyFAIGeometryOptimizer: MPIExecutor = MPIExecutor("OptimizePyFAIGeometry") -"""Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" \ No newline at end of file +"""Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" diff --git a/lute/tasks/__init__.py b/lute/tasks/__init__.py index 4ceb1c06..6fd6b8ee 100644 --- a/lute/tasks/__init__.py +++ b/lute/tasks/__init__.py @@ -85,7 +85,7 @@ def import_task(task_name: str) -> Type[Task]: from .mpi_test import TestMultiNodeCommunication return TestMultiNodeCommunication - + if task_name == "OptimizePyFAIGeometry": from .geom_opt import OptimizePyFAIGeometry diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 46912db5..af13f5d6 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -14,11 +14,13 @@ from lute.tasks.task import * import sys -sys.path.append('/sdf/home/l/lconreux/LCLSGeom') + +sys.path.append("/sdf/home/l/lconreux/LCLSGeom") from LCLSGeom.swap_geom import PsanaToPyFAI, PyFAIToCrystFEL, CrystFELToPsana import logging from lute.execution.logging import get_logger + logger = get_logger(__name__) import numpy as np @@ -34,6 +36,7 @@ from scipy.stats import norm from mpi4py import MPI + class BayesGeomOpt: """ Class to perform Geometry Optimization using Bayesian Optimization on pyFAI @@ -74,7 +77,14 @@ def __init__( self.wavelength = wavelength self.order = ["dist", "poni1", "poni2", "rot1", "rot2", "rot3"] self.space = ["poni1", "poni2"] - self.values = {'dist': 0.1,'poni1':0, 'poni2':0, 'rot1':0, 'rot2':0, 'rot3':0} + self.values = { + "dist": 0.1, + "poni1": 0, + "poni2": 0, + "rot1": 0, + "rot2": 0, + "rot3": 0, + } @staticmethod def expected_improvement(X, gp_model, best_y, epsilon=0): @@ -95,15 +105,15 @@ def probability_of_improvement(X, gp_model, best_y, epsilon=0): z = (y_pred - best_y + epsilon) / y_std pi = norm.cdf(z) return pi - + @staticmethod def contextual_improvement(X, gp_model, best_y, hyperparam=None): y_pred, y_std = gp_model.predict(X, return_std=True) cv = np.mean(y_std**2) / best_y - z = (y_pred - best_y + cv) / y_std + z = (y_pred - best_y + cv) / y_std ci = y_pred - best_y * norm.cdf(z) + y_std * norm.pdf(z) return ci - + def build_calibrant(self): """ Define calibrant for optimization @@ -131,7 +141,7 @@ def min_intensity(self, Imin, powder): Minimum intensity to use for control point extraction based on photon energy or max intensity """ if type(Imin) == str: - if 'rayonix' in self.det_type: + if "rayonix" in self.det_type: powder = powder[powder > 1e3] Imin = np.max(powder) * 0.01 else: @@ -143,10 +153,22 @@ def min_intensity(self, Imin, powder): return powder @ignore_warnings(category=ConvergenceWarning) - def bayes_opt_center(self, powder, dist, bounds, res, n_samples=50, n_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): + def bayes_opt_center( + self, + powder, + dist, + bounds, + res, + n_samples=50, + n_iterations=50, + af="ucb", + hyperparam=None, + prior=True, + seed=None, + ): """ Perform Bayesian Optimization on PONI center parameters, for a fixed distance - + Parameters ---------- powder : np.ndarray @@ -176,7 +198,7 @@ def bayes_opt_center(self, powder, dist, bounds, res, n_samples=50, n_iterations if seed is not None: np.random.seed(seed) - self.values['dist'] = dist + self.values["dist"] = dist if res is None: res = self.detector.pixel_size @@ -185,18 +207,31 @@ def bayes_opt_center(self, powder, dist, bounds, res, n_samples=50, n_iterations norm_inputs = {} for p in self.order: if p in self.space: - inputs[p] = np.arange(bounds[p][0], bounds[p][1]+res, res) + inputs[p] = np.arange(bounds[p][0], bounds[p][1] + res, res) norm_inputs[p] = inputs[p] else: inputs[p] = np.array([self.values[p]]) - X = np.array(np.meshgrid(*[inputs[p] for p in self.order])).T.reshape(-1, len(self.order)) - X_space = np.array(np.meshgrid(*[norm_inputs[p] for p in self.space])).T.reshape(-1, len(self.space)) - X_norm = (X_space - np.mean(X_space, axis=0)) / (np.max(X_space, axis=0) - np.min(X_space, axis=0)) + X = np.array(np.meshgrid(*[inputs[p] for p in self.order])).T.reshape( + -1, len(self.order) + ) + X_space = np.array( + np.meshgrid(*[norm_inputs[p] for p in self.space]) + ).T.reshape(-1, len(self.space)) + X_norm = (X_space - np.mean(X_space, axis=0)) / ( + np.max(X_space, axis=0) - np.min(X_space, axis=0) + ) if prior: means = np.mean(X_space, axis=0) - cov = np.diag([((bounds[param][1] - bounds[param][0]) / 5)**2 for param in self.space]) + cov = np.diag( + [ + ((bounds[param][1] - bounds[param][0]) / 5) ** 2 + for param in self.space + ] + ) X_samples = np.random.multivariate_normal(means, cov, n_samples) - X_norm_samples = (X_samples - np.mean(X_space, axis=0)) / (np.max(X_space, axis=0) - np.min(X_space, axis=0)) + X_norm_samples = (X_samples - np.mean(X_space, axis=0)) / ( + np.max(X_space, axis=0) - np.min(X_space, axis=0) + ) for p in self.order: if p not in self.space: idx = self.order.index(p) @@ -210,36 +245,53 @@ def bayes_opt_center(self, powder, dist, bounds, res, n_samples=50, n_iterations y = np.zeros((n_samples)) for i in range(n_samples): dist, poni1, poni2, rot1, rot2, rot3 = X_samples[i] - geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) - sg = SingleGeometry("extract_cp", powder, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + geom_initial = Geometry( + dist=dist, + poni1=poni1, + poni2=poni2, + rot1=rot1, + rot2=rot2, + rot3=rot3, + detector=self.detector, + wavelength=self.calibrant.wavelength, + ) + sg = SingleGeometry( + "extract_cp", + powder, + calibrant=self.calibrant, + detector=self.detector, + geometry=geom_initial, + ) sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) y[i] = len(sg.geometry_refinement.data) - bo_history[f'init_sample_{i+1}'] = {'param':X_samples[i], 'score': y[i]} + bo_history[f"init_sample_{i+1}"] = {"param": X_samples[i], "score": y[i]} y_norm = (y - np.mean(y)) / np.std(y) best_score = np.max(y_norm) - kernel = RBF(length_scale=0.3, length_scale_bounds=(0.2, 0.4)) \ - * ConstantKernel(constant_value=1.0, constant_value_bounds=(0.5, 1.5)) \ - + WhiteKernel(noise_level=0.001, noise_level_bounds = 'fixed') - gp_model = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, random_state=42) + kernel = RBF(length_scale=0.3, length_scale_bounds=(0.2, 0.4)) * ConstantKernel( + constant_value=1.0, constant_value_bounds=(0.5, 1.5) + ) + WhiteKernel(noise_level=0.001, noise_level_bounds="fixed") + gp_model = GaussianProcessRegressor( + kernel=kernel, n_restarts_optimizer=10, random_state=42 + ) gp_model.fit(X_norm_samples, y_norm) visited_idx = list([]) if af == "ucb": if hyperparam is None: - hyperparam = {'beta': 1.96} - hyperparam = hyperparam['beta'] + hyperparam = {"beta": 1.96} + hyperparam = hyperparam["beta"] af = self.upper_confidence_bound elif af == "ei": if hyperparam is None: - hyperparam = {'epsilon': 0} - hyperparam = hyperparam['epsilon'] + hyperparam = {"epsilon": 0} + hyperparam = hyperparam["epsilon"] af = self.expected_improvement elif af == "pi": if hyperparam is None: - hyperparam = {'epsilon': 0} - hyperparam = hyperparam['epsilon'] + hyperparam = {"epsilon": 0} + hyperparam = hyperparam["epsilon"] af = self.probability_of_improvement elif af == "ci": af = self.contextual_improvement @@ -247,8 +299,8 @@ def bayes_opt_center(self, powder, dist, bounds, res, n_samples=50, n_iterations for i in range(n_iterations): # 1. Generate the Acquisition Function values using the Gaussian Process Regressor af_values = af(X_norm, gp_model, best_score, hyperparam) - af_values[visited_idx] = -np.inf - + af_values[visited_idx] = -np.inf + # 2. Select the next set of parameters based on the Acquisition Function new_idx = np.argmax(af_values) new_input = X[new_idx] @@ -256,33 +308,85 @@ def bayes_opt_center(self, powder, dist, bounds, res, n_samples=50, n_iterations # 3. Compute the score of the new set of parameters dist, poni1, poni2, rot1, rot2, rot3 = new_input - geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) - sg = SingleGeometry("extract_cp", powder, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + geom_initial = Geometry( + dist=dist, + poni1=poni1, + poni2=poni2, + rot1=rot1, + rot2=rot2, + rot3=rot3, + detector=self.detector, + wavelength=self.calibrant.wavelength, + ) + sg = SingleGeometry( + "extract_cp", + powder, + calibrant=self.calibrant, + detector=self.detector, + geometry=geom_initial, + ) sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) score = len(sg.geometry_refinement.data) y = np.append(y, [score], axis=0) ypred = gp_model.predict(X_norm, return_std=False) - bo_history[f'iteration_{i+1}'] = {'param':X[new_idx], 'score': score, 'pred': ypred, 'af': af_values} + bo_history[f"iteration_{i+1}"] = { + "param": X[new_idx], + "score": score, + "pred": ypred, + "af": af_values, + } X_samples = np.append(X_samples, [X[new_idx]], axis=0) X_norm_samples = np.append(X_norm_samples, [X_norm[new_idx]], axis=0) y_norm = (y - np.mean(y)) / np.std(y) best_score = np.max(y_norm) # 4. Update the Gaussian Process Regressor gp_model.fit(X_norm_samples, y_norm) - + best_idx = np.argmax(y_norm) best_param = X_samples[best_idx] dist, poni1, poni2, rot1, rot2, rot3 = best_param - geom_initial = Geometry(dist=dist, poni1=poni1, poni2=poni2, rot1=rot1, rot2=rot2, rot3=rot3, detector=self.detector, wavelength=self.calibrant.wavelength) - sg = SingleGeometry("extract_cp", powder, calibrant=self.calibrant, detector=self.detector, geometry=geom_initial) + geom_initial = Geometry( + dist=dist, + poni1=poni1, + poni2=poni2, + rot1=rot1, + rot2=rot2, + rot3=rot3, + detector=self.detector, + wavelength=self.calibrant.wavelength, + ) + sg = SingleGeometry( + "extract_cp", + powder, + calibrant=self.calibrant, + detector=self.detector, + geometry=geom_initial, + ) sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) self.sg = sg - residuals = sg.geometry_refinement.refine3(fix=['wavelength']) + residuals = sg.geometry_refinement.refine3(fix=["wavelength"]) params = sg.geometry_refinement.param - result = {'bo_history': bo_history, 'params': params, 'residuals': residuals, 'best_idx': best_idx} + result = { + "bo_history": bo_history, + "params": params, + "residuals": residuals, + "best_idx": best_idx, + } return result - def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterations=50, af="ucb", hyperparam=None, prior=True, seed=None): + def bayes_opt_geom( + self, + powder, + bounds, + res, + Imin="max", + n_samples=50, + n_iterations=50, + af="ucb", + hyperparam=None, + prior=True, + seed=None, + ): """ From guessed initial geometry, optimize the geometry using Bayesian Optimization on pyFAI package @@ -313,6 +417,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat """ from time import time from datetime import datetime + if seed is not None: np.random.seed(seed) @@ -324,7 +429,7 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat if self.rank == 0: print(f"Number of distances to scan: {self.size}") - distances = np.linspace(bounds['dist'][0], bounds['dist'][1], self.size) + distances = np.linspace(bounds["dist"][0], bounds["dist"][1], self.size) else: distances = None @@ -333,28 +438,43 @@ def bayes_opt_geom(self, powder, bounds, res, Imin='max', n_samples=50, n_iterat start = datetime.now() t0 = time() - results = self.bayes_opt_center(powder, dist, bounds, res, n_samples, n_iterations, af, hyperparam, prior, seed) + results = self.bayes_opt_center( + powder, + dist, + bounds, + res, + n_samples, + n_iterations, + af, + hyperparam, + prior, + seed, + ) t1 = time() - with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_center_rank_{self.rank}.txt', 'w') as f: - f.write(f"Bayesian Optimization on Center with distance {dist:.2f} on Rank {self.rank} started at {start} and took {t1-t0:.2f} seconds") + with open( + f"/sdf/home/l/lconreux/launchpad/bayes_opt_center_rank_{self.rank}.txt", "w" + ) as f: + f.write( + f"Bayesian Optimization on Center with distance {dist:.2f} on Rank {self.rank} started at {start} and took {t1-t0:.2f} seconds" + ) self.comm.Barrier() self.scan = {} - self.scan['bo_history'] = self.comm.gather(results['bo_history'], root=0) - self.scan['params'] = self.comm.gather(results['params'], root=0) - self.scan['residuals'] = self.comm.gather(results['residuals'], root=0) - self.scan['best_idx'] = self.comm.gather(results['best_idx'], root=0) + self.scan["bo_history"] = self.comm.gather(results["bo_history"], root=0) + self.scan["params"] = self.comm.gather(results["params"], root=0) + self.scan["residuals"] = self.comm.gather(results["residuals"], root=0) + self.scan["best_idx"] = self.comm.gather(results["best_idx"], root=0) self.finalize() def finalize(self): if self.rank == 0: for key in self.scan.keys(): - self.scan[key] = np.array([item for item in self.scan[key]]) - index = np.argmin(self.scan['residuals']) - self.bo_history = self.scan['bo_history'][index] - self.params = self.scan['params'][index] - self.residuals = self.scan['residuals'][index] - self.best_idx = self.scan['best_idx'][index] + self.scan[key] = np.array([item for item in self.scan[key]]) + index = np.argmin(self.scan["residuals"]) + self.bo_history = self.scan["bo_history"][index] + self.params = self.scan["params"][index] + self.residuals = self.scan["residuals"][index] + self.best_idx = self.scan["best_idx"][index] def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): """ @@ -375,17 +495,16 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ai = sg.geometry_refinement if label is None: label = sg.label - ax.imshow(powder.T, - origin="lower", - cmap="viridis", - vmax=self.Imin) + ax.imshow(powder.T, origin="lower", cmap="viridis", vmax=self.Imin) ax.set_title(label) if ai is not None and cp.calibrant is not None: tth = cp.calibrant.get_2th() ttha = ai.twoThetaArray() - ax.contour(ttha.T, levels=tth, cmap="autumn", linewidths=0.5, linestyles="dashed") + ax.contour( + ttha.T, levels=tth, cmap="autumn", linewidths=0.5, linestyles="dashed" + ) return ax - + def radial_integration(self, result, calibrant=None, label=None, ax=None): """ Display the powder diffraction pattern @@ -421,8 +540,13 @@ def radial_integration(self, result, calibrant=None, label=None, ax=None): x_values = calibrant.get_peaks(unit) if x_values is not None: for x in x_values: - line = lines.Line2D([x, x], ax.axis()[2:4], - color='red', linestyle='--', linewidth=0.5) + line = lines.Line2D( + [x, x], + ax.axis()[2:4], + color="red", + linestyle="--", + linewidth=0.5, + ) ax.add_line(line) ax.set_title("Radial Profile") @@ -430,7 +554,7 @@ def radial_integration(self, result, calibrant=None, label=None, ax=None): ax.set_xlabel(unit.label) ax.set_ylabel("Intensity") - def visualize_results(self, powder, bo_history, detector, params, plot=''): + def visualize_results(self, powder, bo_history, detector, params, plot=""): """ Visualize fit, plotting (1) the BO convergence, (2) the radial profile and (3) the powder image. @@ -447,53 +571,70 @@ def visualize_results(self, powder, bo_history, detector, params, plot=''): plot : str Path to save plot """ - fig = plt.figure(figsize=(8,8),dpi=120) - nrow,ncol=2,2 - irow,icol=0,0 - + fig = plt.figure(figsize=(8, 8), dpi=120) + nrow, ncol = 2, 2 + irow, icol = 0, 0 + # Plotting BO convergence ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) - scores = [bo_history[key]['score'] for key in bo_history.keys()] + scores = [bo_history[key]["score"] for key in bo_history.keys()] ax1.plot(np.maximum.accumulate(scores)) ax1.set_xticks(np.arange(len(scores), step=20)) - ax1.set_xlabel('Iteration') - ax1.set_ylabel('Best score so far') - ax1.set_title('Convergence Plot') + ax1.set_xlabel("Iteration") + ax1.set_ylabel("Best score so far") + ax1.set_title("Convergence Plot") icol += 1 # Plotting radial profiles with peaks - ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol-icol) - ai = AzimuthalIntegrator(dist=params[0], detector=detector, wavelength=self.calibrant.wavelength) + ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) + ai = AzimuthalIntegrator( + dist=params[0], detector=detector, wavelength=self.calibrant.wavelength + ) res = ai.integrate1d(powder, 1000) self.radial_integration(res, calibrant=self.calibrant, ax=ax2) irow += 1 # Plotting stacked powder geometry = Geometry(dist=params[0]) - sg = SingleGeometry(f'Max {self.calibrant_name}', powder, calibrant=self.calibrant, detector=detector, geometry=geometry) + sg = SingleGeometry( + f"Max {self.calibrant_name}", + powder, + calibrant=self.calibrant, + detector=detector, + geometry=geometry, + ) sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) - ax3 = plt.subplot2grid((nrow, ncol), (irow, 0), rowspan=nrow-irow, colspan=ncol) + ax3 = plt.subplot2grid( + (nrow, ncol), (irow, 0), rowspan=nrow - irow, colspan=ncol + ) self.display(sg=sg, ax=ax3) - if plot != '': + if plot != "": fig.savefig(plot, dpi=300) + class OptimizePyFAIGeometry(Task): """Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" - def __init__(self, *, params: OptimizePyFAIGeometryParameters, use_mpi: bool = True) -> None: + def __init__( + self, *, params: OptimizePyFAIGeometryParameters, use_mpi: bool = True + ) -> None: super().__init__(params=params, use_mpi=use_mpi) def _run(self) -> None: from time import time from datetime import datetime + msg = Message(contents="Starting PyFAI geometry optimization", signal="") self._report_to_executor(msg) msg = Message(contents="Building PyFAI detector", signal="") self._report_to_executor(msg) detector = self.build_pyFAI_detector() rank = MPI.COMM_WORLD.Get_rank() - msg = Message(contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}", signal="") + msg = Message( + contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}", + signal="", + ) self._report_to_executor(msg) optimizer = BayesGeomOpt( exp=self._task_parameters.exp, @@ -523,20 +664,40 @@ def _run(self) -> None: if optimizer.rank == 0: msg = Message(contents="Optimization complete", signal="") self._report_to_executor(msg) - msg = Message(contents=f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") + msg = Message( + contents=f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}" + ) self._report_to_executor(msg) - msg = Message(contents=f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})", signal="") + msg = Message( + contents=f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})", + signal="", + ) self._report_to_executor(msg) - msg = Message(contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", signal="") + msg = Message( + contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", + signal="", + ) self._report_to_executor(msg) - msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") + msg = Message( + contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="" + ) self._report_to_executor(msg) - plot = f'{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png' - with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt', 'w') as f: - f.write(f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n") - f.write(f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e} \n") - f.write(f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e}) \n") - f.write(f"Rotations: \u03B8x = {optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e} \n") + plot = f"{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png" + with open( + f"/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt", "w" + ) as f: + f.write( + f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n" + ) + f.write( + f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e} \n" + ) + f.write( + f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e}) \n" + ) + f.write( + f"Rotations: \u03B8x = {optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e} \n" + ) f.write(f"Final Residuals: {optimizer.residuals:.2e}") detector = self.update_geometry(optimizer) optimizer.visualize_results( @@ -551,20 +712,36 @@ def _run(self) -> None: def build_pyFAI_detector(self): """ - Fetch the geometry data and build a pyFAI detector object. + Fetch the geometry data and build a pyFAI detector object. """ in_file = self._task_parameters.in_file det_type = self._task_parameters.det_type psana_to_pyfai = PsanaToPyFAI(in_file=in_file, det_type=det_type) detector = psana_to_pyfai.detector return detector - + def update_geometry(self, optimizer): """ Update the geometry and write a new .geom file and .data file """ - PyFAIToCrystFEL(detector=optimizer.detector, params=optimizer.params, psana_file=self._task_parameters.in_file, out_file=self._task_parameters.in_file.replace("0-end.data", f"r{self._task_parameters.run:0>4}.geom")) - CrystFELToPsana(in_file=self._task_parameters.in_file.replace("0-end.data", f"r{self._task_parameters.run:0>4}.geom"), det_type=optimizer.det_type, out_file=self._task_parameters.out_file) - psana_to_pyfai = PsanaToPyFAI(in_file=self._task_parameters.out_file, det_type=self._task_parameters.det_type) + PyFAIToCrystFEL( + detector=optimizer.detector, + params=optimizer.params, + psana_file=self._task_parameters.in_file, + out_file=self._task_parameters.in_file.replace( + "0-end.data", f"r{self._task_parameters.run:0>4}.geom" + ), + ) + CrystFELToPsana( + in_file=self._task_parameters.in_file.replace( + "0-end.data", f"r{self._task_parameters.run:0>4}.geom" + ), + det_type=optimizer.det_type, + out_file=self._task_parameters.out_file, + ) + psana_to_pyfai = PsanaToPyFAI( + in_file=self._task_parameters.out_file, + det_type=self._task_parameters.det_type, + ) detector = psana_to_pyfai.detector - return detector \ No newline at end of file + return detector From 182741968c25b8295393c96e99e897d3ed20b383 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 8 Nov 2024 15:00:18 -0800 Subject: [PATCH 047/375] [test] added --map-by slot to mpi args for testing --- lute/execution/executor.py | 2 +- lute/tasks/geom_opt.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lute/execution/executor.py b/lute/execution/executor.py index 3bf9fc26..8085a346 100644 --- a/lute/execution/executor.py +++ b/lute/execution/executor.py @@ -1151,7 +1151,7 @@ def _submit_cmd(self, executable_path: str, params: str) -> str: nprocs: int = max( int(os.environ.get("SLURM_NPROCS", len(os.sched_getaffinity(0)))) - 1, 1 ) - mpi_cmd: str = f"mpirun -np {nprocs} --bind-to none" + mpi_cmd: str = f"mpirun -np {nprocs} --map-by slot" if __debug__: py_cmd = f"python -B -u -m mpi4py.run {executable_path} {params}" else: diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 46912db5..084a3b66 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -12,10 +12,11 @@ from lute.execution.ipc import Message from lute.io.models.geom_opt import * from lute.tasks.task import * +from lute.tasks.dataclasses import * import sys sys.path.append('/sdf/home/l/lconreux/LCLSGeom') -from LCLSGeom.swap_geom import PsanaToPyFAI, PyFAIToCrystFEL, CrystFELToPsana +from LCLSGeom.swap_geom import PsanaToPyFAI, PyFAIToCrystFEL, CrystFELToPsana, get_beam_center import logging from lute.execution.logging import get_logger @@ -523,9 +524,10 @@ def _run(self) -> None: if optimizer.rank == 0: msg = Message(contents="Optimization complete", signal="") self._report_to_executor(msg) - msg = Message(contents=f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e}") + distance, cx, cy = get_beam_center(optimizer.params) + msg = Message(contents=f"Detector Distance to Point of Normal Incidence: {distance:.2e}") self._report_to_executor(msg) - msg = Message(contents=f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e})", signal="") + msg = Message(contents=f"Beam center: ({cx:.2e}, {cy:.2e})", signal="") self._report_to_executor(msg) msg = Message(contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", signal="") self._report_to_executor(msg) @@ -534,8 +536,8 @@ def _run(self) -> None: plot = f'{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png' with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt', 'w') as f: f.write(f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n") - f.write(f"Detector Distance to Point of Normal Incidence: {optimizer.params[0]:.2e} \n") - f.write(f"Beam center: ({optimizer.params[1]:.2e}, {optimizer.params[2]:.2e}) \n") + f.write(f"Detector Distance to Point of Normal Incidence: {distance:.2e} \n") + f.write(f"Beam center: ({cx:.2e}, {cy:.2e}) \n") f.write(f"Rotations: \u03B8x = {optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e} \n") f.write(f"Final Residuals: {optimizer.residuals:.2e}") detector = self.update_geometry(optimizer) From 7fd425950a303f1b6e058d8c920827d415a4589a Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 8 Nov 2024 23:07:33 +0000 Subject: [PATCH 048/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 5585d12f..65cd5ac2 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -17,7 +17,12 @@ import sys sys.path.append("/sdf/home/l/lconreux/LCLSGeom") -from LCLSGeom.swap_geom import PsanaToPyFAI, PyFAIToCrystFEL, CrystFELToPsana, get_beam_center +from LCLSGeom.swap_geom import ( + PsanaToPyFAI, + PyFAIToCrystFEL, + CrystFELToPsana, + get_beam_center, +) import logging from lute.execution.logging import get_logger @@ -666,20 +671,35 @@ def _run(self) -> None: msg = Message(contents="Optimization complete", signal="") self._report_to_executor(msg) distance, cx, cy = get_beam_center(optimizer.params) - msg = Message(contents=f"Detector Distance to Point of Normal Incidence: {distance:.2e}") + msg = Message( + contents=f"Detector Distance to Point of Normal Incidence: {distance:.2e}" + ) self._report_to_executor(msg) msg = Message(contents=f"Beam center: ({cx:.2e}, {cy:.2e})", signal="") self._report_to_executor(msg) - msg = Message(contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", signal="") + msg = Message( + contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", + signal="", + ) self._report_to_executor(msg) - msg = Message(contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="") + msg = Message( + contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="" + ) self._report_to_executor(msg) - plot = f'{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png' - with open(f'/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt', 'w') as f: - f.write(f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n") - f.write(f"Detector Distance to Point of Normal Incidence: {distance:.2e} \n") + plot = f"{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png" + with open( + f"/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt", "w" + ) as f: + f.write( + f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n" + ) + f.write( + f"Detector Distance to Point of Normal Incidence: {distance:.2e} \n" + ) f.write(f"Beam center: ({cx:.2e}, {cy:.2e}) \n") - f.write(f"Rotations: \u03B8x = {optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e} \n") + f.write( + f"Rotations: \u03B8x = {optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e} \n" + ) f.write(f"Final Residuals: {optimizer.residuals:.2e}") detector = self.update_geometry(optimizer) optimizer.visualize_results( From b00107adc4484133e3f99d816bbae8d0883a4d83 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 10 Nov 2024 11:04:36 -0800 Subject: [PATCH 049/375] [test] --map-by core time testings --- lute/execution/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/execution/executor.py b/lute/execution/executor.py index 8085a346..04dd5619 100644 --- a/lute/execution/executor.py +++ b/lute/execution/executor.py @@ -1151,7 +1151,7 @@ def _submit_cmd(self, executable_path: str, params: str) -> str: nprocs: int = max( int(os.environ.get("SLURM_NPROCS", len(os.sched_getaffinity(0)))) - 1, 1 ) - mpi_cmd: str = f"mpirun -np {nprocs} --map-by slot" + mpi_cmd: str = f"mpirun -np {nprocs} --map-by core" if __debug__: py_cmd = f"python -B -u -m mpi4py.run {executable_path} {params}" else: From 8d434fc153afee635f5f872dc3656a4b337a0506 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 10 Nov 2024 17:31:36 -0800 Subject: [PATCH 050/375] [test] Commented writing of files for MPI debugging --- lute/tasks/geom_opt.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 65cd5ac2..d21d5903 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -457,12 +457,12 @@ def bayes_opt_geom( seed, ) t1 = time() - with open( - f"/sdf/home/l/lconreux/launchpad/bayes_opt_center_rank_{self.rank}.txt", "w" - ) as f: - f.write( - f"Bayesian Optimization on Center with distance {dist:.2f} on Rank {self.rank} started at {start} and took {t1-t0:.2f} seconds" - ) + # with open( + # f"/sdf/home/l/lconreux/launchpad/bayes_opt_center_rank_{self.rank}.txt", "w" + # ) as f: + # f.write( + # f"Bayesian Optimization on Center with distance {dist:.2f} on Rank {self.rank} started at {start} and took {t1-t0:.2f} seconds" + # ) self.comm.Barrier() self.scan = {} @@ -687,20 +687,20 @@ def _run(self) -> None: ) self._report_to_executor(msg) plot = f"{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png" - with open( - f"/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt", "w" - ) as f: - f.write( - f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n" - ) - f.write( - f"Detector Distance to Point of Normal Incidence: {distance:.2e} \n" - ) - f.write(f"Beam center: ({cx:.2e}, {cy:.2e}) \n") - f.write( - f"Rotations: \u03B8x = {optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e} \n" - ) - f.write(f"Final Residuals: {optimizer.residuals:.2e}") + # with open( + # f"/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt", "w" + # ) as f: + # f.write( + # f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n" + # ) + # f.write( + # f"Detector Distance to Point of Normal Incidence: {distance:.2e} \n" + # ) + # f.write(f"Beam center: ({cx:.2e}, {cy:.2e}) \n") + # f.write( + # f"Rotations: \u03B8x = {optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e} \n" + # ) + # f.write(f"Final Residuals: {optimizer.residuals:.2e}") detector = self.update_geometry(optimizer) optimizer.visualize_results( powder=optimizer.powder, From 86aa79ec1391d0f8d0fbd939c56e6ebe377ad29c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 11 Nov 2024 18:53:44 -0800 Subject: [PATCH 051/375] [edit] Updated message to executor distance to sample --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d21d5903..c782e9dc 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -672,7 +672,7 @@ def _run(self) -> None: self._report_to_executor(msg) distance, cx, cy = get_beam_center(optimizer.params) msg = Message( - contents=f"Detector Distance to Point of Normal Incidence: {distance:.2e}" + contents=f"Detector Distance to Sample: {distance:.2e}" ) self._report_to_executor(msg) msg = Message(contents=f"Beam center: ({cx:.2e}, {cy:.2e})", signal="") From 7a72abd713012eb9853b09964c4286d8439048a4 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Tue, 12 Nov 2024 02:54:09 +0000 Subject: [PATCH 052/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index c782e9dc..195d94e2 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -671,9 +671,7 @@ def _run(self) -> None: msg = Message(contents="Optimization complete", signal="") self._report_to_executor(msg) distance, cx, cy = get_beam_center(optimizer.params) - msg = Message( - contents=f"Detector Distance to Sample: {distance:.2e}" - ) + msg = Message(contents=f"Detector Distance to Sample: {distance:.2e}") self._report_to_executor(msg) msg = Message(contents=f"Beam center: ({cx:.2e}, {cy:.2e})", signal="") self._report_to_executor(msg) From 06e35f51161b98037b5377beeeeadf591038a426 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 12 Nov 2024 17:50:06 -0800 Subject: [PATCH 053/375] [edit] Updated build_pyFAI_detector method and min_intensity bug resolved for rayonix --- lute/tasks/geom_opt.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index c782e9dc..80431ae0 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -14,6 +14,8 @@ from lute.tasks.task import * from lute.tasks.dataclasses import * +import psana + import sys sys.path.append("/sdf/home/l/lconreux/LCLSGeom") @@ -147,11 +149,7 @@ def min_intensity(self, Imin, powder): Minimum intensity to use for control point extraction based on photon energy or max intensity """ if type(Imin) == str: - if "rayonix" in self.det_type: - powder = powder[powder > 1e3] - Imin = np.max(powder) * 0.01 - else: - Imin = np.max(powder) * 0.01 + Imin = np.max(powder) * 0.01 else: Imin = Imin * self.photon_energy self.Imin = Imin @@ -718,7 +716,17 @@ def build_pyFAI_detector(self): """ in_file = self._task_parameters.in_file det_type = self._task_parameters.det_type - psana_to_pyfai = PsanaToPyFAI(in_file=in_file, det_type=det_type) + ds_args = f"exp={self.exp}:run={self.run}:idx" + self.ds = psana.DataSource(ds_args) + self.det = psana.Detector(det_type, self.ds.env()) + self.shape = self.det.shape() + if det_type.lower() == "rayonix": + env = self.ds.env() + cfg = env.configStore() + self.pixel_size = cfg.get(psana.Rayonix.ConfigV2).pixelWidth() + else: + self.pixel_size = self.det.pixel_size(self.ds.env()) + psana_to_pyfai = PsanaToPyFAI(in_file=in_file, det_type=det_type, pixel_size=self.pixel_size, shape=self.shape) detector = psana_to_pyfai.detector return detector From 57594491eadb38d01491c3413e32390af9bf9716 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Wed, 13 Nov 2024 01:51:35 +0000 Subject: [PATCH 054/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index a99ced0e..24bc4e75 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -724,7 +724,12 @@ def build_pyFAI_detector(self): self.pixel_size = cfg.get(psana.Rayonix.ConfigV2).pixelWidth() else: self.pixel_size = self.det.pixel_size(self.ds.env()) - psana_to_pyfai = PsanaToPyFAI(in_file=in_file, det_type=det_type, pixel_size=self.pixel_size, shape=self.shape) + psana_to_pyfai = PsanaToPyFAI( + in_file=in_file, + det_type=det_type, + pixel_size=self.pixel_size, + shape=self.shape, + ) detector = psana_to_pyfai.detector return detector From ba19784f535749c2500c2b0e9e84f48988261377 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 12 Nov 2024 17:54:32 -0800 Subject: [PATCH 055/375] [edit] Fix _task_parameters omission --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index a99ced0e..9100e37e 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -714,7 +714,7 @@ def build_pyFAI_detector(self): """ in_file = self._task_parameters.in_file det_type = self._task_parameters.det_type - ds_args = f"exp={self.exp}:run={self.run}:idx" + ds_args = f"exp={self._task_parameters.exp}:run={self._task_parameters.run}:idx" self.ds = psana.DataSource(ds_args) self.det = psana.Detector(det_type, self.ds.env()) self.shape = self.det.shape() From fae000386a436a170f2dde5394dc4abb124b0188 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 12 Nov 2024 18:27:57 -0800 Subject: [PATCH 056/375] [edit] Fix an omission on pixel_values --- lute/tasks/geom_opt.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 5d036773..78912d34 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -133,6 +133,9 @@ def build_calibrant(self): """ self.calibrant_name = self.calibrant calibrant = CALIBRANT_FACTORY(self.calibrant) + ds_args = f"exp={self.exp}:run={self.run}:idx" + self.ds = psana.DataSource(ds_args) + self.wavelength = self.ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 photon_energy = 1.23984197386209e-09 / self.wavelength self.photon_energy = photon_energy calibrant.wavelength = self.wavelength @@ -751,6 +754,8 @@ def update_geometry(self, optimizer): ), det_type=optimizer.det_type, out_file=self._task_parameters.out_file, + pixel_size=self.pixel_size, + shape=self.shape, ) psana_to_pyfai = PsanaToPyFAI( in_file=self._task_parameters.out_file, From 4ceb88009c207b8b3421f217190eabee459991c1 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Wed, 13 Nov 2024 02:28:23 +0000 Subject: [PATCH 057/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 78912d34..5f80c3ea 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -135,7 +135,9 @@ def build_calibrant(self): calibrant = CALIBRANT_FACTORY(self.calibrant) ds_args = f"exp={self.exp}:run={self.run}:idx" self.ds = psana.DataSource(ds_args) - self.wavelength = self.ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 + self.wavelength = ( + self.ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 + ) photon_energy = 1.23984197386209e-09 / self.wavelength self.photon_energy = photon_energy calibrant.wavelength = self.wavelength From 8d44ac52dc9ddaeb3a8b347d2d2e9f0476a5df62 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 12 Nov 2024 18:30:10 -0800 Subject: [PATCH 058/375] [edit] Fix an omission on pixel_values again in PsanaToPyFAI --- lute/tasks/geom_opt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 78912d34..057c03c5 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -760,6 +760,8 @@ def update_geometry(self, optimizer): psana_to_pyfai = PsanaToPyFAI( in_file=self._task_parameters.out_file, det_type=self._task_parameters.det_type, + pixel_size=self.pixel_size, + shape=self.shape, ) detector = psana_to_pyfai.detector return detector From 73f720b4718951a01b2e7da7498a382ecf416851 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 12 Nov 2024 18:47:02 -0800 Subject: [PATCH 059/375] [edit] Fix scaling to meter of pixel_size again in PsanaToPyFAI --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index f307a0f8..3b45e98d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -726,9 +726,9 @@ def build_pyFAI_detector(self): if det_type.lower() == "rayonix": env = self.ds.env() cfg = env.configStore() - self.pixel_size = cfg.get(psana.Rayonix.ConfigV2).pixelWidth() + self.pixel_size = cfg.get(psana.Rayonix.ConfigV2).pixelWidth() * 1e-6 else: - self.pixel_size = self.det.pixel_size(self.ds.env()) + self.pixel_size = self.det.pixel_size(self.ds.env()) * 1e-6 psana_to_pyfai = PsanaToPyFAI( in_file=in_file, det_type=det_type, From c6a2134bcbcd3e39c8b36547bf99f03e947c22ae Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 08:55:09 -0800 Subject: [PATCH 060/375] [edit] Replace Imin=max/100 by Imin=percentile(90) --- lute/io/models/geom_opt.py | 12 +---- lute/tasks/geom_opt.py | 101 +++++++------------------------------ 2 files changed, 20 insertions(+), 93 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index e31db638..2874a8e2 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -57,11 +57,6 @@ class BayesGeomOptParameters(BaseModel): description="Number of iterations to run the Bayesian optimization.", ) - Imin: Optional[Union[str, int]] = Field( - "max", - description="Minimal Intensity for extracting key control points, either based on heuristics on the maximal intensity, or a multiple of the photon energy", - ) - prior: Optional[bool] = Field( True, description="Whether to use a gaussian prior centered on the search space for the Bayesian optimization or randomly pick samples.", @@ -97,7 +92,7 @@ class BayesGeomOptParameters(BaseModel): det_type: str = Field( "", - description="Detector type. Currently supported: 'ePix10k2M', 'ePix10kaQuad', 'Rayonix', 'Rayonix2', 'Jungfrau1M', 'Jungfrau4M'", + description="Detector type. Currently supported: 'ePix10k2M', 'ePix10kaQuad', 'Rayonix', 'Jungfrau1M', 'Jungfrau4M'", ) date: str = Field( @@ -125,11 +120,6 @@ class BayesGeomOptParameters(BaseModel): description="Calibrant used for the calibration supported by pyFAI: https://github.com/silx-kit/pyFAI/tree/main/src/pyFAI/resources/calibration", ) - wavelength: float = Field( - 1e-10, - description="Wavelength of the X-ray beam in meters.", - ) - out_file: str = Field( "", description="Path to the output .data file containing the optimized detector geometry.", diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 3b45e98d..62d310a0 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -29,7 +29,7 @@ import logging from lute.execution.logging import get_logger -logger = get_logger(__name__) +logger: logging.Logger = get_logger(__name__) import numpy as np import matplotlib.pyplot as plt @@ -61,8 +61,6 @@ class BayesGeomOpt: PyFAI detector object calibrant : str Calibrant name - wavelength : float - Wavelength of the experiment """ def __init__( @@ -72,7 +70,6 @@ def __init__( det_type, detector, calibrant, - wavelength, ): self.exp = exp self.run = run @@ -82,7 +79,6 @@ def __init__( self.size = self.comm.Get_size() self.detector = detector self.calibrant = calibrant - self.wavelength = wavelength self.order = ["dist", "poni1", "poni2", "rot1", "rot2", "rot3"] self.space = ["poni1", "poni2"] self.values = { @@ -134,32 +130,28 @@ def build_calibrant(self): self.calibrant_name = self.calibrant calibrant = CALIBRANT_FACTORY(self.calibrant) ds_args = f"exp={self.exp}:run={self.run}:idx" - self.ds = psana.DataSource(ds_args) + ds = psana.DataSource(ds_args) self.wavelength = ( - self.ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 + ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 ) photon_energy = 1.23984197386209e-09 / self.wavelength self.photon_energy = photon_energy calibrant.wavelength = self.wavelength self.calibrant = calibrant - def min_intensity(self, Imin, powder): + def min_intensity(self, powder): """ Define minimal intensity for control point extraction Note: this is a heuristic that has been found to work well but may need some tuning. Parameters ---------- - Imin : int or str - Minimum intensity to use for control point extraction based on photon energy or max intensity + Imin : float + Minimum intensity to use for control point extraction based on intensity distribution """ - if type(Imin) == str: - Imin = np.max(powder) * 0.01 - else: - Imin = Imin * self.photon_energy + Imin = np.percentile(powder, 90) self.Imin = Imin self.powder = powder - return powder @ignore_warnings(category=ConvergenceWarning) def bayes_opt_center( @@ -206,9 +198,7 @@ def bayes_opt_center( if seed is not None: np.random.seed(seed) - self.values["dist"] = dist - if res is None: res = self.detector.pixel_size @@ -388,7 +378,6 @@ def bayes_opt_geom( powder, bounds, res, - Imin="max", n_samples=50, n_iterations=50, af="ucb", @@ -407,8 +396,6 @@ def bayes_opt_geom( Dictionary of bounds and resolution for search parameters res : float Resolution of the grid used to discretize the parameter search space - Imin : int or str - Minimum intensity to use for control point extraction based on photon energy or max intensity values : dict Dictionary of values for fixed parameters n_samples : int @@ -424,8 +411,6 @@ def bayes_opt_geom( seed : int Random seed for reproducibility """ - from time import time - from datetime import datetime if seed is not None: np.random.seed(seed) @@ -434,19 +419,17 @@ def bayes_opt_geom( self.build_calibrant() - powder = self.min_intensity(Imin, powder) + self.min_intensity(powder) if self.rank == 0: - print(f"Number of distances to scan: {self.size}") + logger.info(f"Number of distances to scan: {self.size}") distances = np.linspace(bounds["dist"][0], bounds["dist"][1], self.size) else: distances = None dist = self.comm.scatter(distances, root=0) - print(f"Rank {self.rank} is working on distance {dist}") + logger.info(f"Rank {self.rank} is working on distance {dist}") - start = datetime.now() - t0 = time() results = self.bayes_opt_center( powder, dist, @@ -459,13 +442,6 @@ def bayes_opt_geom( prior, seed, ) - t1 = time() - # with open( - # f"/sdf/home/l/lconreux/launchpad/bayes_opt_center_rank_{self.rank}.txt", "w" - # ) as f: - # f.write( - # f"Bayesian Optimization on Center with distance {dist:.2f} on Rank {self.rank} started at {start} and took {t1-t0:.2f} seconds" - # ) self.comm.Barrier() self.scan = {} @@ -631,37 +607,23 @@ def __init__( super().__init__(params=params, use_mpi=use_mpi) def _run(self) -> None: - from time import time - from datetime import datetime - - msg = Message(contents="Starting PyFAI geometry optimization", signal="") - self._report_to_executor(msg) - msg = Message(contents="Building PyFAI detector", signal="") - self._report_to_executor(msg) + logger.info("Starting PyFAI geometry optimization") + logger.info("Building PyFAI detector") detector = self.build_pyFAI_detector() rank = MPI.COMM_WORLD.Get_rank() - msg = Message( - contents=f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}", - signal="", - ) - self._report_to_executor(msg) + logger.info(f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}") optimizer = BayesGeomOpt( exp=self._task_parameters.exp, run=self._task_parameters.run, det_type=self._task_parameters.det_type, detector=detector, calibrant=self._task_parameters.calibrant, - wavelength=self._task_parameters.wavelength, ) - msg = Message(contents="Running Bayesian Optimization", signal="") - self._report_to_executor(msg) - start = datetime.now() - t0 = time() + logger.info("Running Bayesian Optimization") optimizer.bayes_opt_geom( powder=self._task_parameters.powder, bounds=self._task_parameters.bo_params.bounds, res=self._task_parameters.bo_params.res, - Imin=self._task_parameters.bo_params.Imin, n_samples=self._task_parameters.bo_params.n_samples, n_iterations=self._task_parameters.bo_params.n_iterations, af=self._task_parameters.bo_params.af, @@ -669,39 +631,14 @@ def _run(self) -> None: prior=self._task_parameters.bo_params.prior, seed=self._task_parameters.bo_params.seed, ) - t1 = time() if optimizer.rank == 0: - msg = Message(contents="Optimization complete", signal="") - self._report_to_executor(msg) + logger.info("Optimization complete") distance, cx, cy = get_beam_center(optimizer.params) - msg = Message(contents=f"Detector Distance to Sample: {distance:.2e}") - self._report_to_executor(msg) - msg = Message(contents=f"Beam center: ({cx:.2e}, {cy:.2e})", signal="") - self._report_to_executor(msg) - msg = Message( - contents=f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})", - signal="", - ) - self._report_to_executor(msg) - msg = Message( - contents=f"Final Residuals: {optimizer.residuals:.2e}", signal="" - ) - self._report_to_executor(msg) + logger.info(f"Detector Distance to Sample: {distance:.2e}") + logger.info(f"Beam center: ({cx:.2e}, {cy:.2e})") + logger.info(f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})") + logger.info(f"Final Residuals: {optimizer.residuals:.2e}") plot = f"{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png" - # with open( - # f"/sdf/home/l/lconreux/launchpad/bayes_opt_geom_rank_{rank}.txt", "w" - # ) as f: - # f.write( - # f"Bayesian Optimization Geometry started at {start} took {t1-t0:.2f} seconds \n" - # ) - # f.write( - # f"Detector Distance to Point of Normal Incidence: {distance:.2e} \n" - # ) - # f.write(f"Beam center: ({cx:.2e}, {cy:.2e}) \n") - # f.write( - # f"Rotations: \u03B8x = {optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e} \n" - # ) - # f.write(f"Final Residuals: {optimizer.residuals:.2e}") detector = self.update_geometry(optimizer) optimizer.visualize_results( powder=optimizer.powder, From 6c90d8cb48a07221aeb867e60d7cf680f5fe823d Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 14 Nov 2024 16:55:34 +0000 Subject: [PATCH 061/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 62d310a0..194cd114 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -131,9 +131,7 @@ def build_calibrant(self): calibrant = CALIBRANT_FACTORY(self.calibrant) ds_args = f"exp={self.exp}:run={self.run}:idx" ds = psana.DataSource(ds_args) - self.wavelength = ( - ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 - ) + self.wavelength = ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 photon_energy = 1.23984197386209e-09 / self.wavelength self.photon_energy = photon_energy calibrant.wavelength = self.wavelength @@ -611,7 +609,9 @@ def _run(self) -> None: logger.info("Building PyFAI detector") detector = self.build_pyFAI_detector() rank = MPI.COMM_WORLD.Get_rank() - logger.info(f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}") + logger.info( + f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}" + ) optimizer = BayesGeomOpt( exp=self._task_parameters.exp, run=self._task_parameters.run, @@ -636,7 +636,9 @@ def _run(self) -> None: distance, cx, cy = get_beam_center(optimizer.params) logger.info(f"Detector Distance to Sample: {distance:.2e}") logger.info(f"Beam center: ({cx:.2e}, {cy:.2e})") - logger.info(f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})") + logger.info( + f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" + ) logger.info(f"Final Residuals: {optimizer.residuals:.2e}") plot = f"{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png" detector = self.update_geometry(optimizer) From 4d675b6fc83fb713555b80a20d803baeaa32bcb3 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 09:43:02 -0800 Subject: [PATCH 062/375] [edit] Remove hot pixels + percentile(95) --- lute/tasks/geom_opt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 62d310a0..5c492bc6 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -149,9 +149,11 @@ def min_intensity(self, powder): Imin : float Minimum intensity to use for control point extraction based on intensity distribution """ - Imin = np.percentile(powder, 90) + powder[powder > 1e4] = 0 + Imin = np.percentile(powder, 95) self.Imin = Imin self.powder = powder + return powder @ignore_warnings(category=ConvergenceWarning) def bayes_opt_center( @@ -419,7 +421,7 @@ def bayes_opt_geom( self.build_calibrant() - self.min_intensity(powder) + powder = self.min_intensity(powder) if self.rank == 0: logger.info(f"Number of distances to scan: {self.size}") From 25bdb2ee41e2ad48731eacea625a4191ae74a86a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 09:51:53 -0800 Subject: [PATCH 063/375] [edit] percentile 98 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 399876d1..097d57e1 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -148,7 +148,7 @@ def min_intensity(self, powder): Minimum intensity to use for control point extraction based on intensity distribution """ powder[powder > 1e4] = 0 - Imin = np.percentile(powder, 95) + Imin = np.percentile(powder, 98) self.Imin = Imin self.powder = powder return powder From c6672ee1dd9324041b012e8bfd15a6b8f4ce11ab Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 09:57:27 -0800 Subject: [PATCH 064/375] [edit] Change max rings to 10 instead of 5 --- lute/tasks/geom_opt.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 097d57e1..a4febb0e 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -89,6 +89,8 @@ def __init__( "rot2": 0, "rot3": 0, } + MAX_RINGS = 10 + @staticmethod def expected_improvement(X, gp_model, best_y, epsilon=0): @@ -261,7 +263,7 @@ def bayes_opt_center( detector=self.detector, geometry=geom_initial, ) - sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + sg.extract_cp(max_rings=self.MAX_RINGS, pts_per_deg=1, Imin=self.Imin) y[i] = len(sg.geometry_refinement.data) bo_history[f"init_sample_{i+1}"] = {"param": X_samples[i], "score": y[i]} @@ -324,7 +326,7 @@ def bayes_opt_center( detector=self.detector, geometry=geom_initial, ) - sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + sg.extract_cp(max_rings=self.MAX_RINGS, pts_per_deg=1, Imin=self.Imin) score = len(sg.geometry_refinement.data) y = np.append(y, [score], axis=0) ypred = gp_model.predict(X_norm, return_std=False) @@ -361,7 +363,7 @@ def bayes_opt_center( detector=self.detector, geometry=geom_initial, ) - sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + sg.extract_cp(max_rings=self.MAX_RINGS, pts_per_deg=1, Imin=self.Imin) self.sg = sg residuals = sg.geometry_refinement.refine3(fix=["wavelength"]) params = sg.geometry_refinement.param @@ -588,7 +590,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): detector=detector, geometry=geometry, ) - sg.extract_cp(max_rings=5, pts_per_deg=1, Imin=self.Imin) + sg.extract_cp(max_rings=self.MAX_RINGS, pts_per_deg=1, Imin=self.Imin) ax3 = plt.subplot2grid( (nrow, ncol), (irow, 0), rowspan=nrow - irow, colspan=ncol ) From 780b880f679bcf6994d4cc2d12f1efece5146025 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 14 Nov 2024 17:57:50 +0000 Subject: [PATCH 065/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index a4febb0e..df950d02 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -91,7 +91,6 @@ def __init__( } MAX_RINGS = 10 - @staticmethod def expected_improvement(X, gp_model, best_y, epsilon=0): y_pred, y_std = gp_model.predict(X, return_std=True) From 8e0011205e7679d3506566a98bfe83ab463a8e48 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 09:59:12 -0800 Subject: [PATCH 066/375] [edit] Set max_rings=5 but percentile(99) --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index a4febb0e..029efe86 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -89,7 +89,7 @@ def __init__( "rot2": 0, "rot3": 0, } - MAX_RINGS = 10 + MAX_RINGS = 5 @staticmethod @@ -150,7 +150,7 @@ def min_intensity(self, powder): Minimum intensity to use for control point extraction based on intensity distribution """ powder[powder > 1e4] = 0 - Imin = np.percentile(powder, 98) + Imin = np.percentile(powder, 99) self.Imin = Imin self.powder = powder return powder From 743a9b76abb850c80f12860a954e38728f99560c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 10:02:12 -0800 Subject: [PATCH 067/375] [edit] Max_rings=10 and percentile(99) --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 5b1f3031..7c974c92 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -89,7 +89,7 @@ def __init__( "rot2": 0, "rot3": 0, } - MAX_RINGS = 5 + self.MAX_RINGS = 10 @staticmethod def expected_improvement(X, gp_model, best_y, epsilon=0): From 6bced5d3131ffa4a37275e7dc911d06d6b687a50 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 10:13:13 -0800 Subject: [PATCH 068/375] [test] Only percentile(99) --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7c974c92..0122e9ed 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -89,7 +89,7 @@ def __init__( "rot2": 0, "rot3": 0, } - self.MAX_RINGS = 10 + self.MAX_RINGS = 5 @staticmethod def expected_improvement(X, gp_model, best_y, epsilon=0): @@ -148,7 +148,7 @@ def min_intensity(self, powder): Imin : float Minimum intensity to use for control point extraction based on intensity distribution """ - powder[powder > 1e4] = 0 + #powder[powder > 1e4] = 0 Imin = np.percentile(powder, 99) self.Imin = Imin self.powder = powder From 4cb07574bfe69b3948975e1a46a0597aa1dccd50 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 14 Nov 2024 18:13:41 +0000 Subject: [PATCH 069/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0122e9ed..7014a18f 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -148,7 +148,7 @@ def min_intensity(self, powder): Imin : float Minimum intensity to use for control point extraction based on intensity distribution """ - #powder[powder > 1e4] = 0 + # powder[powder > 1e4] = 0 Imin = np.percentile(powder, 99) self.Imin = Imin self.powder = powder From 5edfaca9e260f07c3f82b9f7df7f4f5987a2934e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 10:20:25 -0800 Subject: [PATCH 070/375] [test] max_rings=10 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0122e9ed..664a0a1b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -89,7 +89,7 @@ def __init__( "rot2": 0, "rot3": 0, } - self.MAX_RINGS = 5 + self.MAX_RINGS = 10 @staticmethod def expected_improvement(X, gp_model, best_y, epsilon=0): From d61026f40003a2217127aed2c6bc5c02300e2794 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 10:28:32 -0800 Subject: [PATCH 071/375] [test] Add thresholding filter with thresh=1e4 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 377b0b51..7c974c92 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -148,7 +148,7 @@ def min_intensity(self, powder): Imin : float Minimum intensity to use for control point extraction based on intensity distribution """ - # powder[powder > 1e4] = 0 + powder[powder > 1e4] = 0 Imin = np.percentile(powder, 99) self.Imin = Imin self.powder = powder From 7043e12409867802fe1a46bafccebec277d8a4b6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 11:07:04 -0800 Subject: [PATCH 072/375] [edit] Add max_rings and Imin as args --- lute/io/models/geom_opt.py | 10 ++++++++++ lute/tasks/geom_opt.py | 37 +++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 2874a8e2..aee87b97 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -47,6 +47,16 @@ class BayesGeomOptParameters(BaseModel): description="Resolution of the grid used to discretize the parameter search space.", ) + Imin: float = Field( + 90, + description="Minimum intensity threshold for the Bayesian optimization based on intensity distribution percentile.", + ) + + max_rings: int = Field( + 5, + description="Maximum number of rings to be used for the Bayesian optimization.", + ) + n_samples: Optional[int] = Field( 50, description="Number of random starts to initialize the Bayesian optimization.", diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7c974c92..3dd20dc3 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -89,7 +89,6 @@ def __init__( "rot2": 0, "rot3": 0, } - self.MAX_RINGS = 10 @staticmethod def expected_improvement(X, gp_model, best_y, epsilon=0): @@ -138,7 +137,7 @@ def build_calibrant(self): calibrant.wavelength = self.wavelength self.calibrant = calibrant - def min_intensity(self, powder): + def min_intensity(self, Imin, powder): """ Define minimal intensity for control point extraction Note: this is a heuristic that has been found to work well but may need some tuning. @@ -147,12 +146,14 @@ def min_intensity(self, powder): ---------- Imin : float Minimum intensity to use for control point extraction based on intensity distribution + powder : np.ndarray + Powder image """ powder[powder > 1e4] = 0 - Imin = np.percentile(powder, 99) + Imin = np.percentile(powder, Imin) self.Imin = Imin self.powder = powder - return powder + return Imin, powder @ignore_warnings(category=ConvergenceWarning) def bayes_opt_center( @@ -161,6 +162,8 @@ def bayes_opt_center( dist, bounds, res, + Imin=90, + max_rings=5, n_samples=50, n_iterations=50, af="ucb", @@ -181,8 +184,10 @@ def bayes_opt_center( Dictionary of bounds for each parameter res : float Resolution of the grid used to discretize the parameter search space - values : dict - Dictionary of values for fixed parameters + Imin : float + Minimum intensity threshold for control point extraction based on intensity distribution percentile + max_rings : int + Maximum number of rings to use for control point extraction n_samples : int Number of samples to initialize the GP model n_iterations : int @@ -262,7 +267,7 @@ def bayes_opt_center( detector=self.detector, geometry=geom_initial, ) - sg.extract_cp(max_rings=self.MAX_RINGS, pts_per_deg=1, Imin=self.Imin) + sg.extract_cp(max_rings=max_rings, pts_per_deg=1, Imin=Imin) y[i] = len(sg.geometry_refinement.data) bo_history[f"init_sample_{i+1}"] = {"param": X_samples[i], "score": y[i]} @@ -325,7 +330,7 @@ def bayes_opt_center( detector=self.detector, geometry=geom_initial, ) - sg.extract_cp(max_rings=self.MAX_RINGS, pts_per_deg=1, Imin=self.Imin) + sg.extract_cp(max_rings=max_rings, pts_per_deg=1, Imin=Imin) score = len(sg.geometry_refinement.data) y = np.append(y, [score], axis=0) ypred = gp_model.predict(X_norm, return_std=False) @@ -362,7 +367,7 @@ def bayes_opt_center( detector=self.detector, geometry=geom_initial, ) - sg.extract_cp(max_rings=self.MAX_RINGS, pts_per_deg=1, Imin=self.Imin) + sg.extract_cp(max_rings=max_rings, pts_per_deg=1, Imin=Imin) self.sg = sg residuals = sg.geometry_refinement.refine3(fix=["wavelength"]) params = sg.geometry_refinement.param @@ -379,6 +384,8 @@ def bayes_opt_geom( powder, bounds, res, + Imin=90, + max_rings=5, n_samples=50, n_iterations=50, af="ucb", @@ -397,8 +404,10 @@ def bayes_opt_geom( Dictionary of bounds and resolution for search parameters res : float Resolution of the grid used to discretize the parameter search space - values : dict - Dictionary of values for fixed parameters + Imin : float + Minimum intensity threshold for control point extraction based on intensity distribution percentile + max_rings : int + Maximum number of rings to use for control point extraction n_samples : int Number of samples to initialize the GP model n_iterations : int @@ -420,7 +429,7 @@ def bayes_opt_geom( self.build_calibrant() - powder = self.min_intensity(powder) + Imin, powder = self.min_intensity(Imin, powder) if self.rank == 0: logger.info(f"Number of distances to scan: {self.size}") @@ -436,6 +445,8 @@ def bayes_opt_geom( dist, bounds, res, + Imin, + max_rings, n_samples, n_iterations, af, @@ -627,6 +638,8 @@ def _run(self) -> None: powder=self._task_parameters.powder, bounds=self._task_parameters.bo_params.bounds, res=self._task_parameters.bo_params.res, + Imin=self._task_parameters.bo_params.Imin, + max_rings=self._task_parameters.bo_params.max_rings, n_samples=self._task_parameters.bo_params.n_samples, n_iterations=self._task_parameters.bo_params.n_iterations, af=self._task_parameters.bo_params.af, From c7af1380a37821572b151ef8b1f57de705996b68 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 11:18:54 -0800 Subject: [PATCH 073/375] [edit] fix an omission --- lute/tasks/geom_opt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 3dd20dc3..a3c34f1e 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -429,6 +429,7 @@ def bayes_opt_geom( self.build_calibrant() + self.max_rings = max_rings Imin, powder = self.min_intensity(Imin, powder) if self.rank == 0: @@ -600,7 +601,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): detector=detector, geometry=geometry, ) - sg.extract_cp(max_rings=self.MAX_RINGS, pts_per_deg=1, Imin=self.Imin) + sg.extract_cp(max_rings=self.max_rings, pts_per_deg=1, Imin=self.Imin) ax3 = plt.subplot2grid( (nrow, ncol), (irow, 0), rowspan=nrow - irow, colspan=ncol ) From ac39a56269b17063139885f335f1b8953b8986f0 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 14 Nov 2024 15:41:36 -0800 Subject: [PATCH 074/375] [edit] Preprocess powder + set up default Imin to 99-percentile --- lute/io/models/geom_opt.py | 2 +- lute/tasks/geom_opt.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index aee87b97..b98d9de4 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -48,7 +48,7 @@ class BayesGeomOptParameters(BaseModel): ) Imin: float = Field( - 90, + 99, description="Minimum intensity threshold for the Bayesian optimization based on intensity distribution percentile.", ) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index a3c34f1e..8c48fc1d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -149,8 +149,12 @@ def min_intensity(self, Imin, powder): powder : np.ndarray Powder image """ - powder[powder > 1e4] = 0 - Imin = np.percentile(powder, Imin) + mean = np.mean(powder) + std = np.std(powder) + threshold = mean + 2 * std + nice_pix = powder < threshold + Imin = np.percentile(powder[nice_pix], Imin) + powder[nice_pix] = 0 self.Imin = Imin self.powder = powder return Imin, powder From 7d31b6452b3c4f403f529a336e535d12e39df29f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 14:04:24 -0800 Subject: [PATCH 075/375] [edit] remove powder[nice_pix]=0 --- lute/tasks/geom_opt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8c48fc1d..0e4d7ef3 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -151,10 +151,9 @@ def min_intensity(self, Imin, powder): """ mean = np.mean(powder) std = np.std(powder) - threshold = mean + 2 * std + threshold = mean + 3 * std nice_pix = powder < threshold Imin = np.percentile(powder[nice_pix], Imin) - powder[nice_pix] = 0 self.Imin = Imin self.powder = powder return Imin, powder From 763e2be3b9a485cd53b7c2a315af23b9a02c72f1 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 14:33:25 -0800 Subject: [PATCH 076/375] [edit] Fix display --- lute/tasks/geom_opt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0e4d7ef3..d0f9249e 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -496,8 +496,9 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ai = sg.geometry_refinement if label is None: label = sg.label - ax.imshow(powder.T, origin="lower", cmap="viridis", vmax=self.Imin) - ax.set_title(label) + ax.imshow(powder.T, origin="lower", cmap="viridis", vmin=np.min(powder), vmax=self.Imin) + cbar = plt.colorbar(ax=ax, orientation='vertical') + cbar.set_label('Intensity') if ai is not None and cp.calibrant is not None: tth = cp.calibrant.get_2th() ttha = ai.twoThetaArray() From 4f7f7ce5e3c13de3b08518aa1ab5d7249f48df00 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 15 Nov 2024 22:33:53 +0000 Subject: [PATCH 077/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d0f9249e..18dc56f1 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -496,9 +496,15 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ai = sg.geometry_refinement if label is None: label = sg.label - ax.imshow(powder.T, origin="lower", cmap="viridis", vmin=np.min(powder), vmax=self.Imin) - cbar = plt.colorbar(ax=ax, orientation='vertical') - cbar.set_label('Intensity') + ax.imshow( + powder.T, + origin="lower", + cmap="viridis", + vmin=np.min(powder), + vmax=self.Imin, + ) + cbar = plt.colorbar(ax=ax, orientation="vertical") + cbar.set_label("Intensity") if ai is not None and cp.calibrant is not None: tth = cp.calibrant.get_2th() ttha = ai.twoThetaArray() From 1e76b4f04bbf216f24d2740aa22339a431332499 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 14:50:52 -0800 Subject: [PATCH 078/375] [edit] Fix display --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d0f9249e..fc16c2db 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -496,8 +496,8 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ai = sg.geometry_refinement if label is None: label = sg.label - ax.imshow(powder.T, origin="lower", cmap="viridis", vmin=np.min(powder), vmax=self.Imin) - cbar = plt.colorbar(ax=ax, orientation='vertical') + img = ax.imshow(powder.T, origin="lower", cmap="viridis", vmin=np.min(powder), vmax=self.Imin) + cbar = plt.colorbar(img, ax=ax, orientation='vertical') cbar.set_label('Intensity') if ai is not None and cp.calibrant is not None: tth = cp.calibrant.get_2th() From a09efb6eaf918ac6422f03a9560b1223799d8939 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 15 Nov 2024 22:52:32 +0000 Subject: [PATCH 079/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index fc16c2db..1c1fc1ca 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -496,9 +496,15 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ai = sg.geometry_refinement if label is None: label = sg.label - img = ax.imshow(powder.T, origin="lower", cmap="viridis", vmin=np.min(powder), vmax=self.Imin) - cbar = plt.colorbar(img, ax=ax, orientation='vertical') - cbar.set_label('Intensity') + img = ax.imshow( + powder.T, + origin="lower", + cmap="viridis", + vmin=np.min(powder), + vmax=self.Imin, + ) + cbar = plt.colorbar(img, ax=ax, orientation="vertical") + cbar.set_label("Intensity") if ai is not None and cp.calibrant is not None: tth = cp.calibrant.get_2th() ttha = ai.twoThetaArray() From 4547e148c1dfa93168c1138eb19ac288d3613532 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 15:24:56 -0800 Subject: [PATCH 080/375] [edit] Fix plotting powder intensity colorbar scale --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index fc16c2db..ee12c667 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -496,7 +496,7 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ai = sg.geometry_refinement if label is None: label = sg.label - img = ax.imshow(powder.T, origin="lower", cmap="viridis", vmin=np.min(powder), vmax=self.Imin) + img = ax.imshow(powder.T, origin="lower", cmap="viridis", vmin=np.percentile(powder, 5), vmax=self.Imin) cbar = plt.colorbar(img, ax=ax, orientation='vertical') cbar.set_label('Intensity') if ai is not None and cp.calibrant is not None: From e44ed18baebbd51dd55e57645ad3507ebeafa915 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 15 Nov 2024 23:25:52 +0000 Subject: [PATCH 081/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index ee12c667..4de93c5f 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -496,9 +496,15 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ai = sg.geometry_refinement if label is None: label = sg.label - img = ax.imshow(powder.T, origin="lower", cmap="viridis", vmin=np.percentile(powder, 5), vmax=self.Imin) - cbar = plt.colorbar(img, ax=ax, orientation='vertical') - cbar.set_label('Intensity') + img = ax.imshow( + powder.T, + origin="lower", + cmap="viridis", + vmin=np.percentile(powder, 5), + vmax=self.Imin, + ) + cbar = plt.colorbar(img, ax=ax, orientation="vertical") + cbar.set_label("Intensity") if ai is not None and cp.calibrant is not None: tth = cp.calibrant.get_2th() ttha = ai.twoThetaArray() From f79df97a4fc5bced1df61158ec32f9da446adc2e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 15:26:58 -0800 Subject: [PATCH 082/375] [edit] Set default max_rings to 10 --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index b98d9de4..b0a46b0f 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -53,7 +53,7 @@ class BayesGeomOptParameters(BaseModel): ) max_rings: int = Field( - 5, + 10, description="Maximum number of rings to be used for the Bayesian optimization.", ) From a98f152b23f93f9d2ac137d60a80cada90348428 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 15:51:13 -0800 Subject: [PATCH 083/375] [edit] Set vmin=np.percentile(powder, 25) --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 4de93c5f..1fb69e61 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -500,7 +500,7 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): powder.T, origin="lower", cmap="viridis", - vmin=np.percentile(powder, 5), + vmin=np.percentile(powder, 25), vmax=self.Imin, ) cbar = plt.colorbar(img, ax=ax, orientation="vertical") From 8367369a76e71f386d92930d130b86f6a8f8eb1a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 15:57:50 -0800 Subject: [PATCH 084/375] [edit] Compute a custom score for exploring large distance range --- lute/tasks/geom_opt.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 1fb69e61..bcbd7498 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -373,12 +373,14 @@ def bayes_opt_center( sg.extract_cp(max_rings=max_rings, pts_per_deg=1, Imin=Imin) self.sg = sg residuals = sg.geometry_refinement.refine3(fix=["wavelength"]) + score = len(sg.geometry_refinement.data) * np.log(residuals) params = sg.geometry_refinement.param result = { "bo_history": bo_history, "params": params, "residuals": residuals, "best_idx": best_idx, + "score": score, } return result @@ -465,17 +467,19 @@ def bayes_opt_geom( self.scan["params"] = self.comm.gather(results["params"], root=0) self.scan["residuals"] = self.comm.gather(results["residuals"], root=0) self.scan["best_idx"] = self.comm.gather(results["best_idx"], root=0) + self.scan["score"] = self.comm.gather(results["score"], root=0) self.finalize() def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - index = np.argmin(self.scan["residuals"]) + index = np.argmin(self.scan["score"]) self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] self.residuals = self.scan["residuals"][index] self.best_idx = self.scan["best_idx"][index] + self.score = self.scan["score"][index] def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): """ From 80572f11d83bed559468914ca4ef66c6c2125f2b Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 16:08:37 -0800 Subject: [PATCH 085/375] [edit] Change back to residuals --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index bcbd7498..7ed6d2fa 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -474,7 +474,7 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - index = np.argmin(self.scan["score"]) + index = np.argmin(self.scan["residuals"]) self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] self.residuals = self.scan["residuals"][index] From e7d8dd6f6d47986fcbd75db8987410fc2cb9568f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 16:35:08 -0800 Subject: [PATCH 086/375] [edit] Trying a score based on two stats --- lute/tasks/geom_opt.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7ed6d2fa..32b7d21c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -372,15 +372,15 @@ def bayes_opt_center( ) sg.extract_cp(max_rings=max_rings, pts_per_deg=1, Imin=Imin) self.sg = sg - residuals = sg.geometry_refinement.refine3(fix=["wavelength"]) - score = len(sg.geometry_refinement.data) * np.log(residuals) + residual = sg.geometry_refinement.refine3(fix=["wavelength"]) + score = len(sg.geometry_refinement.data) params = sg.geometry_refinement.param result = { "bo_history": bo_history, "params": params, - "residuals": residuals, - "best_idx": best_idx, + "residual": residual, "score": score, + "best_idx": best_idx, } return result @@ -465,21 +465,25 @@ def bayes_opt_geom( self.scan = {} self.scan["bo_history"] = self.comm.gather(results["bo_history"], root=0) self.scan["params"] = self.comm.gather(results["params"], root=0) - self.scan["residuals"] = self.comm.gather(results["residuals"], root=0) - self.scan["best_idx"] = self.comm.gather(results["best_idx"], root=0) + self.scan["residual"] = self.comm.gather(results["residual"], root=0) self.scan["score"] = self.comm.gather(results["score"], root=0) + self.scan["best_idx"] = self.comm.gather(results["best_idx"], root=0) self.finalize() def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - index = np.argmin(self.scan["residuals"]) + norm_residuals = (self.scan["residual"] - np.min(self.scan["residual"]))/(np.max(self.scan["residual"])-np.min(self.scan["residual"])) + norm_score = (self.scan["score"] - np.min(self.scan["score"]))/(np.max(self.scan["score"])-np.min(self.scan["score"])) + lb = 0.5 + final_score = lb * norm_residuals + (1 - lb) * norm_score + index = np.argmin(final_score) self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] - self.residuals = self.scan["residuals"][index] - self.best_idx = self.scan["best_idx"][index] + self.residual = self.scan["residual"][index] self.score = self.scan["score"][index] + self.best_idx = self.scan["best_idx"][index] def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): """ @@ -504,7 +508,7 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): powder.T, origin="lower", cmap="viridis", - vmin=np.percentile(powder, 25), + vmin=np.mean(powder), vmax=self.Imin, ) cbar = plt.colorbar(img, ax=ax, orientation="vertical") @@ -594,7 +598,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): ax1.set_xticks(np.arange(len(scores), step=20)) ax1.set_xlabel("Iteration") ax1.set_ylabel("Best score so far") - ax1.set_title("Convergence Plot") + ax1.set_title(f"Convergence Plot, best score: {int(self.score)}") icol += 1 # Plotting radial profiles with peaks @@ -670,7 +674,7 @@ def _run(self) -> None: logger.info( f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" ) - logger.info(f"Final Residuals: {optimizer.residuals:.2e}") + logger.info(f"Final Residuals: {optimizer.residual:.2e}") plot = f"{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png" detector = self.update_geometry(optimizer) optimizer.visualize_results( From cdf8f9a16beee01bb91965ec14c9c453e5d3ae7e Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sat, 16 Nov 2024 00:35:36 +0000 Subject: [PATCH 087/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 32b7d21c..66af5265 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -474,8 +474,12 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - norm_residuals = (self.scan["residual"] - np.min(self.scan["residual"]))/(np.max(self.scan["residual"])-np.min(self.scan["residual"])) - norm_score = (self.scan["score"] - np.min(self.scan["score"]))/(np.max(self.scan["score"])-np.min(self.scan["score"])) + norm_residuals = (self.scan["residual"] - np.min(self.scan["residual"])) / ( + np.max(self.scan["residual"]) - np.min(self.scan["residual"]) + ) + norm_score = (self.scan["score"] - np.min(self.scan["score"])) / ( + np.max(self.scan["score"]) - np.min(self.scan["score"]) + ) lb = 0.5 final_score = lb * norm_residuals + (1 - lb) * norm_score index = np.argmin(final_score) From 675f46adae9dd5764e50c2a57abb0c79828ada6f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 16:37:03 -0800 Subject: [PATCH 088/375] [edit] Trying a score based on two stats --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 32b7d21c..8b5eea29 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -477,7 +477,7 @@ def finalize(self): norm_residuals = (self.scan["residual"] - np.min(self.scan["residual"]))/(np.max(self.scan["residual"])-np.min(self.scan["residual"])) norm_score = (self.scan["score"] - np.min(self.scan["score"]))/(np.max(self.scan["score"])-np.min(self.scan["score"])) lb = 0.5 - final_score = lb * norm_residuals + (1 - lb) * norm_score + final_score = lb * norm_residuals - (1 - lb) * norm_score index = np.argmin(final_score) self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] From 5e54a8a81626f81fbc39393b438f4f4c3c3de29f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 16:59:19 -0800 Subject: [PATCH 089/375] [edit] Add diagnostic plots and prints --- lute/tasks/geom_opt.py | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 9b9cb6ae..062cc427 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -474,15 +474,11 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - norm_residuals = (self.scan["residual"] - np.min(self.scan["residual"])) / ( - np.max(self.scan["residual"]) - np.min(self.scan["residual"]) - ) - norm_score = (self.scan["score"] - np.min(self.scan["score"])) / ( - np.max(self.scan["score"]) - np.min(self.scan["score"]) - ) - lb = 0.5 - final_score = lb * norm_residuals - (1 - lb) * norm_score - index = np.argmin(final_score) + logger.info(f"Mean Score: {np.mean(self.scan['score'])}") + logger.info(f"STD Score: {np.std(self.scan['score'])}") + logger.info(f"5% Percentile Score: {np.percentile(self.scan['score'], 5)}") + logger.info(f"95% Percentile Score: {np.percentile(self.scan['score'], 95)}") + index = np.argmin(self.scan["residual"]) self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] self.residual = self.scan["residual"][index] @@ -512,8 +508,8 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): powder.T, origin="lower", cmap="viridis", - vmin=np.mean(powder), - vmax=self.Imin, + vmin=np.percentile(powder, 5), + vmax=np.percentile(powder, 95), ) cbar = plt.colorbar(img, ax=ax, orientation="vertical") cbar.set_label("Intensity") @@ -591,8 +587,8 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): plot : str Path to save plot """ - fig = plt.figure(figsize=(8, 8), dpi=120) - nrow, ncol = 2, 2 + fig = plt.figure(figsize=(12, 8), dpi=180) + nrow, ncol = 2, 3 irow, icol = 0, 0 # Plotting BO convergence @@ -605,13 +601,23 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): ax1.set_title(f"Convergence Plot, best score: {int(self.score)}") icol += 1 + # Plotting scores vs distance + ax2 = plt.subplot2grid((nrow, ncol), (irow, icol)) + scores = self.scan["score"] + ax2.plot(scores) + ax2.set_xticks(np.arange(len(scores), step=20)) + ax2.set_xlabel("Distance index") + ax2.set_ylabel("Score") + ax2.set_title("Number of Control Points vs Distance") + icol += 1 + # Plotting radial profiles with peaks - ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) + ax3 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) ai = AzimuthalIntegrator( dist=params[0], detector=detector, wavelength=self.calibrant.wavelength ) res = ai.integrate1d(powder, 1000) - self.radial_integration(res, calibrant=self.calibrant, ax=ax2) + self.radial_integration(res, calibrant=self.calibrant, ax=ax3) irow += 1 # Plotting stacked powder @@ -624,13 +630,13 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): geometry=geometry, ) sg.extract_cp(max_rings=self.max_rings, pts_per_deg=1, Imin=self.Imin) - ax3 = plt.subplot2grid( + ax4 = plt.subplot2grid( (nrow, ncol), (irow, 0), rowspan=nrow - irow, colspan=ncol ) - self.display(sg=sg, ax=ax3) + self.display(sg=sg, ax=ax4) if plot != "": - fig.savefig(plot, dpi=300) + fig.savefig(plot, dpi=180) class OptimizePyFAIGeometry(Task): From 3c1d527531f19fd74b17d3c264fbfef4d5f4123d Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sat, 16 Nov 2024 00:59:47 +0000 Subject: [PATCH 090/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 062cc427..4e4da0a1 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -477,7 +477,9 @@ def finalize(self): logger.info(f"Mean Score: {np.mean(self.scan['score'])}") logger.info(f"STD Score: {np.std(self.scan['score'])}") logger.info(f"5% Percentile Score: {np.percentile(self.scan['score'], 5)}") - logger.info(f"95% Percentile Score: {np.percentile(self.scan['score'], 95)}") + logger.info( + f"95% Percentile Score: {np.percentile(self.scan['score'], 95)}" + ) index = np.argmin(self.scan["residual"]) self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] From 5eeab95f1d64cfa76b8ba05717b12dbf4518f1cf Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 17:04:36 -0800 Subject: [PATCH 091/375] [edit] Clean logs out of rank-dependent prints --- lute/tasks/geom_opt.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 062cc427..6240ef54 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -444,7 +444,6 @@ def bayes_opt_geom( distances = None dist = self.comm.scatter(distances, root=0) - logger.info(f"Rank {self.rank} is working on distance {dist}") results = self.bayes_opt_center( powder, @@ -648,13 +647,7 @@ def __init__( super().__init__(params=params, use_mpi=use_mpi) def _run(self) -> None: - logger.info("Starting PyFAI geometry optimization") - logger.info("Building PyFAI detector") detector = self.build_pyFAI_detector() - rank = MPI.COMM_WORLD.Get_rank() - logger.info( - f"Setting up Bayesian Optimization for {self._task_parameters.exp} run {self._task_parameters.run} on {self._task_parameters.det_type} on rank {rank}" - ) optimizer = BayesGeomOpt( exp=self._task_parameters.exp, run=self._task_parameters.run, @@ -662,7 +655,6 @@ def _run(self) -> None: detector=detector, calibrant=self._task_parameters.calibrant, ) - logger.info("Running Bayesian Optimization") optimizer.bayes_opt_geom( powder=self._task_parameters.powder, bounds=self._task_parameters.bo_params.bounds, From 2ef1c10402400d0ea2e994ad5c73724dc1e19a26 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 17:44:23 -0800 Subject: [PATCH 092/375] [edit] Add diagnostics plots --- lute/tasks/geom_opt.py | 51 +++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8e4b91a3..cf62057d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -571,6 +571,22 @@ def radial_integration(self, result, calibrant=None, label=None, ax=None): ax.set_xlabel(unit.label) ax.set_ylabel("Intensity") + def distance_scan(self, ax): + scores = self.scan["score"] + ax.plot(scores) + ax.set_xticks(np.arange(len(scores), step=20)) + ax.set_xlabel("Distance index") + ax.set_ylabel("Score") + ax.set_title("Number of Control Points vs Distance") + + def residual_scan(self, ax): + residuals = self.scan["residual"] + ax.plot(residuals) + ax.set_xticks(np.arange(len(residuals), step=20)) + ax.set_xlabel("Distance index") + ax.set_ylabel("Residual") + ax.set_title("Residual vs Distance") + def visualize_results(self, powder, bo_history, detector, params, plot=""): """ Visualize fit, plotting (1) the BO convergence, (2) the radial profile and (3) the powder image. @@ -588,8 +604,8 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): plot : str Path to save plot """ - fig = plt.figure(figsize=(12, 8), dpi=180) - nrow, ncol = 2, 3 + fig = plt.figure(figsize=(8, 12), dpi=180) + nrow, ncol = 3, 2 irow, icol = 0, 0 # Plotting BO convergence @@ -602,26 +618,17 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): ax1.set_title(f"Convergence Plot, best score: {int(self.score)}") icol += 1 - # Plotting scores vs distance - ax2 = plt.subplot2grid((nrow, ncol), (irow, icol)) - scores = self.scan["score"] - ax2.plot(scores) - ax2.set_xticks(np.arange(len(scores), step=20)) - ax2.set_xlabel("Distance index") - ax2.set_ylabel("Score") - ax2.set_title("Number of Control Points vs Distance") - icol += 1 - # Plotting radial profiles with peaks - ax3 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) + ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) ai = AzimuthalIntegrator( dist=params[0], detector=detector, wavelength=self.calibrant.wavelength ) res = ai.integrate1d(powder, 1000) - self.radial_integration(res, calibrant=self.calibrant, ax=ax3) + self.radial_integration(res, calibrant=self.calibrant, ax=ax2) irow += 1 # Plotting stacked powder + ax3 = plt.subplot2grid((nrow, ncol), (irow, 0), colspan=ncol) geometry = Geometry(dist=params[0]) sg = SingleGeometry( f"Max {self.calibrant_name}", @@ -631,10 +638,18 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): geometry=geometry, ) sg.extract_cp(max_rings=self.max_rings, pts_per_deg=1, Imin=self.Imin) - ax4 = plt.subplot2grid( - (nrow, ncol), (irow, 0), rowspan=nrow - irow, colspan=ncol - ) - self.display(sg=sg, ax=ax4) + self.display(sg=sg, ax=ax3) + irow += 1 + icol = 0 + + # Plotting score scan over distance + ax4 = plt.subplot2grid((nrow, ncol), (irow, icol)) + self.distance_scan(ax=ax4) + icol += 1 + + # Plotting residual scan over distance + ax5 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) + self.residual_scan(ax=ax5) if plot != "": fig.savefig(plot, dpi=180) From 7ad687c6c13a32b5ad2bec3426bde0338b874775 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 19:21:29 -0800 Subject: [PATCH 093/375] [edit] Final edit for plotting diagnostics --- lute/tasks/geom_opt.py | 95 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index cf62057d..915a610c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -439,6 +439,7 @@ def bayes_opt_geom( if self.rank == 0: logger.info(f"Number of distances to scan: {self.size}") + self.bounds = bounds distances = np.linspace(bounds["dist"][0], bounds["dist"][1], self.size) else: distances = None @@ -473,12 +474,6 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - logger.info(f"Mean Score: {np.mean(self.scan['score'])}") - logger.info(f"STD Score: {np.std(self.scan['score'])}") - logger.info(f"5% Percentile Score: {np.percentile(self.scan['score'], 5)}") - logger.info( - f"95% Percentile Score: {np.percentile(self.scan['score'], 95)}" - ) index = np.argmin(self.scan["residual"]) self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] @@ -571,22 +566,82 @@ def radial_integration(self, result, calibrant=None, label=None, ax=None): ax.set_xlabel(unit.label) ax.set_ylabel("Intensity") - def distance_scan(self, ax): + def score_distance_scan(self, bounds, ax): + """ + Plot the score scan over distance + + Parameters + ---------- + bounds : dict + Dictionary of bounds for each parameter + ax : plt.Axes + Matplotlib axes + """ scores = self.scan["score"] - ax.plot(scores) + distances = np.linspace(bounds['dist'][0], bounds['dist'][1], len(scores)) + ax.plot(distances, scores) ax.set_xticks(np.arange(len(scores), step=20)) + ax.set_yticks(scores[::100]) + ax.set_yticklabels([f"{v/1000:.1f}k" for v in scores[::100]]) ax.set_xlabel("Distance index") ax.set_ylabel("Score") ax.set_title("Number of Control Points vs Distance") - def residual_scan(self, ax): + def residual_distance_scan(self, bounds, ax): + """ + Plot the residual scan over distance + + Parameters + ---------- + bounds : dict + Dictionary of bounds for each parameter + ax : plt.Axes + Matplotlib axes + """ residuals = self.scan["residual"] - ax.plot(residuals) + distances = np.linspace(bounds['dist'][0], bounds['dist'][1], len(residuals)) + ax.plot(distances, residuals) + best_dist = distances[np.argmin(residuals)] + ax.axvline(best_dist, color="red", linestyle="--", label=f"Best distance: {best_dist:.2e}") ax.set_xticks(np.arange(len(residuals), step=20)) + ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) ax.set_xlabel("Distance index") ax.set_ylabel("Residual") ax.set_title("Residual vs Distance") + def hist_and_compute_stats(self, powder, exp, run, ax): + """ + Plot histogram of pixel intensities and compute statistics + + Parameters + ---------- + powder : np.ndarray + Powder image + exp : str + Experiment name + run : int + Run number + ax : plt.Axes + Matplotlib axes + """ + mean = np.mean(powder) + threshold = np.mean(powder)+3*np.std(powder) + nice_pix = powder < threshold + mean = np.mean(powder[nice_pix]) + std_dev = np.std(powder[nice_pix]) + percentile_99 = np.percentile(powder[nice_pix], 99) + _ = ax.hist(powder[nice_pix], bins=1000, color='skyblue', edgecolor='black', alpha=0.7, label="Pixel Intensities") + ax.axvline(mean, color='red', linestyle='--', linewidth=1.5, label=f'Mean ({mean:.2f})') + ax.axvline(mean + std_dev, color='orange', linestyle='--', linewidth=1.5, label=f'Mean + 1 Std ({mean + std_dev:.2f})') + ax.axvline(mean + 2 * std_dev, color='green', linestyle='--', linewidth=1.5, label=f'Mean + 2 Std ({mean + 2 * std_dev:.2f})') + ax.axvline(mean + 3 * std_dev, color='blue', linestyle='--', linewidth=1.5, label=f'Mean + 3 Std ({mean + 3 * std_dev:.2f})') + ax.axvline(percentile_99, color='purple', linestyle=':', linewidth=1.5, label=f'99th Percentile ({percentile_99:.2f})') + ax.set_xlim(0, mean + 5 * std_dev) + ax.set_xlabel('Pixel Intensity') + ax.set_ylabel('Frequency') + ax.set_title(f'Histogram of Pixel Intensities with Statistical Thresholds for {exp} run {run}') + ax.legend() + def visualize_results(self, powder, bo_history, detector, params, plot=""): """ Visualize fit, plotting (1) the BO convergence, (2) the radial profile and (3) the powder image. @@ -618,17 +673,23 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): ax1.set_title(f"Convergence Plot, best score: {int(self.score)}") icol += 1 - # Plotting radial profiles with peaks + # Plotting histogram of pixel intensities ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) + self.hist_and_compute_stats(powder, self.exp, self.run, ax2) + irow += 1 + icol = 0 + + # Plotting radial profiles with peaks + ax3 = plt.subplot2grid((nrow, ncol), (irow, icol)) ai = AzimuthalIntegrator( dist=params[0], detector=detector, wavelength=self.calibrant.wavelength ) res = ai.integrate1d(powder, 1000) - self.radial_integration(res, calibrant=self.calibrant, ax=ax2) - irow += 1 + self.radial_integration(res, calibrant=self.calibrant, ax=ax3) + icol += 1 # Plotting stacked powder - ax3 = plt.subplot2grid((nrow, ncol), (irow, 0), colspan=ncol) + ax4 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) geometry = Geometry(dist=params[0]) sg = SingleGeometry( f"Max {self.calibrant_name}", @@ -638,18 +699,18 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): geometry=geometry, ) sg.extract_cp(max_rings=self.max_rings, pts_per_deg=1, Imin=self.Imin) - self.display(sg=sg, ax=ax3) + self.display(sg=sg, ax=ax4) irow += 1 icol = 0 # Plotting score scan over distance ax4 = plt.subplot2grid((nrow, ncol), (irow, icol)) - self.distance_scan(ax=ax4) + self.score_distance_scan(self.bounds, ax4) icol += 1 # Plotting residual scan over distance ax5 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - self.residual_scan(ax=ax5) + self.residual_distance_scan(self.bounds, ax5) if plot != "": fig.savefig(plot, dpi=180) From 026f4599cef1d6ed86790dfcdc1e64c349db6b49 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sat, 16 Nov 2024 03:21:54 +0000 Subject: [PATCH 094/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 68 +++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 915a610c..6a18cdd5 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -578,7 +578,7 @@ def score_distance_scan(self, bounds, ax): Matplotlib axes """ scores = self.scan["score"] - distances = np.linspace(bounds['dist'][0], bounds['dist'][1], len(scores)) + distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.set_xticks(np.arange(len(scores), step=20)) ax.set_yticks(scores[::100]) @@ -599,10 +599,15 @@ def residual_distance_scan(self, bounds, ax): Matplotlib axes """ residuals = self.scan["residual"] - distances = np.linspace(bounds['dist'][0], bounds['dist'][1], len(residuals)) + distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(residuals)) ax.plot(distances, residuals) best_dist = distances[np.argmin(residuals)] - ax.axvline(best_dist, color="red", linestyle="--", label=f"Best distance: {best_dist:.2e}") + ax.axvline( + best_dist, + color="red", + linestyle="--", + label=f"Best distance: {best_dist:.2e}", + ) ax.set_xticks(np.arange(len(residuals), step=20)) ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) ax.set_xlabel("Distance index") @@ -625,21 +630,56 @@ def hist_and_compute_stats(self, powder, exp, run, ax): Matplotlib axes """ mean = np.mean(powder) - threshold = np.mean(powder)+3*np.std(powder) - nice_pix = powder < threshold + threshold = np.mean(powder) + 3 * np.std(powder) + nice_pix = powder < threshold mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) percentile_99 = np.percentile(powder[nice_pix], 99) - _ = ax.hist(powder[nice_pix], bins=1000, color='skyblue', edgecolor='black', alpha=0.7, label="Pixel Intensities") - ax.axvline(mean, color='red', linestyle='--', linewidth=1.5, label=f'Mean ({mean:.2f})') - ax.axvline(mean + std_dev, color='orange', linestyle='--', linewidth=1.5, label=f'Mean + 1 Std ({mean + std_dev:.2f})') - ax.axvline(mean + 2 * std_dev, color='green', linestyle='--', linewidth=1.5, label=f'Mean + 2 Std ({mean + 2 * std_dev:.2f})') - ax.axvline(mean + 3 * std_dev, color='blue', linestyle='--', linewidth=1.5, label=f'Mean + 3 Std ({mean + 3 * std_dev:.2f})') - ax.axvline(percentile_99, color='purple', linestyle=':', linewidth=1.5, label=f'99th Percentile ({percentile_99:.2f})') + _ = ax.hist( + powder[nice_pix], + bins=1000, + color="skyblue", + edgecolor="black", + alpha=0.7, + label="Pixel Intensities", + ) + ax.axvline( + mean, color="red", linestyle="--", linewidth=1.5, label=f"Mean ({mean:.2f})" + ) + ax.axvline( + mean + std_dev, + color="orange", + linestyle="--", + linewidth=1.5, + label=f"Mean + 1 Std ({mean + std_dev:.2f})", + ) + ax.axvline( + mean + 2 * std_dev, + color="green", + linestyle="--", + linewidth=1.5, + label=f"Mean + 2 Std ({mean + 2 * std_dev:.2f})", + ) + ax.axvline( + mean + 3 * std_dev, + color="blue", + linestyle="--", + linewidth=1.5, + label=f"Mean + 3 Std ({mean + 3 * std_dev:.2f})", + ) + ax.axvline( + percentile_99, + color="purple", + linestyle=":", + linewidth=1.5, + label=f"99th Percentile ({percentile_99:.2f})", + ) ax.set_xlim(0, mean + 5 * std_dev) - ax.set_xlabel('Pixel Intensity') - ax.set_ylabel('Frequency') - ax.set_title(f'Histogram of Pixel Intensities with Statistical Thresholds for {exp} run {run}') + ax.set_xlabel("Pixel Intensity") + ax.set_ylabel("Frequency") + ax.set_title( + f"Histogram of Pixel Intensities with Statistical Thresholds for {exp} run {run}" + ) ax.legend() def visualize_results(self, powder, bo_history, detector, params, plot=""): From 7c1b016582aee9062d97c80622128cd6918383e2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 19:23:48 -0800 Subject: [PATCH 095/375] [edit] Final edit for plotting diagnostics --- lute/tasks/geom_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 915a610c..873f3e26 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -603,6 +603,7 @@ def residual_distance_scan(self, bounds, ax): ax.plot(distances, residuals) best_dist = distances[np.argmin(residuals)] ax.axvline(best_dist, color="red", linestyle="--", label=f"Best distance: {best_dist:.2e}") + ax.set_yscale("log") ax.set_xticks(np.arange(len(residuals), step=20)) ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) ax.set_xlabel("Distance index") From e310267873727bfd7d279033c630e204adadf861 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sat, 16 Nov 2024 03:24:37 +0000 Subject: [PATCH 096/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 09e88830..dcf26264 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -602,7 +602,12 @@ def residual_distance_scan(self, bounds, ax): distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(residuals)) ax.plot(distances, residuals) best_dist = distances[np.argmin(residuals)] - ax.axvline(best_dist, color="red", linestyle="--", label=f"Best distance: {best_dist:.2e}") + ax.axvline( + best_dist, + color="red", + linestyle="--", + label=f"Best distance: {best_dist:.2e}", + ) ax.set_yscale("log") ax.set_xticks(np.arange(len(residuals), step=20)) ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) From 5cdfccd258fc83ee9edf073410e362f6aee72f30 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 19:36:15 -0800 Subject: [PATCH 097/375] [edit] Final edit of plotting diagnostics --- lute/tasks/geom_opt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 09e88830..b1f73a62 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -605,7 +605,6 @@ def residual_distance_scan(self, bounds, ax): ax.axvline(best_dist, color="red", linestyle="--", label=f"Best distance: {best_dist:.2e}") ax.set_yscale("log") ax.set_xticks(np.arange(len(residuals), step=20)) - ax.ticklabel_format(axis="y", style="sci", scilimits=(0, 0)) ax.set_xlabel("Distance index") ax.set_ylabel("Residual") ax.set_title("Residual vs Distance") From 345ccb426644026528edf505969c6035364ce500 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 15 Nov 2024 19:53:32 -0800 Subject: [PATCH 098/375] [edit] Final edit of plotting diagnostics --- lute/tasks/geom_opt.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 1c91fc91..6eaa827b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -580,9 +580,6 @@ def score_distance_scan(self, bounds, ax): scores = self.scan["score"] distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) - ax.set_xticks(np.arange(len(scores), step=20)) - ax.set_yticks(scores[::100]) - ax.set_yticklabels([f"{v/1000:.1f}k" for v in scores[::100]]) ax.set_xlabel("Distance index") ax.set_ylabel("Score") ax.set_title("Number of Control Points vs Distance") @@ -609,7 +606,6 @@ def residual_distance_scan(self, bounds, ax): label=f"Best distance: {best_dist:.2e}", ) ax.set_yscale("log") - ax.set_xticks(np.arange(len(residuals), step=20)) ax.set_xlabel("Distance index") ax.set_ylabel("Residual") ax.set_title("Residual vs Distance") @@ -678,9 +674,9 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax.set_xlabel("Pixel Intensity") ax.set_ylabel("Frequency") ax.set_title( - f"Histogram of Pixel Intensities with Statistical Thresholds for {exp} run {run}" + f"Histogram of Pixel Intensities \n for {exp} run {run}" ) - ax.legend() + ax.legend(fontsize='x-small') def visualize_results(self, powder, bo_history, detector, params, plot=""): """ @@ -699,7 +695,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): plot : str Path to save plot """ - fig = plt.figure(figsize=(8, 12), dpi=180) + fig = plt.figure(figsize=(10, 14), dpi=180) nrow, ncol = 3, 2 irow, icol = 0, 0 From 04ce28318118a114c138fa53aef260873234cb22 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sat, 16 Nov 2024 03:53:56 +0000 Subject: [PATCH 099/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 6eaa827b..bc871209 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -673,10 +673,8 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax.set_xlim(0, mean + 5 * std_dev) ax.set_xlabel("Pixel Intensity") ax.set_ylabel("Frequency") - ax.set_title( - f"Histogram of Pixel Intensities \n for {exp} run {run}" - ) - ax.legend(fontsize='x-small') + ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") + ax.legend(fontsize="x-small") def visualize_results(self, powder, bo_history, detector, params, plot=""): """ From 7bbfec65aa037927b546d878d16bc935e178a25a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 17 Nov 2024 10:41:45 -0800 Subject: [PATCH 100/375] [edit] Fix figsize and dpi --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 6eaa827b..3c8a2301 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -695,7 +695,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): plot : str Path to save plot """ - fig = plt.figure(figsize=(10, 14), dpi=180) + fig = plt.figure(figsize=(12, 16), dpi=180) nrow, ncol = 3, 2 irow, icol = 0, 0 From 069d3938728a28b794af487ca045bf920af3aaa0 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 19 Nov 2024 10:17:01 -0800 Subject: [PATCH 101/375] [edit] Filter out residuals for which #CP < 10th-percentile --- lute/tasks/geom_opt.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0d9db779..13eae0d4 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -474,7 +474,10 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - index = np.argmin(self.scan["residual"]) + percentile_10 = np.percentile(self.scan["score"], 10) + logger.info(f"10th Score Percentile: {percentile_10:.2e}") + index = np.argmin(self.scan["residual"][self.scan["score"] > percentile_10]) + self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] self.residual = self.scan["residual"][index] @@ -578,9 +581,13 @@ def score_distance_scan(self, bounds, ax): Matplotlib axes """ scores = self.scan["score"] + percentile_10 = np.percentile(scores, 10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) - ax.set_xlabel("Distance index") + ax.axhline(percentile_10, color="red", linestyle="--", + label=f"10th Percentile: {percentile_10:.2e}", + ) + ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.set_title("Number of Control Points vs Distance") @@ -598,15 +605,15 @@ def residual_distance_scan(self, bounds, ax): residuals = self.scan["residual"] distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(residuals)) ax.plot(distances, residuals) - best_dist = distances[np.argmin(residuals)] + best_dist = distances[self.index] ax.axvline( best_dist, - color="red", + color="green", linestyle="--", label=f"Best distance: {best_dist:.2e}", ) ax.set_yscale("log") - ax.set_xlabel("Distance index") + ax.set_xlabel("Distance (m)") ax.set_ylabel("Residual") ax.set_title("Residual vs Distance") @@ -789,7 +796,7 @@ def _run(self) -> None: f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" ) logger.info(f"Final Residuals: {optimizer.residual:.2e}") - plot = f"{self._task_parameters.work_dir}figs/bayes_opt_geom_r{optimizer.run:0>4}.png" + plot = f"{self._task_parameters.work_dir}figs/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" detector = self.update_geometry(optimizer) optimizer.visualize_results( powder=optimizer.powder, From c28ac057057dce86f47afd4de09b0bd5717dc797 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Tue, 19 Nov 2024 18:17:35 +0000 Subject: [PATCH 102/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 13eae0d4..ca417520 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -584,7 +584,10 @@ def score_distance_scan(self, bounds, ax): percentile_10 = np.percentile(scores, 10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) - ax.axhline(percentile_10, color="red", linestyle="--", + ax.axhline( + percentile_10, + color="red", + linestyle="--", label=f"10th Percentile: {percentile_10:.2e}", ) ax.set_xlabel("Distance (m)") From d14f3558e1f4deb78d217c665f14cab999037687 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 19 Nov 2024 11:12:46 -0800 Subject: [PATCH 103/375] [edit] Change path to update geometry to have both .geom and .data in same output geom folder --- lute/tasks/geom_opt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 13eae0d4..d4b71be9 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -796,7 +796,7 @@ def _run(self) -> None: f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" ) logger.info(f"Final Residuals: {optimizer.residual:.2e}") - plot = f"{self._task_parameters.work_dir}figs/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" + plot = f"{self._task_parameters.work_dir}/figs/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" detector = self.update_geometry(optimizer) optimizer.visualize_results( powder=optimizer.powder, @@ -841,13 +841,13 @@ def update_geometry(self, optimizer): detector=optimizer.detector, params=optimizer.params, psana_file=self._task_parameters.in_file, - out_file=self._task_parameters.in_file.replace( - "0-end.data", f"r{self._task_parameters.run:0>4}.geom" + out_file=self._task_parameters.out_file.replace( + f"{self._task_parameters.run}-end.data", f"r{self._task_parameters.run:0>4}.geom" ), ) CrystFELToPsana( in_file=self._task_parameters.in_file.replace( - "0-end.data", f"r{self._task_parameters.run:0>4}.geom" + f"{self._task_parameters.run}-end.data", f"r{self._task_parameters.run:0>4}.geom" ), det_type=optimizer.det_type, out_file=self._task_parameters.out_file, From c61871f23036106a5b2ef6357e4a3cfed6cc5b9e Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Tue, 19 Nov 2024 19:13:25 +0000 Subject: [PATCH 104/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7b98296e..d38940d8 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -845,12 +845,14 @@ def update_geometry(self, optimizer): params=optimizer.params, psana_file=self._task_parameters.in_file, out_file=self._task_parameters.out_file.replace( - f"{self._task_parameters.run}-end.data", f"r{self._task_parameters.run:0>4}.geom" + f"{self._task_parameters.run}-end.data", + f"r{self._task_parameters.run:0>4}.geom", ), ) CrystFELToPsana( in_file=self._task_parameters.in_file.replace( - f"{self._task_parameters.run}-end.data", f"r{self._task_parameters.run:0>4}.geom" + f"{self._task_parameters.run}-end.data", + f"r{self._task_parameters.run:0>4}.geom", ), det_type=optimizer.det_type, out_file=self._task_parameters.out_file, From 94dfc1587d83118d598e333da87d47a9952511af Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 19 Nov 2024 11:24:06 -0800 Subject: [PATCH 105/375] [edit] Change path to update geometry to have both .geom and .data in same output geom folder --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7b98296e..86117a8b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -849,7 +849,7 @@ def update_geometry(self, optimizer): ), ) CrystFELToPsana( - in_file=self._task_parameters.in_file.replace( + in_file=self._task_parameters.out_file.replace( f"{self._task_parameters.run}-end.data", f"r{self._task_parameters.run:0>4}.geom" ), det_type=optimizer.det_type, From f46de9f9f90bf5c2b7d05854b5b5f33342713e11 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Tue, 19 Nov 2024 19:25:03 +0000 Subject: [PATCH 106/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 6566278d..820d308c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -851,7 +851,8 @@ def update_geometry(self, optimizer): ) CrystFELToPsana( in_file=self._task_parameters.out_file.replace( - f"{self._task_parameters.run}-end.data", f"r{self._task_parameters.run:0>4}.geom" + f"{self._task_parameters.run}-end.data", + f"r{self._task_parameters.run:0>4}.geom", ), det_type=optimizer.det_type, out_file=self._task_parameters.out_file, From 756e50c6d0f166d5f24d6973c3a662ac07f6e68f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 19 Nov 2024 11:39:11 -0800 Subject: [PATCH 107/375] [edit] Since filtering valid residuals reset index of array, need to shift properly to return min of residuals --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 6566278d..8fce14ce 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -476,7 +476,9 @@ def finalize(self): self.scan[key] = np.array([item for item in self.scan[key]]) percentile_10 = np.percentile(self.scan["score"], 10) logger.info(f"10th Score Percentile: {percentile_10:.2e}") - index = np.argmin(self.scan["residual"][self.scan["score"] > percentile_10]) + valid_indices = np.where(self.scan["score"] > percentile_10)[0] + shift_index = np.argmin(self.scan["residual"][valid_indices]) + index = valid_indices[shift_index] self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] From e4799e50425db60bbd73eb3e616037b1833c3bab Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 19 Nov 2024 12:26:13 -0800 Subject: [PATCH 108/375] [edit] Change filtering from 10 to 15% scores --- lute/tasks/geom_opt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d3b0f5fb..2690d419 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -474,9 +474,9 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - percentile_10 = np.percentile(self.scan["score"], 10) - logger.info(f"10th Score Percentile: {percentile_10:.2e}") - valid_indices = np.where(self.scan["score"] > percentile_10)[0] + percentile_15 = np.percentile(self.scan["score"], 15) + logger.info(f"10th Score Percentile: {percentile_15:.2e}") + valid_indices = np.where(self.scan["score"] > percentile_15)[0] shift_index = np.argmin(self.scan["residual"][valid_indices]) index = valid_indices[shift_index] self.index = index @@ -583,14 +583,14 @@ def score_distance_scan(self, bounds, ax): Matplotlib axes """ scores = self.scan["score"] - percentile_10 = np.percentile(scores, 10) + percentile_15 = np.percentile(scores, 15) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( percentile_10, color="red", linestyle="--", - label=f"10th Percentile: {percentile_10:.2e}", + label=f"15th Percentile: {percentile_15:.2e}", ) ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") From 7e71cf4fe487541330cc6e1f14b894014a2a0a0d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 19 Nov 2024 12:51:36 -0800 Subject: [PATCH 109/375] [edit] Change filtering from 10 to 15% scores --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 2690d419..b7348089 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -587,7 +587,7 @@ def score_distance_scan(self, bounds, ax): distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( - percentile_10, + percentile_15, color="red", linestyle="--", label=f"15th Percentile: {percentile_15:.2e}", From 8390cbd0d3e35aad8d3261e106f81f9a43ac4c13 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 19 Nov 2024 13:03:14 -0800 Subject: [PATCH 110/375] [edit] Change filtering from 15 to 10% scores --- lute/tasks/geom_opt.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index b7348089..d3b0f5fb 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -474,9 +474,9 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - percentile_15 = np.percentile(self.scan["score"], 15) - logger.info(f"10th Score Percentile: {percentile_15:.2e}") - valid_indices = np.where(self.scan["score"] > percentile_15)[0] + percentile_10 = np.percentile(self.scan["score"], 10) + logger.info(f"10th Score Percentile: {percentile_10:.2e}") + valid_indices = np.where(self.scan["score"] > percentile_10)[0] shift_index = np.argmin(self.scan["residual"][valid_indices]) index = valid_indices[shift_index] self.index = index @@ -583,14 +583,14 @@ def score_distance_scan(self, bounds, ax): Matplotlib axes """ scores = self.scan["score"] - percentile_15 = np.percentile(scores, 15) + percentile_10 = np.percentile(scores, 10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( - percentile_15, + percentile_10, color="red", linestyle="--", - label=f"15th Percentile: {percentile_15:.2e}", + label=f"10th Percentile: {percentile_10:.2e}", ) ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") From 585ce2ffead051a47f0fe6fbc4a8a67d7110fad1 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 19 Nov 2024 15:18:45 -0800 Subject: [PATCH 111/375] [test] Change treshold from mean+3std to mean+2std --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d3b0f5fb..6688b937 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -638,7 +638,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): Matplotlib axes """ mean = np.mean(powder) - threshold = np.mean(powder) + 3 * np.std(powder) + threshold = np.mean(powder) + 2 * np.std(powder) nice_pix = powder < threshold mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) From 954d10f2e7cd465f50f1ae8f3d2638651f3f05db Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 20 Nov 2024 20:36:17 -0800 Subject: [PATCH 112/375] [edit] Skip distances for which all initial samples have score 0, hence dividing by 0 later --- lute/tasks/geom_opt.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 6688b937..f2a534bd 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -274,6 +274,16 @@ def bayes_opt_center( y[i] = len(sg.geometry_refinement.data) bo_history[f"init_sample_{i+1}"] = {"param": X_samples[i], "score": y[i]} + if np.all(y == 0): + result = { + "bo_history": bo_history, + "params": [dist, 0, 0, 0, 0, 0], + "residual": 0, + "score": 0, + "best_idx": 0, + } + logger.warning(f"All samples have score 0 for dist={dist}. Skipping Bayesian Optimization.") + return result y_norm = (y - np.mean(y)) / np.std(y) best_score = np.max(y_norm) From 4c59b7ddcd2f0b5457dcc2b662b51407b9ba2056 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 21 Nov 2024 04:36:45 +0000 Subject: [PATCH 113/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index f2a534bd..5b02e1ef 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -282,7 +282,9 @@ def bayes_opt_center( "score": 0, "best_idx": 0, } - logger.warning(f"All samples have score 0 for dist={dist}. Skipping Bayesian Optimization.") + logger.warning( + f"All samples have score 0 for dist={dist}. Skipping Bayesian Optimization." + ) return result y_norm = (y - np.mean(y)) / np.std(y) best_score = np.max(y_norm) From 1436a6fd346ad1d26c9cd6538ec7232403de5987 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 20 Nov 2024 20:52:54 -0800 Subject: [PATCH 114/375] [edit] Skip distances for which all initial samples have score 0, hence dividing by 0 later + taken into account for filtering out bad residuals --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index f2a534bd..0d689c4e 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -484,7 +484,7 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - percentile_10 = np.percentile(self.scan["score"], 10) + percentile_10 = np.percentile(self.scan["score"][self.scan["score"] > 0], 10) logger.info(f"10th Score Percentile: {percentile_10:.2e}") valid_indices = np.where(self.scan["score"] > percentile_10)[0] shift_index = np.argmin(self.scan["residual"][valid_indices]) From b7bb7678fb5ed5cf0c301b35128946bfa76a18ca Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 20 Nov 2024 21:04:24 -0800 Subject: [PATCH 115/375] [edit] Skip distances for which all initial samples have score 0, hence dividing by 0 later + taken into account for filtering out bad residuals --- lute/tasks/geom_opt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 3b8214e0..e25dffbf 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -486,7 +486,8 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - percentile_10 = np.percentile(self.scan["score"][self.scan["score"] > 0], 10) + non_zero_scores = np.where(self.scan["score"] > 0)[0] + percentile_10 = np.percentile(self.scan["score"][non_zero_scores], 10) logger.info(f"10th Score Percentile: {percentile_10:.2e}") valid_indices = np.where(self.scan["score"] > percentile_10)[0] shift_index = np.argmin(self.scan["residual"][valid_indices]) From 9a4c745d8fe94bd344399bb571407e63a7910ba3 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 20 Nov 2024 21:16:22 -0800 Subject: [PATCH 116/375] [edit] Fix plots after recent modifications --- lute/tasks/geom_opt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index e25dffbf..2677496c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -596,7 +596,8 @@ def score_distance_scan(self, bounds, ax): Matplotlib axes """ scores = self.scan["score"] - percentile_10 = np.percentile(scores, 10) + non_zero_scores = np.where(scores > 0)[0] + percentile_10 = np.percentile(scores[non_zero_scores], 10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( @@ -607,6 +608,7 @@ def score_distance_scan(self, bounds, ax): ) ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") + ax.legend(fontsize="x-small") ax.set_title("Number of Control Points vs Distance") def residual_distance_scan(self, bounds, ax): @@ -630,6 +632,7 @@ def residual_distance_scan(self, bounds, ax): linestyle="--", label=f"Best distance: {best_dist:.2e}", ) + ax.legend(fontsize="x-small") ax.set_yscale("log") ax.set_xlabel("Distance (m)") ax.set_ylabel("Residual") From b9bc1479061fb0c7fe0c4c7834dac1526e56a888 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 10:00:00 -0800 Subject: [PATCH 117/375] [test] Replace 10% percentile with mean-std threshold for distance scores --- lute/tasks/geom_opt.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 2677496c..53094044 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -488,8 +488,12 @@ def finalize(self): self.scan[key] = np.array([item for item in self.scan[key]]) non_zero_scores = np.where(self.scan["score"] > 0)[0] percentile_10 = np.percentile(self.scan["score"][non_zero_scores], 10) + mean = np.mean(self.scan["score"][non_zero_scores]) + std = np.std(self.scan["score"][non_zero_scores]) + threshold = mean - std + logger.info(f"Threshold Score: {threshold:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - valid_indices = np.where(self.scan["score"] > percentile_10)[0] + valid_indices = np.where(self.scan["score"] > threshold)[0] shift_index = np.argmin(self.scan["residual"][valid_indices]) index = valid_indices[shift_index] self.index = index @@ -597,6 +601,9 @@ def score_distance_scan(self, bounds, ax): """ scores = self.scan["score"] non_zero_scores = np.where(scores > 0)[0] + mean = np.mean(scores[non_zero_scores]) + std = np.std(scores[non_zero_scores]) + threshold = mean - std percentile_10 = np.percentile(scores[non_zero_scores], 10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) @@ -606,6 +613,7 @@ def score_distance_scan(self, bounds, ax): linestyle="--", label=f"10th Percentile: {percentile_10:.2e}", ) + ax.axhline(threshold, color="orange", linestyle="--", label=f"Threshold: {threshold:.2e}") ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.legend(fontsize="x-small") From b7a0ae9284e71876a9b81263022a7189bd8aa016 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 21 Nov 2024 18:00:31 +0000 Subject: [PATCH 118/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 53094044..b6f50322 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -613,7 +613,12 @@ def score_distance_scan(self, bounds, ax): linestyle="--", label=f"10th Percentile: {percentile_10:.2e}", ) - ax.axhline(threshold, color="orange", linestyle="--", label=f"Threshold: {threshold:.2e}") + ax.axhline( + threshold, + color="orange", + linestyle="--", + label=f"Threshold: {threshold:.2e}", + ) ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.legend(fontsize="x-small") From 72aa4a79185c3d3e173cbb871a0e9bec52035568 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 10:20:36 -0800 Subject: [PATCH 119/375] [test] Replace threshold with 12% percentile --- lute/tasks/geom_opt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 53094044..10abf215 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -487,12 +487,12 @@ def finalize(self): for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) non_zero_scores = np.where(self.scan["score"] > 0)[0] - percentile_10 = np.percentile(self.scan["score"][non_zero_scores], 10) + percentile_12 = np.percentile(self.scan["score"][non_zero_scores], 12) mean = np.mean(self.scan["score"][non_zero_scores]) std = np.std(self.scan["score"][non_zero_scores]) - threshold = mean - std + threshold = mean - 2 * std logger.info(f"Threshold Score: {threshold:.2e}") - logger.info(f"10th Score Percentile: {percentile_10:.2e}") + logger.info(f"12th Score Percentile: {percentile_12:.2e}") valid_indices = np.where(self.scan["score"] > threshold)[0] shift_index = np.argmin(self.scan["residual"][valid_indices]) index = valid_indices[shift_index] @@ -603,15 +603,15 @@ def score_distance_scan(self, bounds, ax): non_zero_scores = np.where(scores > 0)[0] mean = np.mean(scores[non_zero_scores]) std = np.std(scores[non_zero_scores]) - threshold = mean - std - percentile_10 = np.percentile(scores[non_zero_scores], 10) + threshold = mean - 2 * std + percentile_12 = np.percentile(scores[non_zero_scores], 12) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( - percentile_10, + percentile_12, color="red", linestyle="--", - label=f"10th Percentile: {percentile_10:.2e}", + label=f"12th Percentile: {percentile_12:.2e}", ) ax.axhline(threshold, color="orange", linestyle="--", label=f"Threshold: {threshold:.2e}") ax.set_xlabel("Distance (m)") From 1085680429b6899115231743d7e993c3e694553f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 10:42:21 -0800 Subject: [PATCH 120/375] [edit] Add some metrics to quantify score number of control points vs distance distribution --- lute/tasks/geom_opt.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 1c7b6403..d934e02d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -490,10 +490,10 @@ def finalize(self): percentile_12 = np.percentile(self.scan["score"][non_zero_scores], 12) mean = np.mean(self.scan["score"][non_zero_scores]) std = np.std(self.scan["score"][non_zero_scores]) - threshold = mean - 2 * std + threshold = mean - std logger.info(f"Threshold Score: {threshold:.2e}") logger.info(f"12th Score Percentile: {percentile_12:.2e}") - valid_indices = np.where(self.scan["score"] > threshold)[0] + valid_indices = np.where(self.scan["score"] > percentile_12)[0] shift_index = np.argmin(self.scan["residual"][valid_indices]) index = valid_indices[shift_index] self.index = index @@ -602,22 +602,39 @@ def score_distance_scan(self, bounds, ax): scores = self.scan["score"] non_zero_scores = np.where(scores > 0)[0] mean = np.mean(scores[non_zero_scores]) - std = np.std(scores[non_zero_scores]) - threshold = mean - 2 * std + mini = np.min(scores[non_zero_scores]) + std_dev = np.std(scores[non_zero_scores]) percentile_12 = np.percentile(scores[non_zero_scores], 12) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( percentile_12, - color="red", + color="purple", linestyle="--", label=f"12th Percentile: {percentile_12:.2e}", ) - ax.axhline( - threshold, + ax.axvline( + mean, color="red", linestyle="--", linewidth=1.5, label=f"Mean ({mean:.2f})" + ) + ax.axvline( + mean - std_dev, color="orange", linestyle="--", - label=f"Threshold: {threshold:.2e}", + linewidth=1.5, + label=f"Mean - 1 Std ({mean - std_dev:.2f})", + ) + ax.axvline( + mean - 2 * std_dev, + color="green", + linestyle="--", + linewidth=1.5, + label=f"Mean - 2 Std ({mean - 2 * std_dev:.2f})", + ) + ax.axhline( + mini+std_dev, + color="yellow", + linestyle="--", + label=f"Minimum + 1 Std: {mini + std_dev:.2e}", ) ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") From 9d8b30afd8ec9f8e3c397aeb9b1761e41a12db97 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 21 Nov 2024 18:42:48 +0000 Subject: [PATCH 121/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d934e02d..703ac7da 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -631,7 +631,7 @@ def score_distance_scan(self, bounds, ax): label=f"Mean - 2 Std ({mean - 2 * std_dev:.2f})", ) ax.axhline( - mini+std_dev, + mini + std_dev, color="yellow", linestyle="--", label=f"Minimum + 1 Std: {mini + std_dev:.2e}", From 0dda99b87a416abdc27bd64fd21bddbe32cc1364 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 10:49:44 -0800 Subject: [PATCH 122/375] [edit] Add some metrics to quantify score number of control points vs distance distribution --- lute/tasks/geom_opt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d934e02d..7aee1558 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -613,17 +613,17 @@ def score_distance_scan(self, bounds, ax): linestyle="--", label=f"12th Percentile: {percentile_12:.2e}", ) - ax.axvline( + ax.axhline( mean, color="red", linestyle="--", linewidth=1.5, label=f"Mean ({mean:.2f})" ) - ax.axvline( + ax.axhline( mean - std_dev, color="orange", linestyle="--", linewidth=1.5, label=f"Mean - 1 Std ({mean - std_dev:.2f})", ) - ax.axvline( + ax.axhline( mean - 2 * std_dev, color="green", linestyle="--", From 9c9793c4f79e1b93985857be249772cb79879980 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 11:59:11 -0800 Subject: [PATCH 123/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 61 ++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index fd54468c..041dcf5a 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -137,6 +137,25 @@ def build_calibrant(self): calibrant.wavelength = self.wavelength self.calibrant = calibrant + def build_mask(self, central=True, edges=True): + """ + Mask pixels marked as false status, edges and central pixels of panels + + Parameters + ---------- + central : bool + Mask central pixel of panels + edges : bool + Mask edges of panels + """ + ds_args = f"exp={self.exp}:run={self.run}:idx" + ds = psana.DataSource(ds_args) + det = psana.Detector(self.det_type, ds.env()) + mask = det.mask_v2(par=self.run, central=central, edges=edges) + if len(mask.shape) != 2: + mask = np.reshape(mask, (mask.shape[0] * mask.shape[1], mask.shape[2])) + return mask + def min_intensity(self, Imin, powder): """ Define minimal intensity for control point extraction @@ -446,6 +465,10 @@ def bayes_opt_geom( self.build_calibrant() + mask = self.build_mask() + + powder = powder * mask + self.max_rings = max_rings Imin, powder = self.min_intensity(Imin, powder) @@ -487,13 +510,13 @@ def finalize(self): for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) non_zero_scores = np.where(self.scan["score"] > 0)[0] - percentile_12 = np.percentile(self.scan["score"][non_zero_scores], 12) + percentile_10 = np.percentile(self.scan["score"][non_zero_scores], 10) mean = np.mean(self.scan["score"][non_zero_scores]) - std = np.std(self.scan["score"][non_zero_scores]) - threshold = mean - std - logger.info(f"Threshold Score: {threshold:.2e}") - logger.info(f"12th Score Percentile: {percentile_12:.2e}") - valid_indices = np.where(self.scan["score"] > percentile_12)[0] + std_dev = np.std(self.scan["score"][non_zero_scores]) + logger.info(f"Mean Score: {mean:.2e}") + logger.info(f"Score Std Dev: {std_dev:.2e}") + logger.info(f"12th Score Percentile: {percentile_10:.2e}") + valid_indices = np.where(self.scan["score"] > percentile_10)[0] shift_index = np.argmin(self.scan["residual"][valid_indices]) index = valid_indices[shift_index] self.index = index @@ -602,16 +625,15 @@ def score_distance_scan(self, bounds, ax): scores = self.scan["score"] non_zero_scores = np.where(scores > 0)[0] mean = np.mean(scores[non_zero_scores]) - mini = np.min(scores[non_zero_scores]) std_dev = np.std(scores[non_zero_scores]) - percentile_12 = np.percentile(scores[non_zero_scores], 12) + percentile_10 = np.percentile(scores[non_zero_scores], 10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( - percentile_12, + percentile_10, color="purple", linestyle="--", - label=f"12th Percentile: {percentile_12:.2e}", + label=f"10th Percentile: {percentile_10:.2e}", ) ax.axhline( mean, color="red", linestyle="--", linewidth=1.5, label=f"Mean ({mean:.2f})" @@ -623,19 +645,6 @@ def score_distance_scan(self, bounds, ax): linewidth=1.5, label=f"Mean - 1 Std ({mean - std_dev:.2f})", ) - ax.axhline( - mean - 2 * std_dev, - color="green", - linestyle="--", - linewidth=1.5, - label=f"Mean - 2 Std ({mean - 2 * std_dev:.2f})", - ) - ax.axhline( - mini + std_dev, - color="yellow", - linestyle="--", - label=f"Minimum + 1 Std: {mini + std_dev:.2e}", - ) ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.legend(fontsize="x-small") @@ -776,6 +785,12 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): ai = AzimuthalIntegrator( dist=params[0], detector=detector, wavelength=self.calibrant.wavelength ) + if self.det_type.lower() == "rayonix": + radius = powder.shape[0] / 4 + row, col = np.ogrid[:powder.shape[0], :powder.shape[1]] + center = (powder.shape[0] / 2, powder.shape[1] / 2) + mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius ** 2 + powder = powder * mask res = ai.integrate1d(powder, 1000) self.radial_integration(res, calibrant=self.calibrant, ax=ax3) icol += 1 From ef8b9db4af65f8d4d70f083ec50bc5a2c0806b77 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 21 Nov 2024 19:59:37 +0000 Subject: [PATCH 124/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 041dcf5a..ebf90de9 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -787,9 +787,9 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): ) if self.det_type.lower() == "rayonix": radius = powder.shape[0] / 4 - row, col = np.ogrid[:powder.shape[0], :powder.shape[1]] + row, col = np.ogrid[: powder.shape[0], : powder.shape[1]] center = (powder.shape[0] / 2, powder.shape[1] / 2) - mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius ** 2 + mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius**2 powder = powder * mask res = ai.integrate1d(powder, 1000) self.radial_integration(res, calibrant=self.calibrant, ax=ax3) From a4e3894744419befc2dd3d8c697db81f8b8a973e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 12:01:41 -0800 Subject: [PATCH 125/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 041dcf5a..dbf69053 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -790,8 +790,8 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): row, col = np.ogrid[:powder.shape[0], :powder.shape[1]] center = (powder.shape[0] / 2, powder.shape[1] / 2) mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius ** 2 - powder = powder * mask - res = ai.integrate1d(powder, 1000) + masked_powder = powder * mask + res = ai.integrate1d(masked_powder, 1000) self.radial_integration(res, calibrant=self.calibrant, ax=ax3) icol += 1 From 77d401c5469b6aa4eb62cd75e2b41d6ae956d87c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 12:02:14 -0800 Subject: [PATCH 126/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 78547e4b..aa4ec723 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -789,15 +789,9 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): radius = powder.shape[0] / 4 row, col = np.ogrid[: powder.shape[0], : powder.shape[1]] center = (powder.shape[0] / 2, powder.shape[1] / 2) -<<<<<<< HEAD mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius ** 2 masked_powder = powder * mask res = ai.integrate1d(masked_powder, 1000) -======= - mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius**2 - powder = powder * mask - res = ai.integrate1d(powder, 1000) ->>>>>>> ef8b9db4af65f8d4d70f083ec50bc5a2c0806b77 self.radial_integration(res, calibrant=self.calibrant, ax=ax3) icol += 1 From 7c4372f189a44429a97b43a715df718ad516e104 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 21 Nov 2024 20:02:41 +0000 Subject: [PATCH 127/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index aa4ec723..3b00accb 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -789,7 +789,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): radius = powder.shape[0] / 4 row, col = np.ogrid[: powder.shape[0], : powder.shape[1]] center = (powder.shape[0] / 2, powder.shape[1] / 2) - mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius ** 2 + mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius**2 masked_powder = powder * mask res = ai.integrate1d(masked_powder, 1000) self.radial_integration(res, calibrant=self.calibrant, ax=ax3) From 220a4ecaa6bf2d7098508aa3f8910ae58498a112 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 12:09:42 -0800 Subject: [PATCH 128/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index aa4ec723..467b6c09 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -73,7 +73,7 @@ def __init__( ): self.exp = exp self.run = run - self.det_type = det_type.lower() + self.det_type = det_type self.comm = MPI.COMM_WORLD self.rank = self.comm.Get_rank() self.size = self.comm.Get_size() From 7355241d69ed5b1500b69510346ac00a0cde7593 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 12:19:41 -0800 Subject: [PATCH 129/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 977a9c6f..1c3803d5 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -151,7 +151,10 @@ def build_mask(self, central=True, edges=True): ds_args = f"exp={self.exp}:run={self.run}:idx" ds = psana.DataSource(ds_args) det = psana.Detector(self.det_type, ds.env()) - mask = det.mask_v2(par=self.run, central=central, edges=edges) + runner = next(ds.runs()) + evt = runner.event(self.times[0]) + runnum = evt.run() + mask = det.mask_v2(par=runnum, central=central, edges=edges) if len(mask.shape) != 2: mask = np.reshape(mask, (mask.shape[0] * mask.shape[1], mask.shape[2])) return mask From e7a352d34e8443883b717831b1849217a4a9aa4d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 12:22:14 -0800 Subject: [PATCH 130/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 1c3803d5..cfc05905 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -152,7 +152,7 @@ def build_mask(self, central=True, edges=True): ds = psana.DataSource(ds_args) det = psana.Detector(self.det_type, ds.env()) runner = next(ds.runs()) - evt = runner.event(self.times[0]) + evt = runner.event(runner.times[0]) runnum = evt.run() mask = det.mask_v2(par=runnum, central=central, edges=edges) if len(mask.shape) != 2: From 71a4bae1131452a082f17dac01b28a9862abfc46 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 12:24:34 -0800 Subject: [PATCH 131/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index cfc05905..00e949fc 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -152,7 +152,7 @@ def build_mask(self, central=True, edges=True): ds = psana.DataSource(ds_args) det = psana.Detector(self.det_type, ds.env()) runner = next(ds.runs()) - evt = runner.event(runner.times[0]) + evt = runner.event(runner.times()[0]) runnum = evt.run() mask = det.mask_v2(par=runnum, central=central, edges=edges) if len(mask.shape) != 2: From e6e757d234a42ad2abf5affa2a069700c7be7240 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 12:46:16 -0800 Subject: [PATCH 132/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 00e949fc..8b1ab04c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -788,6 +788,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): ai = AzimuthalIntegrator( dist=params[0], detector=detector, wavelength=self.calibrant.wavelength ) + masked_powder = powder if self.det_type.lower() == "rayonix": radius = powder.shape[0] / 4 row, col = np.ogrid[: powder.shape[0], : powder.shape[1]] From 477080c26e5d8113858578582f59a231e1d87a22 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 14:02:14 -0800 Subject: [PATCH 133/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8b1ab04c..0486d7f0 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -154,9 +154,13 @@ def build_mask(self, central=True, edges=True): runner = next(ds.runs()) evt = runner.event(runner.times()[0]) runnum = evt.run() - mask = det.mask_v2(par=runnum, central=central, edges=edges) - if len(mask.shape) != 2: - mask = np.reshape(mask, (mask.shape[0] * mask.shape[1], mask.shape[2])) + try: + mask = det.mask_v2(par=runnum, central=central, edges=edges) + except: + mask = None + if mask is not None: + if len(mask.shape) != 2: + mask = np.reshape(mask, (mask.shape[0] * mask.shape[1], mask.shape[2])) return mask def min_intensity(self, Imin, powder): @@ -171,10 +175,11 @@ def min_intensity(self, Imin, powder): powder : np.ndarray Powder image """ - mean = np.mean(powder) - std = np.std(powder) + non_zero = powder > 0 + mean = np.mean(powder[non_zero]) + std = np.std(powder[non_zero]) threshold = mean + 3 * std - nice_pix = powder < threshold + nice_pix = powder[non_zero] < threshold Imin = np.percentile(powder[nice_pix], Imin) self.Imin = Imin self.powder = powder @@ -469,8 +474,8 @@ def bayes_opt_geom( self.build_calibrant() mask = self.build_mask() - - powder = powder * mask + if mask is not None: + powder = powder * mask self.max_rings = max_rings Imin, powder = self.min_intensity(Imin, powder) From 9d68d9283ee4c907ea297d5f1a58d35162e464b7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 14:08:54 -0800 Subject: [PATCH 134/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0486d7f0..6334f066 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -180,7 +180,7 @@ def min_intensity(self, Imin, powder): std = np.std(powder[non_zero]) threshold = mean + 3 * std nice_pix = powder[non_zero] < threshold - Imin = np.percentile(powder[nice_pix], Imin) + Imin = np.percentile(powder.flatten()[nice_pix], Imin) self.Imin = Imin self.powder = powder return Imin, powder From 8303356c4bb01cfb14c97cad2fcca42adcae6acb Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 14:49:30 -0800 Subject: [PATCH 135/375] [edit] Mask false status pixels, edges and central pixels of panels + add circular mask for Rayonix radial integration --- lute/tasks/geom_opt.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 6334f066..70fb5702 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -175,12 +175,12 @@ def min_intensity(self, Imin, powder): powder : np.ndarray Powder image """ - non_zero = powder > 0 - mean = np.mean(powder[non_zero]) - std = np.std(powder[non_zero]) + masked_powder = np.ma.masked_array(powder, 0) + mean = np.mean(masked_powder) + std = np.std(masked_powder) threshold = mean + 3 * std - nice_pix = powder[non_zero] < threshold - Imin = np.percentile(powder.flatten()[nice_pix], Imin) + nice_pix = masked_powder < threshold + Imin = np.percentile(masked_powder[nice_pix], Imin) self.Imin = Imin self.powder = powder return Imin, powder @@ -523,7 +523,7 @@ def finalize(self): std_dev = np.std(self.scan["score"][non_zero_scores]) logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") - logger.info(f"12th Score Percentile: {percentile_10:.2e}") + logger.info(f"10th Score Percentile: {percentile_10:.2e}") valid_indices = np.where(self.scan["score"] > percentile_10)[0] shift_index = np.argmin(self.scan["residual"][valid_indices]) index = valid_indices[shift_index] From 1fda86c7ea05be88ee7edaae6ea73c8def2e65e0 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 15:06:10 -0800 Subject: [PATCH 136/375] [edit] Reduce Rayonix radial integration to circle within 1920/4 * sqrt(2) --- lute/tasks/geom_opt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 70fb5702..aa130e1b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -795,7 +795,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): ) masked_powder = powder if self.det_type.lower() == "rayonix": - radius = powder.shape[0] / 4 + radius = np.sqrt(2) * powder.shape[0] / 4 row, col = np.ogrid[: powder.shape[0], : powder.shape[1]] center = (powder.shape[0] / 2, powder.shape[1] / 2) mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius**2 @@ -820,13 +820,13 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): icol = 0 # Plotting score scan over distance - ax4 = plt.subplot2grid((nrow, ncol), (irow, icol)) - self.score_distance_scan(self.bounds, ax4) + ax5 = plt.subplot2grid((nrow, ncol), (irow, icol)) + self.score_distance_scan(self.bounds, ax5) icol += 1 # Plotting residual scan over distance - ax5 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - self.residual_distance_scan(self.bounds, ax5) + ax6 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) + self.residual_distance_scan(self.bounds, ax6) if plot != "": fig.savefig(plot, dpi=180) From f48f0d98090c813a2734a07503b4a31930c0a096 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 15:29:09 -0800 Subject: [PATCH 137/375] [edit] Find peaks in scores vs distance to estimate regions where best residual could be --- lute/tasks/geom_opt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index aa130e1b..a702e8bc 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -42,6 +42,7 @@ from sklearn.utils._testing import ignore_warnings from sklearn.exceptions import ConvergenceWarning from scipy.stats import norm +from scipy.signal import find_peaks from mpi4py import MPI @@ -635,6 +636,7 @@ def score_distance_scan(self, bounds, ax): mean = np.mean(scores[non_zero_scores]) std_dev = np.std(scores[non_zero_scores]) percentile_10 = np.percentile(scores[non_zero_scores], 10) + peaks, _ = find_peaks(scores, distance=5) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( @@ -653,6 +655,7 @@ def score_distance_scan(self, bounds, ax): linewidth=1.5, label=f"Mean - 1 Std ({mean - std_dev:.2f})", ) + ax.plot(distances[peaks], scores[peaks], "x") ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.legend(fontsize="x-small") From 9380a2c97482af07a96f2b05b1aedae57ce410f3 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 15:38:50 -0800 Subject: [PATCH 138/375] [edit] Add find_peaks method to compare possible best residuals --- lute/tasks/geom_opt.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index a702e8bc..939986d7 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -525,9 +525,9 @@ def finalize(self): logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - valid_indices = np.where(self.scan["score"] > percentile_10)[0] - shift_index = np.argmin(self.scan["residual"][valid_indices]) - index = valid_indices[shift_index] + peaks, _ = find_peaks(self.scan["score"], distance=2) + shift_index = np.argmin(self.scan["residual"][peaks]) + index = peaks[shift_index] self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] @@ -636,7 +636,7 @@ def score_distance_scan(self, bounds, ax): mean = np.mean(scores[non_zero_scores]) std_dev = np.std(scores[non_zero_scores]) percentile_10 = np.percentile(scores[non_zero_scores], 10) - peaks, _ = find_peaks(scores, distance=5) + peaks, _ = find_peaks(scores, distance=2) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( @@ -648,13 +648,6 @@ def score_distance_scan(self, bounds, ax): ax.axhline( mean, color="red", linestyle="--", linewidth=1.5, label=f"Mean ({mean:.2f})" ) - ax.axhline( - mean - std_dev, - color="orange", - linestyle="--", - linewidth=1.5, - label=f"Mean - 1 Std ({mean - std_dev:.2f})", - ) ax.plot(distances[peaks], scores[peaks], "x") ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") From fcbd45818354bba5ee4c5ab66b9cd546f5142039 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 15:59:31 -0800 Subject: [PATCH 139/375] [edit] Add find_peaks method to compare possible best residuals --- lute/tasks/geom_opt.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 939986d7..0cf44d3d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -525,7 +525,7 @@ def finalize(self): logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - peaks, _ = find_peaks(self.scan["score"], distance=2) + peaks, _ = find_peaks(self.scan["score"], distance=10, height=2*percentile_10, threshold=percentile_10) shift_index = np.argmin(self.scan["residual"][peaks]) index = peaks[shift_index] self.index = index @@ -636,7 +636,7 @@ def score_distance_scan(self, bounds, ax): mean = np.mean(scores[non_zero_scores]) std_dev = np.std(scores[non_zero_scores]) percentile_10 = np.percentile(scores[non_zero_scores], 10) - peaks, _ = find_peaks(scores, distance=2) + peaks, _ = find_peaks(scores, distance=10, height=2*percentile_10, threshold=percentile_10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( @@ -645,9 +645,6 @@ def score_distance_scan(self, bounds, ax): linestyle="--", label=f"10th Percentile: {percentile_10:.2e}", ) - ax.axhline( - mean, color="red", linestyle="--", linewidth=1.5, label=f"Mean ({mean:.2f})" - ) ax.plot(distances[peaks], scores[peaks], "x") ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") From c0982cbb72e2ef712b5754b459eee58b8a532b6b Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 21 Nov 2024 23:59:56 +0000 Subject: [PATCH 140/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0cf44d3d..d3d708c2 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -525,7 +525,12 @@ def finalize(self): logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - peaks, _ = find_peaks(self.scan["score"], distance=10, height=2*percentile_10, threshold=percentile_10) + peaks, _ = find_peaks( + self.scan["score"], + distance=10, + height=2 * percentile_10, + threshold=percentile_10, + ) shift_index = np.argmin(self.scan["residual"][peaks]) index = peaks[shift_index] self.index = index @@ -636,7 +641,9 @@ def score_distance_scan(self, bounds, ax): mean = np.mean(scores[non_zero_scores]) std_dev = np.std(scores[non_zero_scores]) percentile_10 = np.percentile(scores[non_zero_scores], 10) - peaks, _ = find_peaks(scores, distance=10, height=2*percentile_10, threshold=percentile_10) + peaks, _ = find_peaks( + scores, distance=10, height=2 * percentile_10, threshold=percentile_10 + ) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( From f89c49be9e241379a8d944fc9638356721f035a7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 18:23:16 -0800 Subject: [PATCH 141/375] [test] Threshold powder at mean + 5 std --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0cf44d3d..f11de13c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -179,7 +179,7 @@ def min_intensity(self, Imin, powder): masked_powder = np.ma.masked_array(powder, 0) mean = np.mean(masked_powder) std = np.std(masked_powder) - threshold = mean + 3 * std + threshold = mean + 5 * std nice_pix = masked_powder < threshold Imin = np.percentile(masked_powder[nice_pix], Imin) self.Imin = Imin From 89b25a611f0fb86be8ec87b882d8b70719996bcb Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 19:17:10 -0800 Subject: [PATCH 142/375] [test] find_peaks distance=5 height=percentile_10 --- lute/tasks/geom_opt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index fbbd7e95..027df635 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -527,9 +527,8 @@ def finalize(self): logger.info(f"10th Score Percentile: {percentile_10:.2e}") peaks, _ = find_peaks( self.scan["score"], - distance=10, - height=2 * percentile_10, - threshold=percentile_10, + distance=5, + height=percentile_10, ) shift_index = np.argmin(self.scan["residual"][peaks]) index = peaks[shift_index] From f02b95031f5d9b193b4e16043fb45ec3fe70f87f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 19:26:29 -0800 Subject: [PATCH 143/375] [test] threshold instead of height=percentile --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 027df635..abeb6c24 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -528,7 +528,7 @@ def finalize(self): peaks, _ = find_peaks( self.scan["score"], distance=5, - height=percentile_10, + threshold=percentile_10, ) shift_index = np.argmin(self.scan["residual"][peaks]) index = peaks[shift_index] From d6ff7912c0f275b0145d3a57d766864911097a59 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 21 Nov 2024 19:33:28 -0800 Subject: [PATCH 144/375] [test] no threshold only distance=5 --- lute/tasks/geom_opt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index abeb6c24..66c143c9 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -528,7 +528,6 @@ def finalize(self): peaks, _ = find_peaks( self.scan["score"], distance=5, - threshold=percentile_10, ) shift_index = np.argmin(self.scan["residual"][peaks]) index = peaks[shift_index] From 359ee12f7200ebd1ebc63fa7c00b84b95cd5ede2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 22 Nov 2024 12:44:07 -0800 Subject: [PATCH 145/375] [test] distance=5 height=percentile10 --- lute/tasks/geom_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 66c143c9..027df635 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -528,6 +528,7 @@ def finalize(self): peaks, _ = find_peaks( self.scan["score"], distance=5, + height=percentile_10, ) shift_index = np.argmin(self.scan["residual"][peaks]) index = peaks[shift_index] From 6f8ff19a328b99997a8eb9c2eac97fe1a475f4a4 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 14:49:51 -0800 Subject: [PATCH 146/375] [test] Testing new implementation for finding best Imin --- lute/tasks/geom_opt.py | 77 +++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 49 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 027df635..8cd26e77 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -164,27 +164,38 @@ def build_mask(self, central=True, edges=True): mask = np.reshape(mask, (mask.shape[0] * mask.shape[1], mask.shape[2])) return mask - def min_intensity(self, Imin, powder): + def min_intensity(self, powder): """ Define minimal intensity for control point extraction - Note: this is a heuristic that has been found to work well but may need some tuning. + + The minimal intensity is chosen so that the Signal to Noise Ratio (SNR) is maximized + Signal is defined as the standard deviation of the pixels above the threshold + Noise is defined as the standard deviation of the pixels below the threshold Parameters ---------- - Imin : float - Minimum intensity to use for control point extraction based on intensity distribution powder : np.ndarray Powder image """ - masked_powder = np.ma.masked_array(powder, 0) + masked_powder = powder[powder > 0] mean = np.mean(masked_powder) std = np.std(masked_powder) threshold = mean + 5 * std nice_pix = masked_powder < threshold - Imin = np.percentile(masked_powder[nice_pix], Imin) + self.hist = masked_powder[nice_pix] + SNRs= [] + Imins = np.arange(90, 100, 0.1) + for Imin in Imins: + threshold = np.percentile(masked_powder[nice_pix], Imin) + signal_pixels = masked_powder[nice_pix][masked_powder[nice_pix] > threshold] + signal = np.std(signal_pixels) + noise_pixels = masked_powder[nice_pix][masked_powder[nice_pix] <= threshold] + noise = np.std(noise_pixels) + SNRs.append(signal / noise) + self.q = round(Imins[np.argmax(SNRs)], 1) + Imin = np.percentile(masked_powder[nice_pix], self.q) self.Imin = Imin self.powder = powder - return Imin, powder @ignore_warnings(category=ConvergenceWarning) def bayes_opt_center( @@ -479,7 +490,7 @@ def bayes_opt_geom( powder = powder * mask self.max_rings = max_rings - Imin, powder = self.min_intensity(Imin, powder) + self.min_intensity(powder) if self.rank == 0: logger.info(f"Number of distances to scan: {self.size}") @@ -527,7 +538,7 @@ def finalize(self): logger.info(f"10th Score Percentile: {percentile_10:.2e}") peaks, _ = find_peaks( self.scan["score"], - distance=5, + distance=2, height=percentile_10, ) shift_index = np.argmin(self.scan["residual"][peaks]) @@ -637,12 +648,8 @@ def score_distance_scan(self, bounds, ax): """ scores = self.scan["score"] non_zero_scores = np.where(scores > 0)[0] - mean = np.mean(scores[non_zero_scores]) - std_dev = np.std(scores[non_zero_scores]) percentile_10 = np.percentile(scores[non_zero_scores], 10) - peaks, _ = find_peaks( - scores, distance=10, height=2 * percentile_10, threshold=percentile_10 - ) + peaks, _ = find_peaks(scores, distance=5, height=percentile_10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( @@ -684,7 +691,7 @@ def residual_distance_scan(self, bounds, ax): ax.set_ylabel("Residual") ax.set_title("Residual vs Distance") - def hist_and_compute_stats(self, powder, exp, run, ax): + def hist_and_compute_stats(self, exp, run, ax): """ Plot histogram of pixel intensities and compute statistics @@ -699,14 +706,10 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax : plt.Axes Matplotlib axes """ - mean = np.mean(powder) - threshold = np.mean(powder) + 2 * np.std(powder) - nice_pix = powder < threshold - mean = np.mean(powder[nice_pix]) - std_dev = np.std(powder[nice_pix]) - percentile_99 = np.percentile(powder[nice_pix], 99) + mean = np.mean(self.hist) + std_dev = np.std(self.hist) _ = ax.hist( - powder[nice_pix], + self.hist, bins=1000, color="skyblue", edgecolor="black", @@ -714,37 +717,13 @@ def hist_and_compute_stats(self, powder, exp, run, ax): label="Pixel Intensities", ) ax.axvline( - mean, color="red", linestyle="--", linewidth=1.5, label=f"Mean ({mean:.2f})" - ) - ax.axvline( - mean + std_dev, - color="orange", - linestyle="--", - linewidth=1.5, - label=f"Mean + 1 Std ({mean + std_dev:.2f})", - ) - ax.axvline( - mean + 2 * std_dev, - color="green", - linestyle="--", - linewidth=1.5, - label=f"Mean + 2 Std ({mean + 2 * std_dev:.2f})", - ) - ax.axvline( - mean + 3 * std_dev, - color="blue", - linestyle="--", - linewidth=1.5, - label=f"Mean + 3 Std ({mean + 3 * std_dev:.2f})", - ) - ax.axvline( - percentile_99, + self.imin, color="purple", linestyle=":", linewidth=1.5, - label=f"99th Percentile ({percentile_99:.2f})", + label=f"{self.q} th Percentile ({self.Imin:.2f})", ) - ax.set_xlim(0, mean + 5 * std_dev) + ax.set_xlim([0, mean + 5 * std_dev]) ax.set_xlabel("Pixel Intensity") ax.set_ylabel("Frequency") ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") From 5a178453cb9a6cb75b98333b6251932ba1b2b5b8 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 24 Nov 2024 22:50:16 +0000 Subject: [PATCH 147/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8cd26e77..09004902 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -167,7 +167,7 @@ def build_mask(self, central=True, edges=True): def min_intensity(self, powder): """ Define minimal intensity for control point extraction - + The minimal intensity is chosen so that the Signal to Noise Ratio (SNR) is maximized Signal is defined as the standard deviation of the pixels above the threshold Noise is defined as the standard deviation of the pixels below the threshold @@ -183,7 +183,7 @@ def min_intensity(self, powder): threshold = mean + 5 * std nice_pix = masked_powder < threshold self.hist = masked_powder[nice_pix] - SNRs= [] + SNRs = [] Imins = np.arange(90, 100, 0.1) for Imin in Imins: threshold = np.percentile(masked_powder[nice_pix], Imin) From 283eb07ead842fc408543ba637a61358b449b925 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 15:01:37 -0800 Subject: [PATCH 148/375] [test] Testing new implementation for finding best Imin --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8cd26e77..515ca4ec 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -762,7 +762,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): # Plotting histogram of pixel intensities ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - self.hist_and_compute_stats(powder, self.exp, self.run, ax2) + self.hist_and_compute_stats(self.exp, self.run, ax2) irow += 1 icol = 0 From 396059b9f8f2fb97cad8da2efdfc0163c6122aec Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 15:14:19 -0800 Subject: [PATCH 149/375] [test] Testing new implementation for finding best Imin --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 4d355764..83fe4fb5 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -717,7 +717,7 @@ def hist_and_compute_stats(self, exp, run, ax): label="Pixel Intensities", ) ax.axvline( - self.imin, + self.Imin, color="purple", linestyle=":", linewidth=1.5, From d35e53d9496bbb61961a17c2cc0268e52f9a3740 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 15:26:58 -0800 Subject: [PATCH 150/375] [test] Testing new implementation for finding best Imin --- lute/tasks/geom_opt.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 83fe4fb5..ae9589ff 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -536,13 +536,7 @@ def finalize(self): logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - peaks, _ = find_peaks( - self.scan["score"], - distance=2, - height=percentile_10, - ) - shift_index = np.argmin(self.scan["residual"][peaks]) - index = peaks[shift_index] + index = np.argmin(self.scan["residual"]) self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] @@ -649,7 +643,6 @@ def score_distance_scan(self, bounds, ax): scores = self.scan["score"] non_zero_scores = np.where(scores > 0)[0] percentile_10 = np.percentile(scores[non_zero_scores], 10) - peaks, _ = find_peaks(scores, distance=5, height=percentile_10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( @@ -658,7 +651,6 @@ def score_distance_scan(self, bounds, ax): linestyle="--", label=f"10th Percentile: {percentile_10:.2e}", ) - ax.plot(distances[peaks], scores[peaks], "x") ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.legend(fontsize="x-small") From ed2db6294e4c7c081ee5dc254df2e42ae3b6296c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 15:38:35 -0800 Subject: [PATCH 151/375] [test] Testing new implementation for finding best Imin --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index ae9589ff..f6f860db 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -536,7 +536,9 @@ def finalize(self): logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - index = np.argmin(self.scan["residual"]) + nice_scores = self.scan["score"] > percentile_10 + shift_index = np.argmin(self.scan["residual"][nice_scores]) + index = np.where(nice_scores)[0][shift_index] self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] From f072fecae5593bc2773ba875fbc4a60c8a844771 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 16:29:18 -0800 Subject: [PATCH 152/375] [test] Testing new implementation for finding best Imin --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index f6f860db..8354a442 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -184,7 +184,7 @@ def min_intensity(self, powder): nice_pix = masked_powder < threshold self.hist = masked_powder[nice_pix] SNRs = [] - Imins = np.arange(90, 100, 0.1) + Imins = np.arange(99, 100, 0.1) for Imin in Imins: threshold = np.percentile(masked_powder[nice_pix], Imin) signal_pixels = masked_powder[nice_pix][masked_powder[nice_pix] > threshold] From b1d41d445b2e47edde77cffee4209656cf15f217 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 16:44:49 -0800 Subject: [PATCH 153/375] [test] Testing new implementation for finding best Imin --- lute/tasks/geom_opt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8354a442..702ce443 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -180,11 +180,11 @@ def min_intensity(self, powder): masked_powder = powder[powder > 0] mean = np.mean(masked_powder) std = np.std(masked_powder) - threshold = mean + 5 * std + threshold = mean + 3 * std nice_pix = masked_powder < threshold - self.hist = masked_powder[nice_pix] + self.hist = masked_powder SNRs = [] - Imins = np.arange(99, 100, 0.1) + Imins = np.arange(90, 100, 0.1) for Imin in Imins: threshold = np.percentile(masked_powder[nice_pix], Imin) signal_pixels = masked_powder[nice_pix][masked_powder[nice_pix] > threshold] From 95fda13a017dddadc245dacba84375e51184398d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 16:58:54 -0800 Subject: [PATCH 154/375] [test] Testing new implementation for finding best Imin --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 702ce443..7d394736 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -182,7 +182,7 @@ def min_intensity(self, powder): std = np.std(masked_powder) threshold = mean + 3 * std nice_pix = masked_powder < threshold - self.hist = masked_powder + self.hist = masked_powder[nice_pix] SNRs = [] Imins = np.arange(90, 100, 0.1) for Imin in Imins: From 2edf7cfaa7e1ca9eb4d0707037122b8437c8160e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 17:14:07 -0800 Subject: [PATCH 155/375] [test] Testing new implementation for finding best Imin --- lute/tasks/geom_opt.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7d394736..d1ce2082 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -181,6 +181,7 @@ def min_intensity(self, powder): mean = np.mean(masked_powder) std = np.std(masked_powder) threshold = mean + 3 * std + logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = masked_powder < threshold self.hist = masked_powder[nice_pix] SNRs = [] @@ -710,6 +711,24 @@ def hist_and_compute_stats(self, exp, run, ax): alpha=0.7, label="Pixel Intensities", ) + ax.axvline( + mean, + color="red", + linestyle="--", + label=f"Mean ({mean:.2f})", + ) + ax.axvline( + mean + std_dev, + color="orange", + linestyle="--", + label=f"Mean + Std Dev ({mean + std_dev:.2f})", + ) + ax.axvline( + mean + 2 * std_dev, + color="yellow", + linestyle="--", + label=f"Mean + 2 Std Dev ({mean + 2 * std_dev:.2f})", + ) ax.axvline( self.Imin, color="purple", @@ -717,7 +736,7 @@ def hist_and_compute_stats(self, exp, run, ax): linewidth=1.5, label=f"{self.q} th Percentile ({self.Imin:.2f})", ) - ax.set_xlim([0, mean + 5 * std_dev]) + ax.set_xlim([0, mean + 3 * std_dev]) ax.set_xlabel("Pixel Intensity") ax.set_ylabel("Frequency") ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") From 47109c25f5bb6796bf4e5e2d9af2409f83786802 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 17:31:23 -0800 Subject: [PATCH 156/375] [edit] Test new implementation of Imin without taking non zero pixels --- lute/tasks/geom_opt.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d1ce2082..d72a9aa8 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -177,24 +177,22 @@ def min_intensity(self, powder): powder : np.ndarray Powder image """ - masked_powder = powder[powder > 0] - mean = np.mean(masked_powder) - std = np.std(masked_powder) - threshold = mean + 3 * std + mean = np.mean(powder) + std = np.std(powder) + threshold = mean + 5 * std logger.info(f"Threshold for pixel outliers: {threshold:.2e}") - nice_pix = masked_powder < threshold - self.hist = masked_powder[nice_pix] + nice_pix = powder < threshold SNRs = [] Imins = np.arange(90, 100, 0.1) for Imin in Imins: - threshold = np.percentile(masked_powder[nice_pix], Imin) - signal_pixels = masked_powder[nice_pix][masked_powder[nice_pix] > threshold] + threshold = np.percentile(powder[nice_pix], Imin) + signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] signal = np.std(signal_pixels) - noise_pixels = masked_powder[nice_pix][masked_powder[nice_pix] <= threshold] + noise_pixels = powder[nice_pix][powder[nice_pix] <= threshold] noise = np.std(noise_pixels) SNRs.append(signal / noise) self.q = round(Imins[np.argmax(SNRs)], 1) - Imin = np.percentile(masked_powder[nice_pix], self.q) + Imin = np.percentile(powder[nice_pix], self.q) self.Imin = Imin self.powder = powder @@ -646,6 +644,7 @@ def score_distance_scan(self, bounds, ax): scores = self.scan["score"] non_zero_scores = np.where(scores > 0)[0] percentile_10 = np.percentile(scores[non_zero_scores], 10) + mean = np.mean(scores[non_zero_scores]) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.axhline( @@ -654,6 +653,7 @@ def score_distance_scan(self, bounds, ax): linestyle="--", label=f"10th Percentile: {percentile_10:.2e}", ) + ax.axhline(mean, color="red", linestyle="--", label=f"Mean: {mean:.2e}") ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.legend(fontsize="x-small") @@ -686,7 +686,7 @@ def residual_distance_scan(self, bounds, ax): ax.set_ylabel("Residual") ax.set_title("Residual vs Distance") - def hist_and_compute_stats(self, exp, run, ax): + def hist_and_compute_stats(self, powder, exp, run, ax): """ Plot histogram of pixel intensities and compute statistics @@ -701,10 +701,14 @@ def hist_and_compute_stats(self, exp, run, ax): ax : plt.Axes Matplotlib axes """ - mean = np.mean(self.hist) - std_dev = np.std(self.hist) + mean = np.mean(powder) + threshold = np.mean(powder) + 3 * np.std(powder) + nice_pix = powder < threshold + mean = np.mean(powder[nice_pix]) + std_dev = np.std(powder[nice_pix]) + nice_pix = powder < threshold _ = ax.hist( - self.hist, + powder[nice_pix], bins=1000, color="skyblue", edgecolor="black", @@ -736,7 +740,7 @@ def hist_and_compute_stats(self, exp, run, ax): linewidth=1.5, label=f"{self.q} th Percentile ({self.Imin:.2f})", ) - ax.set_xlim([0, mean + 3 * std_dev]) + ax.set_xlim([0, mean + 5 * std_dev]) ax.set_xlabel("Pixel Intensity") ax.set_ylabel("Frequency") ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") @@ -775,7 +779,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): # Plotting histogram of pixel intensities ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - self.hist_and_compute_stats(self.exp, self.run, ax2) + self.hist_and_compute_stats(powder, self.exp, self.run, ax2) irow += 1 icol = 0 From ab3db4abd60ace951acc8c7a03d13d2dd21193d2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 18:17:37 -0800 Subject: [PATCH 157/375] [edit] Test new implementation of Imin without taking non zero pixels --- lute/tasks/geom_opt.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d72a9aa8..b9164a4b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -492,6 +492,7 @@ def bayes_opt_geom( self.min_intensity(powder) if self.rank == 0: + logger.info(f"Optimizing geometry for exp {self.exp} run {self.run} with {self.det_type} detector") logger.info(f"Number of distances to scan: {self.size}") self.bounds = bounds distances = np.linspace(bounds["dist"][0], bounds["dist"][1], self.size) @@ -535,10 +536,10 @@ def finalize(self): logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - nice_scores = self.scan["score"] > percentile_10 - shift_index = np.argmin(self.scan["residual"][nice_scores]) - index = np.where(nice_scores)[0][shift_index] - self.index = index + peaks, _ = find_peaks(self.scan["score"], height=percentile_10) + min_idx = np.argmin(self.scan["residual"][peaks]) + index = peaks[min_idx] + self.index = peaks[min_idx] self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] self.residual = self.scan["residual"][index] @@ -702,7 +703,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): Matplotlib axes """ mean = np.mean(powder) - threshold = np.mean(powder) + 3 * np.std(powder) + threshold = np.mean(powder) + 5 * np.std(powder) nice_pix = powder < threshold mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) @@ -729,7 +730,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ) ax.axvline( mean + 2 * std_dev, - color="yellow", + color="green", linestyle="--", label=f"Mean + 2 Std Dev ({mean + 2 * std_dev:.2f})", ) From 9ee9a4583982721fcb8602f4c68a98537355d456 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Mon, 25 Nov 2024 02:18:02 +0000 Subject: [PATCH 158/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index b9164a4b..7183f9f0 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -492,7 +492,9 @@ def bayes_opt_geom( self.min_intensity(powder) if self.rank == 0: - logger.info(f"Optimizing geometry for exp {self.exp} run {self.run} with {self.det_type} detector") + logger.info( + f"Optimizing geometry for exp {self.exp} run {self.run} with {self.det_type} detector" + ) logger.info(f"Number of distances to scan: {self.size}") self.bounds = bounds distances = np.linspace(bounds["dist"][0], bounds["dist"][1], self.size) @@ -537,7 +539,7 @@ def finalize(self): logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") peaks, _ = find_peaks(self.scan["score"], height=percentile_10) - min_idx = np.argmin(self.scan["residual"][peaks]) + min_idx = np.argmin(self.scan["residual"][peaks]) index = peaks[min_idx] self.index = peaks[min_idx] self.bo_history = self.scan["bo_history"][index] From a39f02a393d1e3ea301acaf4950380a47f69611c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 18:29:40 -0800 Subject: [PATCH 159/375] [edit] Add find peaks to retrieve nice residuals --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index b9164a4b..131114cb 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -645,16 +645,16 @@ def score_distance_scan(self, bounds, ax): scores = self.scan["score"] non_zero_scores = np.where(scores > 0)[0] percentile_10 = np.percentile(scores[non_zero_scores], 10) - mean = np.mean(scores[non_zero_scores]) + peaks, _ = find_peaks(scores, height=percentile_10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) + ax.plot(distances[peaks], scores[peaks], "x", color="red") ax.axhline( percentile_10, color="purple", linestyle="--", label=f"10th Percentile: {percentile_10:.2e}", ) - ax.axhline(mean, color="red", linestyle="--", label=f"Mean: {mean:.2e}") ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.legend(fontsize="x-small") From 9b3d6dde5632cfda49e0e19331cd8b0349bbd042 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 18:47:43 -0800 Subject: [PATCH 160/375] [edit] Fixing Imin overwriting --- lute/tasks/geom_opt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index e8181a4b..e9b11234 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -195,6 +195,7 @@ def min_intensity(self, powder): Imin = np.percentile(powder[nice_pix], self.q) self.Imin = Imin self.powder = powder + return Imin @ignore_warnings(category=ConvergenceWarning) def bayes_opt_center( @@ -489,11 +490,11 @@ def bayes_opt_geom( powder = powder * mask self.max_rings = max_rings - self.min_intensity(powder) + Imin = self.min_intensity(powder) if self.rank == 0: logger.info( - f"Optimizing geometry for exp {self.exp} run {self.run} with {self.det_type} detector" + f"Optimizing geometry for exp {self.exp} run {self.run} with {self.det_type} detector with minimal intensity threshold {Imin:.2e}" ) logger.info(f"Number of distances to scan: {self.size}") self.bounds = bounds From dd893f8926f3b5dc11de2d71a31a7eaa2bf76cf1 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 18:57:56 -0800 Subject: [PATCH 161/375] [edit] Fixing Imin overwriting --- lute/tasks/geom_opt.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index e9b11234..d369a8d3 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -539,10 +539,10 @@ def finalize(self): logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - peaks, _ = find_peaks(self.scan["score"], height=percentile_10) - min_idx = np.argmin(self.scan["residual"][peaks]) - index = peaks[min_idx] - self.index = peaks[min_idx] + nice_scores = self.scan["score"] > percentile_10 + min_idx = np.argmin(self.scan["residual"][nice_scores]) + index = nice_scores[min_idx] + self.index = nice_scores[min_idx] self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] self.residual = self.scan["residual"][index] @@ -648,10 +648,8 @@ def score_distance_scan(self, bounds, ax): scores = self.scan["score"] non_zero_scores = np.where(scores > 0)[0] percentile_10 = np.percentile(scores[non_zero_scores], 10) - peaks, _ = find_peaks(scores, height=percentile_10) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) - ax.plot(distances[peaks], scores[peaks], "x", color="red") ax.axhline( percentile_10, color="purple", From 648b145a042f99c26fccad9d0628eb7c4fa4f598 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 19:08:28 -0800 Subject: [PATCH 162/375] [edit] Fixing Imin overwriting --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d369a8d3..f95624e4 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -539,7 +539,7 @@ def finalize(self): logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - nice_scores = self.scan["score"] > percentile_10 + nice_scores = np.where(self.scan["score"] > percentile_10)[0] min_idx = np.argmin(self.scan["residual"][nice_scores]) index = nice_scores[min_idx] self.index = nice_scores[min_idx] From cc5b6c10bb75ee0b8b94232bd51db7e98206a5e8 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 22:17:01 -0800 Subject: [PATCH 163/375] [edit] Setting bounds for Imin from 95 to 99.9% --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index f95624e4..0f2e32c4 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -183,7 +183,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(90, 100, 0.1) + Imins = np.arange(95, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From 9524d79edea9d7165a33c989325629cebf11447b Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 23:12:31 -0800 Subject: [PATCH 164/375] [edit] Setting bounds for Imin from 95 to 99.9% --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0f2e32c4..0c0dba8d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -183,7 +183,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(95, 100, 0.1) + Imins = np.arange(99, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From ffc89375ee2a39bf6cafbc3e0a3fabecc2abc33c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 24 Nov 2024 23:45:25 -0800 Subject: [PATCH 165/375] [edit] Setting bounds for Imin from 95 to 99.9% + find_peaks for scores --- lute/tasks/geom_opt.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0c0dba8d..64b887bc 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -440,7 +440,7 @@ def bayes_opt_geom( powder, bounds, res, - Imin=90, + Imin=99, max_rings=5, n_samples=50, n_iterations=50, @@ -539,10 +539,10 @@ def finalize(self): logger.info(f"Mean Score: {mean:.2e}") logger.info(f"Score Std Dev: {std_dev:.2e}") logger.info(f"10th Score Percentile: {percentile_10:.2e}") - nice_scores = np.where(self.scan["score"] > percentile_10)[0] - min_idx = np.argmin(self.scan["residual"][nice_scores]) - index = nice_scores[min_idx] - self.index = nice_scores[min_idx] + peaks, _ = find_peaks(self.scan["score"]) + min_idx = np.argmin(self.scan["residual"][peaks]) + index = peaks[min_idx] + self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] self.residual = self.scan["residual"][index] @@ -646,16 +646,10 @@ def score_distance_scan(self, bounds, ax): Matplotlib axes """ scores = self.scan["score"] - non_zero_scores = np.where(scores > 0)[0] - percentile_10 = np.percentile(scores[non_zero_scores], 10) + peaks, _ = find_peaks(self.scan["score"]) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) - ax.axhline( - percentile_10, - color="purple", - linestyle="--", - label=f"10th Percentile: {percentile_10:.2e}", - ) + ax.plot(distances[peaks], scores[peaks], "x", label="Local Maxima") ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.legend(fontsize="x-small") From 0ddce76ae4f72cbd84da9a002e00e2fcc99923f7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 25 Nov 2024 00:00:53 -0800 Subject: [PATCH 166/375] [edit] Setting bounds for Imin from 95 to 99.9% + find_peaks for scores --- lute/tasks/geom_opt.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 64b887bc..7ee793a9 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -532,14 +532,7 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - non_zero_scores = np.where(self.scan["score"] > 0)[0] - percentile_10 = np.percentile(self.scan["score"][non_zero_scores], 10) - mean = np.mean(self.scan["score"][non_zero_scores]) - std_dev = np.std(self.scan["score"][non_zero_scores]) - logger.info(f"Mean Score: {mean:.2e}") - logger.info(f"Score Std Dev: {std_dev:.2e}") - logger.info(f"10th Score Percentile: {percentile_10:.2e}") - peaks, _ = find_peaks(self.scan["score"]) + peaks, _ = find_peaks(self.scan["score"], height=np.mean(self.scan["score"])) min_idx = np.argmin(self.scan["residual"][peaks]) index = peaks[min_idx] self.index = index @@ -646,7 +639,7 @@ def score_distance_scan(self, bounds, ax): Matplotlib axes """ scores = self.scan["score"] - peaks, _ = find_peaks(self.scan["score"]) + peaks, _ = find_peaks(self.scan["score"], height=np.mean(scores)) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.plot(distances[peaks], scores[peaks], "x", label="Local Maxima") From 54a3262408f253239a0ceb83412bd39300b0416c Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Mon, 25 Nov 2024 08:01:18 +0000 Subject: [PATCH 167/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7ee793a9..11763f94 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -532,7 +532,9 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - peaks, _ = find_peaks(self.scan["score"], height=np.mean(self.scan["score"])) + peaks, _ = find_peaks( + self.scan["score"], height=np.mean(self.scan["score"]) + ) min_idx = np.argmin(self.scan["residual"][peaks]) index = peaks[min_idx] self.index = index From d09b15f81db4cf675efdb74d5bce56babe4da26c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 25 Nov 2024 00:12:09 -0800 Subject: [PATCH 168/375] [edit] Setting bounds for Imin from 95 to 99.9% + find_peaks for scores and height=100 --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7ee793a9..0cc89010 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -532,7 +532,7 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - peaks, _ = find_peaks(self.scan["score"], height=np.mean(self.scan["score"])) + peaks, _ = find_peaks(self.scan["score"], height=100) min_idx = np.argmin(self.scan["residual"][peaks]) index = peaks[min_idx] self.index = index @@ -639,7 +639,7 @@ def score_distance_scan(self, bounds, ax): Matplotlib axes """ scores = self.scan["score"] - peaks, _ = find_peaks(self.scan["score"], height=np.mean(scores)) + peaks, _ = find_peaks(self.scan["score"], height=100) distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) ax.plot(distances[peaks], scores[peaks], "x", label="Local Maxima") From 515f95cf9a632d0d43551c4c88e9e51cfe53bf70 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 25 Nov 2024 10:13:36 -0800 Subject: [PATCH 169/375] [edit] Manually add adjacent peaks to find best res --- lute/tasks/geom_opt.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0cc89010..c253233b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -533,8 +533,12 @@ def finalize(self): for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) peaks, _ = find_peaks(self.scan["score"], height=100) - min_idx = np.argmin(self.scan["residual"][peaks]) - index = peaks[min_idx] + for peak in peaks: + close_peaks = np.insert(peaks, peak, peak-1) + close_peaks = np.insert(close_peaks, peak+1, peak+1) + close_peaks = np.unique(close_peaks) + min_idx = np.argmin(self.scan["residual"][close_peaks]) + index = close_peaks[min_idx] self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] From 41f540761f288aac25ad4ad84388f83f8c23c001 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Mon, 25 Nov 2024 18:14:07 +0000 Subject: [PATCH 170/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index c253233b..dac8fcd2 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -534,8 +534,8 @@ def finalize(self): self.scan[key] = np.array([item for item in self.scan[key]]) peaks, _ = find_peaks(self.scan["score"], height=100) for peak in peaks: - close_peaks = np.insert(peaks, peak, peak-1) - close_peaks = np.insert(close_peaks, peak+1, peak+1) + close_peaks = np.insert(peaks, peak, peak - 1) + close_peaks = np.insert(close_peaks, peak + 1, peak + 1) close_peaks = np.unique(close_peaks) min_idx = np.argmin(self.scan["residual"][close_peaks]) index = close_peaks[min_idx] From 5a49036a6ec742e2aecc310dcc867070434d9c23 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 25 Nov 2024 10:51:20 -0800 Subject: [PATCH 171/375] [edit] Manually add adjacent peaks to find best res --- lute/tasks/geom_opt.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index c253233b..d1c397e5 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -533,12 +533,13 @@ def finalize(self): for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) peaks, _ = find_peaks(self.scan["score"], height=100) + close_peaks = set(peaks) for peak in peaks: - close_peaks = np.insert(peaks, peak, peak-1) - close_peaks = np.insert(close_peaks, peak+1, peak+1) - close_peaks = np.unique(close_peaks) - min_idx = np.argmin(self.scan["residual"][close_peaks]) - index = close_peaks[min_idx] + close_peaks.add(peak - 1) + close_peaks.add(peak + 1) + peaks = sorted(list(close_peaks)) + min_idx = np.argmin(self.scan["residual"][peaks]) + index = peaks[min_idx] self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] From ba1e984c3e8d4075621705352e52a4b0db2f8bd4 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 25 Nov 2024 11:02:13 -0800 Subject: [PATCH 172/375] [edit] Manually add adjacent peaks to find best res --- lute/tasks/geom_opt.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 3bea575c..d1c397e5 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -535,19 +535,11 @@ def finalize(self): peaks, _ = find_peaks(self.scan["score"], height=100) close_peaks = set(peaks) for peak in peaks: -<<<<<<< HEAD close_peaks.add(peak - 1) close_peaks.add(peak + 1) peaks = sorted(list(close_peaks)) min_idx = np.argmin(self.scan["residual"][peaks]) index = peaks[min_idx] -======= - close_peaks = np.insert(peaks, peak, peak - 1) - close_peaks = np.insert(close_peaks, peak + 1, peak + 1) - close_peaks = np.unique(close_peaks) - min_idx = np.argmin(self.scan["residual"][close_peaks]) - index = close_peaks[min_idx] ->>>>>>> 41f540761f288aac25ad4ad84388f83f8c23c001 self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] From dc143487110c1cdea1c69bb87ef1fba9ed1e272f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 26 Nov 2024 11:07:37 -0800 Subject: [PATCH 173/375] [edit] Add distance option: either specifies dist bounds to search or specify guess distance to look around --- lute/io/models/geom_opt.py | 2 +- lute/tasks/geom_opt.py | 36 ++++++++++++++---------------------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index b0a46b0f..b09c2ed5 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -35,7 +35,7 @@ class BayesGeomOptParameters(BaseModel): bounds: Dict[str, Tuple[float, float]] = Field( { - "dist": (0.05, 0.5), + "dist": (0.02, 0.6), "poni1": (-0.01, 0.01), "poni2": (-0.01, 0.01), }, diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d1c397e5..4104bc0f 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -497,8 +497,11 @@ def bayes_opt_geom( f"Optimizing geometry for exp {self.exp} run {self.run} with {self.det_type} detector with minimal intensity threshold {Imin:.2e}" ) logger.info(f"Number of distances to scan: {self.size}") - self.bounds = bounds - distances = np.linspace(bounds["dist"][0], bounds["dist"][1], self.size) + if isinstance(bounds['dist'], float): + distances = np.linspace(bounds["dist"]-0.5, bounds["dist"]+0.5, self.size) + else: + distances = np.linspace(bounds["dist"][0], bounds["dist"][1], self.size) + self.distances = distances else: distances = None @@ -532,14 +535,7 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - peaks, _ = find_peaks(self.scan["score"], height=100) - close_peaks = set(peaks) - for peak in peaks: - close_peaks.add(peak - 1) - close_peaks.add(peak + 1) - peaks = sorted(list(close_peaks)) - min_idx = np.argmin(self.scan["residual"][peaks]) - index = peaks[min_idx] + index = np.argmin(self.scan["residual"]) self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] @@ -632,40 +628,36 @@ def radial_integration(self, result, calibrant=None, label=None, ax=None): ax.set_xlabel(unit.label) ax.set_ylabel("Intensity") - def score_distance_scan(self, bounds, ax): + def score_distance_scan(self, distances, ax): """ Plot the score scan over distance Parameters ---------- - bounds : dict - Dictionary of bounds for each parameter + distances : np.array + Array of distances ax : plt.Axes Matplotlib axes """ scores = self.scan["score"] - peaks, _ = find_peaks(self.scan["score"], height=100) - distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(scores)) ax.plot(distances, scores) - ax.plot(distances[peaks], scores[peaks], "x", label="Local Maxima") ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.legend(fontsize="x-small") ax.set_title("Number of Control Points vs Distance") - def residual_distance_scan(self, bounds, ax): + def residual_distance_scan(self, distances, ax): """ Plot the residual scan over distance Parameters ---------- - bounds : dict - Dictionary of bounds for each parameter + distances : np.array + Array of distances ax : plt.Axes Matplotlib axes """ residuals = self.scan["residual"] - distances = np.linspace(bounds["dist"][0], bounds["dist"][1], len(residuals)) ax.plot(distances, residuals) best_dist = distances[self.index] ax.axvline( @@ -810,12 +802,12 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): # Plotting score scan over distance ax5 = plt.subplot2grid((nrow, ncol), (irow, icol)) - self.score_distance_scan(self.bounds, ax5) + self.score_distance_scan(self.distances, ax5) icol += 1 # Plotting residual scan over distance ax6 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - self.residual_distance_scan(self.bounds, ax6) + self.residual_distance_scan(self.distances, ax6) if plot != "": fig.savefig(plot, dpi=180) From a8b118d0ce04769d9425d33c59ab94f4b3e29e34 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Tue, 26 Nov 2024 19:08:06 +0000 Subject: [PATCH 174/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 4104bc0f..02d0ab84 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -497,8 +497,10 @@ def bayes_opt_geom( f"Optimizing geometry for exp {self.exp} run {self.run} with {self.det_type} detector with minimal intensity threshold {Imin:.2e}" ) logger.info(f"Number of distances to scan: {self.size}") - if isinstance(bounds['dist'], float): - distances = np.linspace(bounds["dist"]-0.5, bounds["dist"]+0.5, self.size) + if isinstance(bounds["dist"], float): + distances = np.linspace( + bounds["dist"] - 0.5, bounds["dist"] + 0.5, self.size + ) else: distances = np.linspace(bounds["dist"][0], bounds["dist"][1], self.size) self.distances = distances From c20f1e47c9ef2f59b80b0c56c75a3eccccc101fb Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 26 Nov 2024 11:13:54 -0800 Subject: [PATCH 175/375] [edit] Remove bounds pydantic definition for more flexibility on dist bounds definition --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index b09c2ed5..d96b3c8d 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -33,7 +33,7 @@ class Config(TaskParameters.Config): class BayesGeomOptParameters(BaseModel): """Bayesian optimization hyperparameters.""" - bounds: Dict[str, Tuple[float, float]] = Field( + bounds: Dict[str] = Field( { "dist": (0.02, 0.6), "poni1": (-0.01, 0.01), From e204dc02fec6d6375eadf05aa91b78dd265c3501 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 26 Nov 2024 11:20:40 -0800 Subject: [PATCH 176/375] [edit] Refine bounds pydantic definition for more flexibility on dist bounds definition --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index d96b3c8d..23b80eaf 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -33,7 +33,7 @@ class Config(TaskParameters.Config): class BayesGeomOptParameters(BaseModel): """Bayesian optimization hyperparameters.""" - bounds: Dict[str] = Field( + bounds: Dict[str, Union[float, Tuple[float, float]]] = Field( { "dist": (0.02, 0.6), "poni1": (-0.01, 0.01), From 38e68b0d3f8e924cfb105ee5977e672006b6635b Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 26 Nov 2024 11:28:35 -0800 Subject: [PATCH 177/375] [edit] Wrong def for distance guess bounds my bad --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 02d0ab84..189891ab 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -499,7 +499,7 @@ def bayes_opt_geom( logger.info(f"Number of distances to scan: {self.size}") if isinstance(bounds["dist"], float): distances = np.linspace( - bounds["dist"] - 0.5, bounds["dist"] + 0.5, self.size + bounds["dist"] - 0.05, bounds["dist"] + 0.05, self.size ) else: distances = np.linspace(bounds["dist"][0], bounds["dist"][1], self.size) From 4b2bdea266ac20e84620173f46722a9dc5bcba6a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 27 Nov 2024 16:15:46 -0800 Subject: [PATCH 178/375] [add] Implemented SMD+GeomOpt workflow and updated task and task parameters to account for it --- lute/io/models/geom_opt.py | 28 ++-- lute/tasks/geom_opt.py | 193 ++++++++++++++++++++-------- workflows/airflow/geom_opt_pyfai.py | 41 ++++++ 3 files changed, 192 insertions(+), 70 deletions(-) create mode 100644 workflows/airflow/geom_opt_pyfai.py diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 23b80eaf..40d002a2 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -14,9 +14,10 @@ from pydantic import BaseModel, Field -from .base import TaskParameters, validator -from ..db import read_latest_db_entry +from lute.io.models.base import TaskParameters +from lute.io.models.validators import validate_smd_path +import psana from PSCalib.CalibFileFinder import CalibFileFinder @@ -47,13 +48,8 @@ class BayesGeomOptParameters(BaseModel): description="Resolution of the grid used to discretize the parameter search space.", ) - Imin: float = Field( - 99, - description="Minimum intensity threshold for the Bayesian optimization based on intensity distribution percentile.", - ) - max_rings: int = Field( - 10, + 5, description="Maximum number of rings to be used for the Bayesian optimization.", ) @@ -89,6 +85,8 @@ class BayesGeomOptParameters(BaseModel): None, description="Seed for the random number generator for potential reproducibility.", ) + + _find_smd_path = validate_smd_path("powder") exp: str = Field( "", @@ -173,21 +171,15 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: exp = values["exp"] run = values["run"] cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" - src = "MfxEndstation.0:Epix10ka2M.0" + dsname = f"exp={exp}:run={run}" + ds = psana.DataSource(dsname) + det = psana.Detector(values["det_type"], ds.env()) + src = det.name type = "geometry" cff = CalibFileFinder(cdir) in_file = cff.findCalibFile(src, type, run) return in_file - @validator("powder", always=True) - def validate_powder(cls, powder: str, values: Dict[str, Any]) -> str: - if powder == "": - work_dir = values["work_dir"] - powder: str = read_latest_db_entry( - f"{work_dir}/powder", "ComputePowder", "out_file" - ) - return powder - @validator("out_file", always=True) def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: if out_file == "": diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 189891ab..68c563e2 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -13,9 +13,12 @@ from lute.io.models.geom_opt import * from lute.tasks.task import * from lute.tasks.dataclasses import * +from lute.execution.logging import get_logger import psana - +import os +import logging +from typing import List, Optional, Tuple, Union, Any, cast import sys sys.path.append("/sdf/home/l/lconreux/LCLSGeom") @@ -26,12 +29,9 @@ get_beam_center, ) -import logging -from lute.execution.logging import get_logger - -logger: logging.Logger = get_logger(__name__) - +import h5py # type: ignore import numpy as np +import numpy.typing as npt import matplotlib.pyplot as plt from pyFAI.geometry import Geometry from pyFAI.goniometer import SingleGeometry @@ -45,6 +45,7 @@ from scipy.signal import find_peaks from mpi4py import MPI +logger: logging.Logger = get_logger(__name__) class BayesGeomOpt: """ @@ -814,7 +815,6 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): if plot != "": fig.savefig(plot, dpi=180) - class OptimizePyFAIGeometry(Task): """Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" @@ -823,50 +823,7 @@ def __init__( ) -> None: super().__init__(params=params, use_mpi=use_mpi) - def _run(self) -> None: - detector = self.build_pyFAI_detector() - optimizer = BayesGeomOpt( - exp=self._task_parameters.exp, - run=self._task_parameters.run, - det_type=self._task_parameters.det_type, - detector=detector, - calibrant=self._task_parameters.calibrant, - ) - optimizer.bayes_opt_geom( - powder=self._task_parameters.powder, - bounds=self._task_parameters.bo_params.bounds, - res=self._task_parameters.bo_params.res, - Imin=self._task_parameters.bo_params.Imin, - max_rings=self._task_parameters.bo_params.max_rings, - n_samples=self._task_parameters.bo_params.n_samples, - n_iterations=self._task_parameters.bo_params.n_iterations, - af=self._task_parameters.bo_params.af, - hyperparam=self._task_parameters.bo_params.hyperparams, - prior=self._task_parameters.bo_params.prior, - seed=self._task_parameters.bo_params.seed, - ) - if optimizer.rank == 0: - logger.info("Optimization complete") - distance, cx, cy = get_beam_center(optimizer.params) - logger.info(f"Detector Distance to Sample: {distance:.2e}") - logger.info(f"Beam center: ({cx:.2e}, {cy:.2e})") - logger.info( - f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" - ) - logger.info(f"Final Residuals: {optimizer.residual:.2e}") - plot = f"{self._task_parameters.work_dir}/figs/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" - detector = self.update_geometry(optimizer) - optimizer.visualize_results( - powder=optimizer.powder, - bo_history=optimizer.bo_history, - detector=detector, - params=optimizer.params, - plot=plot, - ) - self._result.payload = plot - self._result.task_status = TaskStatus.COMPLETED - - def build_pyFAI_detector(self): + def _build_pyFAI_detector(self): """ Fetch the geometry data and build a pyFAI detector object. """ @@ -891,9 +848,96 @@ def build_pyFAI_detector(self): detector = psana_to_pyfai.detector return detector - def update_geometry(self, optimizer): + def _check_if_path_and_type(self, string: str) -> Tuple[bool, Optional[str]]: + """ + Check if a string is a valid path and determine the filetype. + + Parameters + ---------- + string : str + String that may be a file path. + + Returns + ------- + is_valid_path : bool + If it is a valid path. + + powder_type : Optional[str] + If is_valid_path, the file type. + """ + is_valid_path: bool = False + powder_type: Optional[str] = None + if os.path.exists(string): + is_valid_path = True + else: + return is_valid_path, powder_type + try: + with h5py.File(string): + powder_type = "smd" + is_valid_path = True + + return is_valid_path, powder_type + except Exception: + ... + + try: + np.load(string) + powder_type = "numpy" + is_valid_path = True + return is_valid_path, powder_type + except ValueError: + ... + + return is_valid_path, powder_type + + def _extract_powder(self, powder_path: str, shape: Tuple) -> Optional[npt.NDArray[np.float64]]: + """ + Extract a powder image from either a smalldata file or numpy array. + + Parameters + ---------- + powder_path : str + Path to the object containing the powder image. + shape : Tuple + Stacked shape of the detector. Powder image has to be reshaped to match detector shape. + + Returns + ------- + powder : Optional[npt.NDArray[np.float64]] + The extracted powder image. + Returns None if no powder could be extracted and no specific error was encountered. + """ + powder = None + if isinstance(powder_path, str): + is_valid: bool + dtype: Optional[str] + is_valid, dtype = self._check_if_path_and_type(powder_path) + if is_valid and dtype == "numpy": + powder = np.load(powder_path) + if powder.shape != shape: + powder = np.reshape(powder, shape) + elif is_valid and dtype == "smd": + h5: h5py.File + with h5py.File(powder_path) as h5: + if self._task_parameters.det_type.lower() == "rayonix": + try: + powder: npt.NDArray[np.float64] = h5[f"Sums/Rayonix_calib_skipFirst_max"] + except: + powder: npt.NDArray[np.float64] = h5[f"Sums/Rayonix_calib_max"] + else: + powder: npt.NDArray[np.float64] = h5[f"Sums/{self._task_parameters.det_type}_calib_max"] + if powder.shape != shape: + powder: npt.NDArray[np.float64] = np.reshape(powder, shape) + return powder + + def _update_geometry(self, optimizer): """ Update the geometry and write a new .geom file and .data file + + Parameters + ---------- + optimizer : BayesGeomOpt + Optimizer object """ PyFAIToCrystFEL( detector=optimizer.detector, @@ -922,3 +966,48 @@ def update_geometry(self, optimizer): ) detector = psana_to_pyfai.detector return detector + + def _run(self) -> None: + detector = self._build_pyFAI_detector() + powder = self._extract_powder(self._task_parameters.powder, detector.shape) + if powder is None: + raise RuntimeError("Unable to extract powder. Cannot continue.") + optimizer = BayesGeomOpt( + exp=self._task_parameters.exp, + run=self._task_parameters.run, + det_type=self._task_parameters.det_type, + detector=detector, + calibrant=self._task_parameters.calibrant, + ) + optimizer.bayes_opt_geom( + powder=powder, + bounds=self._task_parameters.bo_params.bounds, + res=self._task_parameters.bo_params.res, + max_rings=self._task_parameters.bo_params.max_rings, + n_samples=self._task_parameters.bo_params.n_samples, + n_iterations=self._task_parameters.bo_params.n_iterations, + af=self._task_parameters.bo_params.af, + hyperparam=self._task_parameters.bo_params.hyperparams, + prior=self._task_parameters.bo_params.prior, + seed=self._task_parameters.bo_params.seed, + ) + if optimizer.rank == 0: + logger.info("Optimization complete") + distance, cx, cy = get_beam_center(optimizer.params) + logger.info(f"Detector Distance to Sample: {distance:.2e}") + logger.info(f"Beam center: ({cx:.2e}, {cy:.2e})") + logger.info( + f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" + ) + logger.info(f"Final Residuals: {optimizer.residual:.2e}") + plot = f"{self._task_parameters.work_dir}/figs/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" + calib_detector = self._update_geometry(optimizer) + optimizer.visualize_results( + powder=optimizer.powder, + bo_history=optimizer.bo_history, + detector=calib_detector, + params=optimizer.params, + plot=plot, + ) + self._result.summary.append(ElogSummaryPlots(f"Geometry_Fit/r{optimizer.run:0>4}", plot)) + self._result.task_status = TaskStatus.COMPLETED diff --git a/workflows/airflow/geom_opt_pyfai.py b/workflows/airflow/geom_opt_pyfai.py new file mode 100644 index 00000000..d5c8309d --- /dev/null +++ b/workflows/airflow/geom_opt_pyfai.py @@ -0,0 +1,41 @@ +"""Run geometry optimization for centering detector on the beam. + +Performs a Bayesian Optimization coupled with pyFAI least squares fitting of distance, beam center and tilt angles. + +Note: + The task_id MUST match the managed task name when defining DAGs - it is used + by the operator to properly launch it. + + dag_id names must be unique, and they are not namespaced via folder + hierarchy. I.e. all DAGs on an Airflow instance must have unique ids. The + Airflow instance used by LUTE is currently shared by other software - DAG + IDs should always be prefixed with `lute_`. LUTE scripts should append this + internally, so a DAG "lute_test" can be triggered by asking for "test" +""" + +from datetime import datetime +import os +from airflow import DAG +from lute.operators.jidoperators import JIDSlurmOperator + +dag_id: str = f"lute_{os.path.splitext(os.path.basename(__file__))[0]}" +description: str = ( + "Run geometry optimization based on given calibrant. Produce powder using " + "smalldata_tools." +) + +dag: DAG = DAG( + dag_id=dag_id, + start_date=datetime(2024, 9, 3), + schedule_interval=None, + description=description, + is_paused_upon_creation=False, +) + +smd_producer: JIDSlurmOperator = JIDSlurmOperator(task_id="SmallDataProducer", dag=dag) + +geom_optimizer: JIDSlurmOperator = JIDSlurmOperator(max_cores=128, task_id="PyFAIGeometryOptimizer", dag=dag) + + +# Powder production and geometry optimization +smd_producer >> geom_optimizer From 61a1d6bb9f90448bbb2ea6d42676593b539038db Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 28 Nov 2024 00:16:16 +0000 Subject: [PATCH 179/375] Auto-commit black formatting --- lute/io/models/geom_opt.py | 2 +- lute/tasks/geom_opt.py | 28 ++++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 40d002a2..6eba1f20 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -85,7 +85,7 @@ class BayesGeomOptParameters(BaseModel): None, description="Seed for the random number generator for potential reproducibility.", ) - + _find_smd_path = validate_smd_path("powder") exp: str = Field( diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 68c563e2..8a4a5787 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -47,6 +47,7 @@ logger: logging.Logger = get_logger(__name__) + class BayesGeomOpt: """ Class to perform Geometry Optimization using Bayesian Optimization on pyFAI @@ -815,6 +816,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): if plot != "": fig.savefig(plot, dpi=180) + class OptimizePyFAIGeometry(Task): """Optimize detector geometry using PyFAI coupled with Bayesian Optimization.""" @@ -859,10 +861,10 @@ def _check_if_path_and_type(self, string: str) -> Tuple[bool, Optional[str]]: Returns ------- - is_valid_path : bool + is_valid_path : bool If it is a valid path. - powder_type : Optional[str] + powder_type : Optional[str] If is_valid_path, the file type. """ is_valid_path: bool = False @@ -889,8 +891,10 @@ def _check_if_path_and_type(self, string: str) -> Tuple[bool, Optional[str]]: ... return is_valid_path, powder_type - - def _extract_powder(self, powder_path: str, shape: Tuple) -> Optional[npt.NDArray[np.float64]]: + + def _extract_powder( + self, powder_path: str, shape: Tuple + ) -> Optional[npt.NDArray[np.float64]]: """ Extract a powder image from either a smalldata file or numpy array. @@ -921,11 +925,17 @@ def _extract_powder(self, powder_path: str, shape: Tuple) -> Optional[npt.NDArra with h5py.File(powder_path) as h5: if self._task_parameters.det_type.lower() == "rayonix": try: - powder: npt.NDArray[np.float64] = h5[f"Sums/Rayonix_calib_skipFirst_max"] + powder: npt.NDArray[np.float64] = h5[ + f"Sums/Rayonix_calib_skipFirst_max" + ] except: - powder: npt.NDArray[np.float64] = h5[f"Sums/Rayonix_calib_max"] + powder: npt.NDArray[np.float64] = h5[ + f"Sums/Rayonix_calib_max" + ] else: - powder: npt.NDArray[np.float64] = h5[f"Sums/{self._task_parameters.det_type}_calib_max"] + powder: npt.NDArray[np.float64] = h5[ + f"Sums/{self._task_parameters.det_type}_calib_max" + ] if powder.shape != shape: powder: npt.NDArray[np.float64] = np.reshape(powder, shape) return powder @@ -1009,5 +1019,7 @@ def _run(self) -> None: params=optimizer.params, plot=plot, ) - self._result.summary.append(ElogSummaryPlots(f"Geometry_Fit/r{optimizer.run:0>4}", plot)) + self._result.summary.append( + ElogSummaryPlots(f"Geometry_Fit/r{optimizer.run:0>4}", plot) + ) self._result.task_status = TaskStatus.COMPLETED From 6a6bf492330bb2ac1ecf0a8be52e5e30aaa95c7a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 27 Nov 2024 19:13:06 -0800 Subject: [PATCH 180/375] [edit] max_cores=120 --- workflows/airflow/geom_opt_pyfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/airflow/geom_opt_pyfai.py b/workflows/airflow/geom_opt_pyfai.py index d5c8309d..8a3ad98e 100644 --- a/workflows/airflow/geom_opt_pyfai.py +++ b/workflows/airflow/geom_opt_pyfai.py @@ -34,7 +34,7 @@ smd_producer: JIDSlurmOperator = JIDSlurmOperator(task_id="SmallDataProducer", dag=dag) -geom_optimizer: JIDSlurmOperator = JIDSlurmOperator(max_cores=128, task_id="PyFAIGeometryOptimizer", dag=dag) +geom_optimizer: JIDSlurmOperator = JIDSlurmOperator(max_cores=120, task_id="PyFAIGeometryOptimizer", dag=dag) # Powder production and geometry optimization From 3b7701db153fb4cbf9aa7c23eb12f32a9f223f11 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 28 Nov 2024 11:57:30 -0800 Subject: [PATCH 181/375] [edit] Forgot to import validator from pydantic --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 6eba1f20..a5e8421f 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any, Dict, Optional, Union, Tuple -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, validator from lute.io.models.base import TaskParameters from lute.io.models.validators import validate_smd_path From c9a3cf2526ff0aa8dc67217080e7ff8f9670cd10 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 28 Nov 2024 13:18:02 -0800 Subject: [PATCH 182/375] [edit] Create fig folder if not already existing --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8a4a5787..c0639662 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -1010,7 +1010,9 @@ def _run(self) -> None: f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" ) logger.info(f"Final Residuals: {optimizer.residual:.2e}") - plot = f"{self._task_parameters.work_dir}/figs/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" + fig_folder = self._task_parameters.work_dir / "figs" + fig_folder.mkdir(parents=True, exist_ok=True) + plot = f"{fig_folder}/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" calib_detector = self._update_geometry(optimizer) optimizer.visualize_results( powder=optimizer.powder, From 877d0a5198106d22195543cb3cc5d750571712ce Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 28 Nov 2024 21:18:28 +0000 Subject: [PATCH 183/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index c0639662..055cb4f6 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -1012,7 +1012,9 @@ def _run(self) -> None: logger.info(f"Final Residuals: {optimizer.residual:.2e}") fig_folder = self._task_parameters.work_dir / "figs" fig_folder.mkdir(parents=True, exist_ok=True) - plot = f"{fig_folder}/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" + plot = ( + f"{fig_folder}/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" + ) calib_detector = self._update_geometry(optimizer) optimizer.visualize_results( powder=optimizer.powder, From b6eb5c352c55c61ebd9e66a28ff095ee9acec155 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 28 Nov 2024 13:22:22 -0800 Subject: [PATCH 184/375] [edit] Change validators for workflow handling --- lute/io/models/geom_opt.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index a5e8421f..6129384a 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -141,7 +141,7 @@ class BayesGeomOptParameters(BaseModel): @validator("exp", always=True) def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: - if exp == "": + if not exp: exp: str = values["lute_config"].experiment return exp @@ -149,25 +149,25 @@ def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: def validate_run( cls, run: Union[str, int], values: Dict[str, Any] ) -> Union[str, int]: - if run is None: + if not run: run: Union[str, int] = values["lute_config"].run return run @validator("date", always=True) def validate_date(cls, date: str, values: Dict[str, Any]) -> str: - if date == "": + if not date: date: str = values["lute_config"].date return date @validator("work_dir", always=True) def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: - if work_dir == "": + if not work_dir: work_dir: str = values["lute_config"].work_dir return work_dir @validator("in_file", always=True) def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: - if in_file == "": + if not in_file: exp = values["exp"] run = values["run"] cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" @@ -182,7 +182,7 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: @validator("out_file", always=True) def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: - if out_file == "": + if not out_file: in_file = values["in_file"] run = values["run"] out_file: str = in_file.replace("0-end.data", f"{run}-end.data") From 5fd57927e995b66cd56e4dc9135c9c31adb89a28 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 11:13:50 -0800 Subject: [PATCH 185/375] [edit] Add prints for debugging --- lute/io/models/geom_opt.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 6129384a..4d370015 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -142,6 +142,8 @@ class BayesGeomOptParameters(BaseModel): @validator("exp", always=True) def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: if not exp: + print("exp") + print(values.keys()) exp: str = values["lute_config"].experiment return exp @@ -150,24 +152,32 @@ def validate_run( cls, run: Union[str, int], values: Dict[str, Any] ) -> Union[str, int]: if not run: + print("run") + print(values.keys()) run: Union[str, int] = values["lute_config"].run return run @validator("date", always=True) def validate_date(cls, date: str, values: Dict[str, Any]) -> str: if not date: + print("date") + print(values.keys()) date: str = values["lute_config"].date return date @validator("work_dir", always=True) def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: if not work_dir: + print("work_dir") + print(values.keys()) work_dir: str = values["lute_config"].work_dir return work_dir @validator("in_file", always=True) def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: if not in_file: + print("in_file") + print(values.keys()) exp = values["exp"] run = values["run"] cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" From 24fe713341287e913fa4c72e17a29fa65ae2991e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 11:23:31 -0800 Subject: [PATCH 186/375] [edit] Add prints for debugging --- lute/io/models/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 4d370015..b91e462a 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -187,12 +187,14 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: src = det.name type = "geometry" cff = CalibFileFinder(cdir) - in_file = cff.findCalibFile(src, type, run) + in_file: str = cff.findCalibFile(src, type, run) return in_file @validator("out_file", always=True) def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: if not out_file: + print("out_file") + print(values.keys()) in_file = values["in_file"] run = values["run"] out_file: str = in_file.replace("0-end.data", f"{run}-end.data") From 7ec7dda412f1a5059ea9e04323403215ebb1bf50 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 29 Nov 2024 19:23:55 +0000 Subject: [PATCH 187/375] Auto-commit black formatting --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index b91e462a..099256c8 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -194,7 +194,7 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: if not out_file: print("out_file") - print(values.keys()) + print(values.keys()) in_file = values["in_file"] run = values["run"] out_file: str = in_file.replace("0-end.data", f"{run}-end.data") From c8ef29b52209c94f4ab9eff0f3e8aad0ed0589f8 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 11:30:30 -0800 Subject: [PATCH 188/375] [edit] Add prints for debugging --- lute/io/models/geom_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index b91e462a..24776f68 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -188,6 +188,7 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: type = "geometry" cff = CalibFileFinder(cdir) in_file: str = cff.findCalibFile(src, type, run) + print('in_file', in_file) return in_file @validator("out_file", always=True) From 7c372a7b32600ac9dcd0b1bac3ac70dcde573c8c Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 29 Nov 2024 19:30:58 +0000 Subject: [PATCH 189/375] Auto-commit black formatting --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index d2f2d91f..b762fb4e 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -188,7 +188,7 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: type = "geometry" cff = CalibFileFinder(cdir) in_file: str = cff.findCalibFile(src, type, run) - print('in_file', in_file) + print("in_file", in_file) return in_file @validator("out_file", always=True) From 858d4170531e0bf05ec36a64c1863387ba3be503 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 11:41:20 -0800 Subject: [PATCH 190/375] [edit] Fix validator in_file --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index d2f2d91f..eca42bff 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -181,7 +181,7 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: exp = values["exp"] run = values["run"] cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" - dsname = f"exp={exp}:run={run}" + dsname = f"exp={exp}:run={run}:idx" ds = psana.DataSource(dsname) det = psana.Detector(values["det_type"], ds.env()) src = det.name From 9eca5c100ca383c79eba103a622935616a771712 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 11:48:05 -0800 Subject: [PATCH 191/375] [edit] Fix validator in_file --- lute/io/models/geom_opt.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index f6a96301..5fe0b596 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -142,8 +142,6 @@ class BayesGeomOptParameters(BaseModel): @validator("exp", always=True) def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: if not exp: - print("exp") - print(values.keys()) exp: str = values["lute_config"].experiment return exp @@ -152,50 +150,39 @@ def validate_run( cls, run: Union[str, int], values: Dict[str, Any] ) -> Union[str, int]: if not run: - print("run") - print(values.keys()) run: Union[str, int] = values["lute_config"].run return run @validator("date", always=True) def validate_date(cls, date: str, values: Dict[str, Any]) -> str: if not date: - print("date") - print(values.keys()) date: str = values["lute_config"].date return date @validator("work_dir", always=True) def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: if not work_dir: - print("work_dir") - print(values.keys()) work_dir: str = values["lute_config"].work_dir return work_dir @validator("in_file", always=True) def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: if not in_file: - print("in_file") - print(values.keys()) exp = values["exp"] run = values["run"] cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" dsname = f"exp={exp}:run={run}:idx" ds = psana.DataSource(dsname) det = psana.Detector(values["det_type"], ds.env()) - src = det.name + src = str(det.name) type = "geometry" cff = CalibFileFinder(cdir) in_file: str = cff.findCalibFile(src, type, run) - print("in_file", in_file) return in_file @validator("out_file", always=True) def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: if not out_file: - print("out_file") - print(values.keys()) in_file = values["in_file"] run = values["run"] out_file: str = in_file.replace("0-end.data", f"{run}-end.data") From 9dfb05e9269180980bb48b86a272e6e077dc64e7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 11:53:12 -0800 Subject: [PATCH 192/375] [edit] Fix validator in_file --- lute/io/models/geom_opt.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 5fe0b596..f5d5b5b1 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -169,13 +169,19 @@ def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: if not in_file: exp = values["exp"] + print(exp) run = values["run"] + print(run) + det_type = values["det_type"] + print(det_type) cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" dsname = f"exp={exp}:run={run}:idx" ds = psana.DataSource(dsname) - det = psana.Detector(values["det_type"], ds.env()) + det = psana.Detector(det_type, ds.env()) src = str(det.name) + print(src) type = "geometry" + print(type) cff = CalibFileFinder(cdir) in_file: str = cff.findCalibFile(src, type, run) return in_file From e797818de90aca391ba2ab20741b0462e58bda8e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 11:58:23 -0800 Subject: [PATCH 193/375] [edit] Fix validator in_file --- lute/io/models/geom_opt.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index f5d5b5b1..b00c4d1c 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -169,21 +169,17 @@ def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: if not in_file: exp = values["exp"] - print(exp) run = values["run"] - print(run) det_type = values["det_type"] - print(det_type) cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" dsname = f"exp={exp}:run={run}:idx" ds = psana.DataSource(dsname) det = psana.Detector(det_type, ds.env()) src = str(det.name) - print(src) type = "geometry" - print(type) cff = CalibFileFinder(cdir) in_file: str = cff.findCalibFile(src, type, run) + print(in_file) return in_file @validator("out_file", always=True) From 72cdae44d6599f0e45fc5425f042d9359f16644a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 14:23:44 -0800 Subject: [PATCH 194/375] [edit] Fix validator in_file --- lute/io/models/geom_opt.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index b00c4d1c..61bada06 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -18,7 +18,7 @@ from lute.io.models.validators import validate_smd_path import psana -from PSCalib.CalibFileFinder import CalibFileFinder +from PSCalib.CalibFileFinder import find_calib_file class OptimizePyFAIGeometryParameters(TaskParameters): @@ -169,16 +169,18 @@ def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: if not in_file: exp = values["exp"] + print(exp) run = values["run"] + print(run) det_type = values["det_type"] - cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" - dsname = f"exp={exp}:run={run}:idx" - ds = psana.DataSource(dsname) + print(det_type) + cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' + ds_args = f"exp={exp}:run={run}:idx" + ds = psana.DataSource(ds_args) det = psana.Detector(det_type, ds.env()) src = str(det.name) - type = "geometry" - cff = CalibFileFinder(cdir) - in_file: str = cff.findCalibFile(src, type, run) + type = 'geometry' + in_file = find_calib_file(cdir, src, type, run, pbits=1) print(in_file) return in_file From 3e318c58f4ccc5d7acd222ca025ee84dcd021ae1 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 29 Nov 2024 22:24:07 +0000 Subject: [PATCH 195/375] Auto-commit black formatting --- lute/io/models/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 61bada06..c568d4d2 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -174,12 +174,12 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: print(run) det_type = values["det_type"] print(det_type) - cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' + cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" ds_args = f"exp={exp}:run={run}:idx" ds = psana.DataSource(ds_args) det = psana.Detector(det_type, ds.env()) src = str(det.name) - type = 'geometry' + type = "geometry" in_file = find_calib_file(cdir, src, type, run, pbits=1) print(in_file) return in_file From 081b34d4fa8b14ec3c1a2c9a6d8e1f22b4e5d832 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 14:28:21 -0800 Subject: [PATCH 196/375] [edit] Fix validator in_file --- lute/io/models/geom_opt.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 61bada06..9ff8247e 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -169,11 +169,8 @@ def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: if not in_file: exp = values["exp"] - print(exp) run = values["run"] - print(run) det_type = values["det_type"] - print(det_type) cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' ds_args = f"exp={exp}:run={run}:idx" ds = psana.DataSource(ds_args) @@ -181,7 +178,6 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: src = str(det.name) type = 'geometry' in_file = find_calib_file(cdir, src, type, run, pbits=1) - print(in_file) return in_file @validator("out_file", always=True) From 44ea70bc13a887ecb3a67aa40ffbda9083e71f1c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 14:28:53 -0800 Subject: [PATCH 197/375] [edit] Fix validator in_file --- lute/io/models/geom_opt.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 9d949e75..200b8398 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -171,12 +171,7 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: exp = values["exp"] run = values["run"] det_type = values["det_type"] -<<<<<<< HEAD cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' -======= - print(det_type) - cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" ->>>>>>> 3e318c58f4ccc5d7acd222ca025ee84dcd021ae1 ds_args = f"exp={exp}:run={run}:idx" ds = psana.DataSource(ds_args) det = psana.Detector(det_type, ds.env()) From 10d6ac269db24d4ef289ba1a3bb26876f957958c Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 29 Nov 2024 22:29:16 +0000 Subject: [PATCH 198/375] Auto-commit black formatting --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 200b8398..e748967b 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -171,7 +171,7 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: exp = values["exp"] run = values["run"] det_type = values["det_type"] - cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' + cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" ds_args = f"exp={exp}:run={run}:idx" ds = psana.DataSource(ds_args) det = psana.Detector(det_type, ds.env()) From 124cc2614ac675393ee5c3d18cf7a5a26aaf4e4e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 15:11:28 -0800 Subject: [PATCH 199/375] [edit] Fix validator in_file --- lute/io/models/geom_opt.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 200b8398..673c6364 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -172,12 +172,20 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: run = values["run"] det_type = values["det_type"] cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' + print(cdir) ds_args = f"exp={exp}:run={run}:idx" + print(ds_args) ds = psana.DataSource(ds_args) + print(ds) + print(ds.env()) det = psana.Detector(det_type, ds.env()) + print(det) + print(det.name) src = str(det.name) + print(src) type = "geometry" - in_file = find_calib_file(cdir, src, type, run, pbits=1) + print(find_calib_file(cdir, src, type, run, pbits=1)) + in_file: str = find_calib_file(cdir, src, type, run, pbits=1) return in_file @validator("out_file", always=True) From 6905260c20d616641cc1c1ab9b411ab7da59dbe7 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 29 Nov 2024 23:12:18 +0000 Subject: [PATCH 200/375] Auto-commit black formatting --- lute/io/models/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 673c6364..9a91dbb4 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -171,7 +171,7 @@ def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: exp = values["exp"] run = values["run"] det_type = values["det_type"] - cdir = f'/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib' + cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" print(cdir) ds_args = f"exp={exp}:run={run}:idx" print(ds_args) From a213ba2080e156c864bb2875eaedabe67e55187b Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 15:12:33 -0800 Subject: [PATCH 201/375] [edit] Fix extract powder --- lute/tasks/geom_opt.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 055cb4f6..9c14e0ed 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -923,21 +923,11 @@ def _extract_powder( elif is_valid and dtype == "smd": h5: h5py.File with h5py.File(powder_path) as h5: - if self._task_parameters.det_type.lower() == "rayonix": - try: - powder: npt.NDArray[np.float64] = h5[ - f"Sums/Rayonix_calib_skipFirst_max" - ] - except: - powder: npt.NDArray[np.float64] = h5[ - f"Sums/Rayonix_calib_max" - ] - else: - powder: npt.NDArray[np.float64] = h5[ - f"Sums/{self._task_parameters.det_type}_calib_max" - ] - if powder.shape != shape: - powder: npt.NDArray[np.float64] = np.reshape(powder, shape) + powder: npt.NDArray[np.float64] = h5[ + f"Sums/{self._task_parameters.det_type}_calib" + ] + if powder.shape != shape: + powder: npt.NDArray[np.float64] = np.reshape(powder, shape) return powder def _update_geometry(self, optimizer): From 6ac704986cfdd78e3258740b8e918d34523c7456 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 15:17:04 -0800 Subject: [PATCH 202/375] [edit] Fix extract powder --- lute/tasks/geom_opt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 9c14e0ed..04430b1a 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -483,8 +483,6 @@ def bayes_opt_geom( if seed is not None: np.random.seed(seed) - powder = np.load(powder) - self.build_calibrant() mask = self.build_mask() From 3e20c993e97258597c70871457a8a0daf3942021 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 29 Nov 2024 15:20:42 -0800 Subject: [PATCH 203/375] [edit] Fix extract powder --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 04430b1a..6b760de0 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -923,7 +923,7 @@ def _extract_powder( with h5py.File(powder_path) as h5: powder: npt.NDArray[np.float64] = h5[ f"Sums/{self._task_parameters.det_type}_calib" - ] + ][()] if powder.shape != shape: powder: npt.NDArray[np.float64] = np.reshape(powder, shape) return powder From 441a806f8433abc61d8da63ac5be3076ba0175c4 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sat, 30 Nov 2024 15:26:06 -0800 Subject: [PATCH 204/375] [edit] Add validate_calib_path as custom validator + change get wavelength to handle both photon energy conversion and direct fetching of wl --- lute/io/models/geom_opt.py | 27 +++------------------------ lute/io/models/validators.py | 32 ++++++++++++++++++++++++++++++-- lute/tasks/geom_opt.py | 16 +++++++++++----- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 9a91dbb4..6ad59ac7 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -15,7 +15,7 @@ from pydantic import BaseModel, Field, validator from lute.io.models.base import TaskParameters -from lute.io.models.validators import validate_smd_path +from lute.io.models.validators import validate_smd_path, validate_calib_path import psana from PSCalib.CalibFileFinder import find_calib_file @@ -88,6 +88,8 @@ class BayesGeomOptParameters(BaseModel): _find_smd_path = validate_smd_path("powder") + _find_in_file_path = validate_calib_path("in_file") + exp: str = Field( "", description="Experiment name.", @@ -165,29 +167,6 @@ def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: work_dir: str = values["lute_config"].work_dir return work_dir - @validator("in_file", always=True) - def validate_in_file(cls, in_file: str, values: Dict[str, Any]) -> str: - if not in_file: - exp = values["exp"] - run = values["run"] - det_type = values["det_type"] - cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" - print(cdir) - ds_args = f"exp={exp}:run={run}:idx" - print(ds_args) - ds = psana.DataSource(ds_args) - print(ds) - print(ds.env()) - det = psana.Detector(det_type, ds.env()) - print(det) - print(det.name) - src = str(det.name) - print(src) - type = "geometry" - print(find_calib_file(cdir, src, type, run, pbits=1)) - in_file: str = find_calib_file(cdir, src, type, run, pbits=1) - return in_file - @validator("out_file", always=True) def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: if not out_file: diff --git a/lute/io/models/validators.py b/lute/io/models/validators.py index 569afc58..182acee7 100644 --- a/lute/io/models/validators.py +++ b/lute/io/models/validators.py @@ -5,8 +5,8 @@ parameters for validation. """ -__all__ = ["template_parameter_validator", "validate_smd_path"] -__author__ = "Gabriel Dorlhiac" +__all__ = ["template_parameter_validator", "validate_smd_path", "validate_calib_path"] +__author__ = ["Gabriel Dorlhiac", "Louis Conreux"] import os from typing import Dict, Any, Optional @@ -15,6 +15,8 @@ from lute.io.db import read_latest_db_entry +import psana +from PSCalib.CalibFileFinder import find_calib_file def template_parameter_validator(template_params_name: str): """Populates a TaskParameters model with a set of validated TemplateParameters. @@ -68,3 +70,29 @@ def _validate_smd_path(cls, smd_path: str, values: Dict[str, Any]) -> str: return smd_path return validator(smd_path_name, always=True, allow_reuse=True)(_validate_smd_path) + + +def validate_calib_path(calib_path_name: str): + """Finds the path to a valid calibration file or raises an error.""" + + def _validate_calib_path(cls, calib_path: str, values: Dict[str, Any]) -> str: + if calib_path == "": + exp: str = values["lute_config"].experiment + run: int = int(values["lute_config"].run) + try: + det_type: str = values["det_type"] + except: + det_type: str = values["detname"] + cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" + ds_args = f"exp={exp}:run={run}:idx" + ds = psana.DataSource(ds_args) + det = psana.Detector(det_type, ds.env()) + src = str(det.name) + type = "geometry" + calib_path = find_calib_file(cdir, src, type, run, pbits=1) + if calib_path is None: + raise ValueError(f"No calibration file found for {det_type} in {cdir}") + + return calib_path + + return validator(calib_path_name, always=True, allow_reuse=True)(_validate_calib_path) \ No newline at end of file diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 6b760de0..d27fc5a0 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -121,9 +121,9 @@ def contextual_improvement(X, gp_model, best_y, hyperparam=None): ci = y_pred - best_y * norm.cdf(z) + y_std * norm.pdf(z) return ci - def build_calibrant(self): + def set_wavelength_calibrant(self): """ - Define calibrant for optimization + Define calibrant for optimization with appropriate wavelength Parameters ---------- @@ -134,8 +134,14 @@ def build_calibrant(self): calibrant = CALIBRANT_FACTORY(self.calibrant) ds_args = f"exp={self.exp}:run={self.run}:idx" ds = psana.DataSource(ds_args) - self.wavelength = ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 - photon_energy = 1.23984197386209e-09 / self.wavelength + det = psana.Detector(self.det_type, ds.env()) + runner = next(ds.runs()) + evt = runner.event(runner.times()[0]) + photon_energy = det('EBeam').get(evt).ebeamPhotonEnergy() + if photon_energy is None or np.isinf(photon_energy): + self.wavelength = ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 + else: + self.wavelength = 1.23984197386209e-09 / photon_energy self.photon_energy = photon_energy calibrant.wavelength = self.wavelength self.calibrant = calibrant @@ -483,7 +489,7 @@ def bayes_opt_geom( if seed is not None: np.random.seed(seed) - self.build_calibrant() + self.set_wavelength_calibrant() mask = self.build_mask() if mask is not None: From dd0a6ab4bb9810ad312863f3739e8ba921c06f01 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sat, 30 Nov 2024 23:26:30 +0000 Subject: [PATCH 205/375] Auto-commit black formatting --- lute/io/models/validators.py | 5 ++++- lute/tasks/geom_opt.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lute/io/models/validators.py b/lute/io/models/validators.py index 182acee7..1ad7c96d 100644 --- a/lute/io/models/validators.py +++ b/lute/io/models/validators.py @@ -18,6 +18,7 @@ import psana from PSCalib.CalibFileFinder import find_calib_file + def template_parameter_validator(template_params_name: str): """Populates a TaskParameters model with a set of validated TemplateParameters. @@ -95,4 +96,6 @@ def _validate_calib_path(cls, calib_path: str, values: Dict[str, Any]) -> str: return calib_path - return validator(calib_path_name, always=True, allow_reuse=True)(_validate_calib_path) \ No newline at end of file + return validator(calib_path_name, always=True, allow_reuse=True)( + _validate_calib_path + ) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d27fc5a0..7658f4a9 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -137,7 +137,7 @@ def set_wavelength_calibrant(self): det = psana.Detector(self.det_type, ds.env()) runner = next(ds.runs()) evt = runner.event(runner.times()[0]) - photon_energy = det('EBeam').get(evt).ebeamPhotonEnergy() + photon_energy = det("EBeam").get(evt).ebeamPhotonEnergy() if photon_energy is None or np.isinf(photon_energy): self.wavelength = ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 else: From 7c1621277c3ba62634cb9536721d80b16970bc7e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sat, 30 Nov 2024 15:33:52 -0800 Subject: [PATCH 206/375] [edit] Fix fetching wavelength from EBeam --- lute/tasks/geom_opt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d27fc5a0..89a025b1 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -137,7 +137,10 @@ def set_wavelength_calibrant(self): det = psana.Detector(self.det_type, ds.env()) runner = next(ds.runs()) evt = runner.event(runner.times()[0]) - photon_energy = det('EBeam').get(evt).ebeamPhotonEnergy() + try: + photon_energy = psana.Detector('EBeam').get(evt).ebeamPhotonEnergy() + except AttributeError as e: + logger.warning("Event lacking an ebeamPhotonEnergy value.") if photon_energy is None or np.isinf(photon_energy): self.wavelength = ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 else: From 30a0d3ac79597e4fd3cfe65fa081e1b047d30ef8 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sat, 30 Nov 2024 23:34:40 +0000 Subject: [PATCH 207/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 89a025b1..38c7ff31 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -138,7 +138,7 @@ def set_wavelength_calibrant(self): runner = next(ds.runs()) evt = runner.event(runner.times()[0]) try: - photon_energy = psana.Detector('EBeam').get(evt).ebeamPhotonEnergy() + photon_energy = psana.Detector("EBeam").get(evt).ebeamPhotonEnergy() except AttributeError as e: logger.warning("Event lacking an ebeamPhotonEnergy value.") if photon_energy is None or np.isinf(photon_energy): From 1a95eec95d7af6905fda47a515a74fc608841b61 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sat, 30 Nov 2024 15:50:56 -0800 Subject: [PATCH 208/375] [edit] Fig folder creation fixing --- lute/tasks/geom_opt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 89a025b1..90f5ea23 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -1007,8 +1007,8 @@ def _run(self) -> None: f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" ) logger.info(f"Final Residuals: {optimizer.residual:.2e}") - fig_folder = self._task_parameters.work_dir / "figs" - fig_folder.mkdir(parents=True, exist_ok=True) + fig_folder = os.path.join(self._task_parameters.work_dir, "figs") + os.makedirs(fig_folder, exist_ok=True) plot = ( f"{fig_folder}/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" ) @@ -1020,6 +1020,7 @@ def _run(self) -> None: params=optimizer.params, plot=plot, ) + self._result.payload = plot self._result.summary.append( ElogSummaryPlots(f"Geometry_Fit/r{optimizer.run:0>4}", plot) ) From 358d686c8ee6ce3ede6cb5de297d3d28a0f17111 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sat, 30 Nov 2024 23:47:37 -0800 Subject: [PATCH 209/375] [edit] Using right smalldata producer to get max powder --- lute/tasks/geom_opt.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index def529a4..62f6e962 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -930,9 +930,18 @@ def _extract_powder( elif is_valid and dtype == "smd": h5: h5py.File with h5py.File(powder_path) as h5: - powder: npt.NDArray[np.float64] = h5[ - f"Sums/{self._task_parameters.det_type}_calib" - ][()] + try: + if self._task_parameters.det_type == "Rayonix": + powder: npt.NDArray[np.float64] = h5[ + f"Sums/{self._task_parameters.det_type}_calib_skipFirst_max" + ][()] + powder: npt.NDArray[np.float64] = h5[ + f"Sums/{self._task_parameters.det_type}_calib_max" + ][()] + except KeyError: + logger.warning('No "Max" powder found in SmallData. Using "Sum" powder.') + powder: npt.NDArray[np.float64] = h5[ + f"Sums/{self._task_parameters.det_type}_calib"][()] if powder.shape != shape: powder: npt.NDArray[np.float64] = np.reshape(powder, shape) return powder From 44a3aefa9c4b1860f8f162362118d85c60c93aa4 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 1 Dec 2024 07:48:05 +0000 Subject: [PATCH 210/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 62f6e962..149f9461 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -934,14 +934,17 @@ def _extract_powder( if self._task_parameters.det_type == "Rayonix": powder: npt.NDArray[np.float64] = h5[ f"Sums/{self._task_parameters.det_type}_calib_skipFirst_max" - ][()] + ][()] powder: npt.NDArray[np.float64] = h5[ f"Sums/{self._task_parameters.det_type}_calib_max" ][()] except KeyError: - logger.warning('No "Max" powder found in SmallData. Using "Sum" powder.') + logger.warning( + 'No "Max" powder found in SmallData. Using "Sum" powder.' + ) powder: npt.NDArray[np.float64] = h5[ - f"Sums/{self._task_parameters.det_type}_calib"][()] + f"Sums/{self._task_parameters.det_type}_calib" + ][()] if powder.shape != shape: powder: npt.NDArray[np.float64] = np.reshape(powder, shape) return powder From 417bb9f334f8ac2907d35f9578f255bc9e9f28d4 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 1 Dec 2024 00:58:05 -0800 Subject: [PATCH 211/375] [edit] Prevent from cloning smalldata_tools before running SmallDataProducer --- lute/managed_tasks.py | 4 ++-- lute/tasks/geom_opt.py | 12 +++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lute/managed_tasks.py b/lute/managed_tasks.py index e44c5ec4..936879d3 100644 --- a/lute/managed_tasks.py +++ b/lute/managed_tasks.py @@ -39,13 +39,13 @@ ################### SmallDataProducer: Executor = Executor("SubmitSMD") """Runs the production of a smalldata HDF5 file.""" -SmallDataProducer.add_tasklet( +"""SmallDataProducer.add_tasklet( clone_smalldata, ["{{ producer }}"], when="before", set_result=False, set_summary=False, -) +)""" SmallDataXSSAnalyzer: MPIExecutor = MPIExecutor("AnalyzeSmallDataXSS") diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 62f6e962..7670f869 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -934,10 +934,11 @@ def _extract_powder( if self._task_parameters.det_type == "Rayonix": powder: npt.NDArray[np.float64] = h5[ f"Sums/{self._task_parameters.det_type}_calib_skipFirst_max" - ][()] - powder: npt.NDArray[np.float64] = h5[ - f"Sums/{self._task_parameters.det_type}_calib_max" - ][()] + ][()] + else: + powder: npt.NDArray[np.float64] = h5[ + f"Sums/{self._task_parameters.det_type}_calib_max" + ][()] except KeyError: logger.warning('No "Max" powder found in SmallData. Using "Sum" powder.') powder: npt.NDArray[np.float64] = h5[ @@ -1030,7 +1031,4 @@ def _run(self) -> None: plot=plot, ) self._result.payload = plot - self._result.summary.append( - ElogSummaryPlots(f"Geometry_Fit/r{optimizer.run:0>4}", plot) - ) self._result.task_status = TaskStatus.COMPLETED From 51e2daea82be90cff4819b6b4e752b521a03c0b6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 1 Dec 2024 01:42:54 -0800 Subject: [PATCH 212/375] [edit] Relaxed intensity search --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index e0fdebcb..854f9990 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -190,11 +190,11 @@ def min_intensity(self, powder): """ mean = np.mean(powder) std = np.std(powder) - threshold = mean + 5 * std + threshold = mean + 3 * std logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(99, 100, 0.1) + Imins = np.arange(95, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From c7a60455d18c03adc9e943a225bdfb813857c597 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 1 Dec 2024 02:28:13 -0800 Subject: [PATCH 213/375] [edit] Relaxed intensity search --- lute/tasks/geom_opt.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 854f9990..44eca4a1 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -188,13 +188,11 @@ def min_intensity(self, powder): powder : np.ndarray Powder image """ - mean = np.mean(powder) - std = np.std(powder) - threshold = mean + 3 * std + threshold = np.percentile(powder, 95) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(95, 100, 0.1) + Imins = np.arange(98, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From c5fd45b199b85eab01d676291908edd1feffde2a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 1 Dec 2024 02:34:14 -0800 Subject: [PATCH 214/375] [edit] Relaxed intensity search --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 44eca4a1..962a01f2 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -188,7 +188,7 @@ def min_intensity(self, powder): powder : np.ndarray Powder image """ - threshold = np.percentile(powder, 95) + threshold = np.percentile(powder, 99) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] From 7c7d4f9e261d2c363c6217b14ac3a2789850a3aa Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 1 Dec 2024 02:49:52 -0800 Subject: [PATCH 215/375] [edit] Relaxed intensity search --- lute/managed_tasks.py | 4 ++-- lute/tasks/geom_opt.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lute/managed_tasks.py b/lute/managed_tasks.py index 936879d3..e44c5ec4 100644 --- a/lute/managed_tasks.py +++ b/lute/managed_tasks.py @@ -39,13 +39,13 @@ ################### SmallDataProducer: Executor = Executor("SubmitSMD") """Runs the production of a smalldata HDF5 file.""" -"""SmallDataProducer.add_tasklet( +SmallDataProducer.add_tasklet( clone_smalldata, ["{{ producer }}"], when="before", set_result=False, set_summary=False, -)""" +) SmallDataXSSAnalyzer: MPIExecutor = MPIExecutor("AnalyzeSmallDataXSS") diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 962a01f2..a50bdcd6 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -188,7 +188,7 @@ def min_intensity(self, powder): powder : np.ndarray Powder image """ - threshold = np.percentile(powder, 99) + threshold = np.percentile(powder, 99.9) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] From 1f709b274ee075c62a39082fdf7be6d2e9ce1620 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 3 Dec 2024 09:39:51 -0800 Subject: [PATCH 216/375] [edit] Allow for more flexibility in Imin search --- lute/tasks/geom_opt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index a50bdcd6..35afcd4c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -188,11 +188,12 @@ def min_intensity(self, powder): powder : np.ndarray Powder image """ - threshold = np.percentile(powder, 99.9) + mean = np.mean(powder) + threshold = mean + 10 * np.std(powder) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(98, 100, 0.1) + Imins = np.arange(95, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] @@ -696,8 +697,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax : plt.Axes Matplotlib axes """ - mean = np.mean(powder) - threshold = np.mean(powder) + 5 * np.std(powder) + threshold = np.mean(powder) + 10 * np.std(powder) nice_pix = powder < threshold mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) From 1d287f9dd457110c6c8c30ba0f13f968ac5c49ca Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 3 Dec 2024 10:11:14 -0800 Subject: [PATCH 217/375] [edit] Allow for more flexibility in Imin search --- lute/tasks/geom_opt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 35afcd4c..f5ca9596 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -189,11 +189,11 @@ def min_intensity(self, powder): Powder image """ mean = np.mean(powder) - threshold = mean + 10 * np.std(powder) + threshold = mean + 5 * np.std(powder) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(95, 100, 0.1) + Imins = np.arange(95, 100, 0.01) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] @@ -653,7 +653,6 @@ def score_distance_scan(self, distances, ax): ax.plot(distances, scores) ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") - ax.legend(fontsize="x-small") ax.set_title("Number of Control Points vs Distance") def residual_distance_scan(self, distances, ax): From 1cd52d360adfde73f5f53d6d8eb769e9a61d50fb Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 3 Dec 2024 10:26:39 -0800 Subject: [PATCH 218/375] [edit] Allow for more flexibility in Imin search --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index f5ca9596..9ba2077e 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -696,7 +696,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax : plt.Axes Matplotlib axes """ - threshold = np.mean(powder) + 10 * np.std(powder) + threshold = np.mean(powder) + 5 * np.std(powder) nice_pix = powder < threshold mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) From e92f29e898c4a85ee39edc779900b1e475af3ca6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 3 Dec 2024 11:56:35 -0800 Subject: [PATCH 219/375] [edit] Fix scaling error in converting energy_photon to wavelength --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 9ba2077e..bb64557c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -144,7 +144,7 @@ def set_wavelength_calibrant(self): if photon_energy is None or np.isinf(photon_energy): self.wavelength = ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 else: - self.wavelength = 1.23984197386209e-09 / photon_energy + self.wavelength = 1.23984197386209e-06 / photon_energy self.photon_energy = photon_energy calibrant.wavelength = self.wavelength self.calibrant = calibrant From f03f1e6819344bc50727840238e2dede8857f394 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 4 Dec 2024 09:35:24 -0800 Subject: [PATCH 220/375] [test] 6 STDS --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index bb64557c..8895eb56 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -189,7 +189,7 @@ def min_intensity(self, powder): Powder image """ mean = np.mean(powder) - threshold = mean + 5 * np.std(powder) + threshold = mean + 6 * np.std(powder) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] From 8033e2dca25b5fca8553534c70808c04ce5dfd45 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 4 Dec 2024 09:54:54 -0800 Subject: [PATCH 221/375] [test] 5 STDS --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8895eb56..d93da86c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -189,7 +189,7 @@ def min_intensity(self, powder): Powder image """ mean = np.mean(powder) - threshold = mean + 6 * np.std(powder) + threshold = mean + 5 * np.std(powder) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] @@ -201,7 +201,7 @@ def min_intensity(self, powder): noise_pixels = powder[nice_pix][powder[nice_pix] <= threshold] noise = np.std(noise_pixels) SNRs.append(signal / noise) - self.q = round(Imins[np.argmax(SNRs)], 1) + self.q = round(Imins[np.argmax(SNRs)], 2) Imin = np.percentile(powder[nice_pix], self.q) self.Imin = Imin self.powder = powder From 7aabca52b0d70f5a5a974d50a89471cb994589fe Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 4 Dec 2024 10:03:41 -0800 Subject: [PATCH 222/375] [test] 6 STDS --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d93da86c..6a80e0ca 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -189,7 +189,7 @@ def min_intensity(self, powder): Powder image """ mean = np.mean(powder) - threshold = mean + 5 * np.std(powder) + threshold = mean + 6 * np.std(powder) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] From ba7ac44192d3b75e6c184eba36053ab228f33ad0 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 4 Dec 2024 10:04:15 -0800 Subject: [PATCH 223/375] [test] 6 STDS --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 6a80e0ca..fab2d3cf 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -696,7 +696,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax : plt.Axes Matplotlib axes """ - threshold = np.mean(powder) + 5 * np.std(powder) + threshold = np.mean(powder) + 6 * np.std(powder) nice_pix = powder < threshold mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) From 1373113528d626062d9cbbff076e6a558e3dbd5b Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 4 Dec 2024 11:01:51 -0800 Subject: [PATCH 224/375] [test] 8 STDS --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index fab2d3cf..016be83f 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -189,7 +189,7 @@ def min_intensity(self, powder): Powder image """ mean = np.mean(powder) - threshold = mean + 6 * np.std(powder) + threshold = mean + 8 * np.std(powder) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] @@ -696,7 +696,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax : plt.Axes Matplotlib axes """ - threshold = np.mean(powder) + 6 * np.std(powder) + threshold = np.mean(powder) + 8 * np.std(powder) nice_pix = powder < threshold mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) From ad9518d3a62b2befcb2f45cec9dcd11e9584103d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 4 Dec 2024 11:04:04 -0800 Subject: [PATCH 225/375] [test] 10 STDS --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 016be83f..efaadd66 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -189,7 +189,7 @@ def min_intensity(self, powder): Powder image """ mean = np.mean(powder) - threshold = mean + 8 * np.std(powder) + threshold = mean + 10 * np.std(powder) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] @@ -696,7 +696,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax : plt.Axes Matplotlib axes """ - threshold = np.mean(powder) + 8 * np.std(powder) + threshold = np.mean(powder) + 10 * np.std(powder) nice_pix = powder < threshold mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) From 250404a064bebef586287bfddf838b324cf1a306 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 4 Dec 2024 16:16:35 -0800 Subject: [PATCH 226/375] [edit] Handle case where final residual computation is impossible since no control points found for best_score --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index efaadd66..762b8a88 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -433,8 +433,10 @@ def bayes_opt_center( ) sg.extract_cp(max_rings=max_rings, pts_per_deg=1, Imin=Imin) self.sg = sg - residual = sg.geometry_refinement.refine3(fix=["wavelength"]) score = len(sg.geometry_refinement.data) + residual = 0 + if score != 0: + residual = sg.geometry_refinement.refine3(fix=["wavelength"]) params = sg.geometry_refinement.param result = { "bo_history": bo_history, From 711f816b128b374e77b27d065e8362bafefe4551 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 4 Dec 2024 16:17:39 -0800 Subject: [PATCH 227/375] [edit] back to 5 STDS --- lute/tasks/geom_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 762b8a88..428cd3fc 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -189,7 +189,7 @@ def min_intensity(self, powder): Powder image """ mean = np.mean(powder) - threshold = mean + 10 * np.std(powder) + threshold = mean + 5 * np.std(powder) logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] @@ -698,7 +698,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax : plt.Axes Matplotlib axes """ - threshold = np.mean(powder) + 10 * np.std(powder) + threshold = np.mean(powder) + 5 * np.std(powder) nice_pix = powder < threshold mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) From a2b679dae7634b42f3e1e35e26189c196eac34f6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 6 Dec 2024 12:09:44 -0800 Subject: [PATCH 228/375] [edit] Fix unnecessary imports to pass PR tests --- lute/io/models/geom_opt.py | 6 ------ lute/io/models/validators.py | 2 +- lute/tasks/geom_opt.py | 17 +++++++---------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 6ad59ac7..defc4481 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -8,8 +8,6 @@ __all__ = ["OptimizePyFAIGeometryParameters"] __author__ = "Louis Conreux" -import os -from pathlib import Path from typing import Any, Dict, Optional, Union, Tuple from pydantic import BaseModel, Field, validator @@ -17,10 +15,6 @@ from lute.io.models.base import TaskParameters from lute.io.models.validators import validate_smd_path, validate_calib_path -import psana -from PSCalib.CalibFileFinder import find_calib_file - - class OptimizePyFAIGeometryParameters(TaskParameters): """Parameters for optimizing detector geometry using PyFAI and Bayesian optimization. diff --git a/lute/io/models/validators.py b/lute/io/models/validators.py index 1ad7c96d..3ea72531 100644 --- a/lute/io/models/validators.py +++ b/lute/io/models/validators.py @@ -82,7 +82,7 @@ def _validate_calib_path(cls, calib_path: str, values: Dict[str, Any]) -> str: run: int = int(values["lute_config"].run) try: det_type: str = values["det_type"] - except: + except KeyError: det_type: str = values["detname"] cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" ds_args = f"exp={exp}:run={run}:idx" diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 428cd3fc..09098278 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -9,16 +9,15 @@ __all__ = ["OptimizePyFAIGeometry"] __author__ = "Louis Conreux" -from lute.execution.ipc import Message -from lute.io.models.geom_opt import * -from lute.tasks.task import * -from lute.tasks.dataclasses import * +from lute.io.models.geom_opt import OptimizePyFAIGeometryParameters +from lute.tasks.task import Task +from lute.tasks.dataclasses import TaskStatus from lute.execution.logging import get_logger import psana import os import logging -from typing import List, Optional, Tuple, Union, Any, cast +from typing import Optional, Tuple import sys sys.path.append("/sdf/home/l/lconreux/LCLSGeom") @@ -42,7 +41,6 @@ from sklearn.utils._testing import ignore_warnings from sklearn.exceptions import ConvergenceWarning from scipy.stats import norm -from scipy.signal import find_peaks from mpi4py import MPI logger: logging.Logger = get_logger(__name__) @@ -134,12 +132,11 @@ def set_wavelength_calibrant(self): calibrant = CALIBRANT_FACTORY(self.calibrant) ds_args = f"exp={self.exp}:run={self.run}:idx" ds = psana.DataSource(ds_args) - det = psana.Detector(self.det_type, ds.env()) runner = next(ds.runs()) evt = runner.event(runner.times()[0]) try: photon_energy = psana.Detector("EBeam").get(evt).ebeamPhotonEnergy() - except AttributeError as e: + except AttributeError: logger.warning("Event lacking an ebeamPhotonEnergy value.") if photon_energy is None or np.isinf(photon_energy): self.wavelength = ds.env().epicsStore().value("SIOC:SYS0:ML00:AO192") * 1e-9 @@ -168,7 +165,7 @@ def build_mask(self, central=True, edges=True): runnum = evt.run() try: mask = det.mask_v2(par=runnum, central=central, edges=edges) - except: + except AttributeError: mask = None if mask is not None: if len(mask.shape) != 2: @@ -613,7 +610,7 @@ def radial_integration(self, result, calibrant=None, label=None, ax=None): try: unit = result.unit - except: + except AttributeError: unit = None if len(result) == 3: ax.errorbar(*result, label=label) From b8fbbd118ebaf2271d262b754573ff4147057bb2 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 6 Dec 2024 20:10:15 +0000 Subject: [PATCH 229/375] Auto-commit black formatting --- lute/io/models/geom_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index defc4481..c89e5680 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -15,6 +15,7 @@ from lute.io.models.base import TaskParameters from lute.io.models.validators import validate_smd_path, validate_calib_path + class OptimizePyFAIGeometryParameters(TaskParameters): """Parameters for optimizing detector geometry using PyFAI and Bayesian optimization. From 4fbc029fe02a9895076c0bf795cff748ba8681a2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 6 Dec 2024 18:37:57 -0800 Subject: [PATCH 230/375] [edit] Add type: ignore to fix mypy test --- lute/io/models/geom_opt.py | 2 +- lute/io/models/validators.py | 4 ++-- lute/tasks/geom_opt.py | 24 ++++++++++++------------ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index c89e5680..1071e526 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -151,7 +151,7 @@ def validate_run( return run @validator("date", always=True) - def validate_date(cls, date: str, values: Dict[str, Any]) -> str: + def validate_date(cls, date, values: Dict[str, Any]) -> str: if not date: date: str = values["lute_config"].date return date diff --git a/lute/io/models/validators.py b/lute/io/models/validators.py index 3ea72531..685aadb7 100644 --- a/lute/io/models/validators.py +++ b/lute/io/models/validators.py @@ -15,8 +15,8 @@ from lute.io.db import read_latest_db_entry -import psana -from PSCalib.CalibFileFinder import find_calib_file +import psana # type: ignore +from PSCalib.CalibFileFinder import find_calib_file # type: ignore def template_parameter_validator(template_params_name: str): diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 09098278..40d73048 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -14,14 +14,14 @@ from lute.tasks.dataclasses import TaskStatus from lute.execution.logging import get_logger -import psana +import psana # type: ignore import os import logging from typing import Optional, Tuple import sys sys.path.append("/sdf/home/l/lconreux/LCLSGeom") -from LCLSGeom.swap_geom import ( +from LCLSGeom.swap_geom import ( # type: ignore PsanaToPyFAI, PyFAIToCrystFEL, CrystFELToPsana, @@ -31,16 +31,16 @@ import h5py # type: ignore import numpy as np import numpy.typing as npt -import matplotlib.pyplot as plt -from pyFAI.geometry import Geometry -from pyFAI.goniometer import SingleGeometry -from pyFAI.azimuthalIntegrator import AzimuthalIntegrator -from pyFAI.calibrant import CALIBRANT_FACTORY -from sklearn.gaussian_process import GaussianProcessRegressor -from sklearn.gaussian_process.kernels import RBF, ConstantKernel, WhiteKernel -from sklearn.utils._testing import ignore_warnings -from sklearn.exceptions import ConvergenceWarning -from scipy.stats import norm +import matplotlib.pyplot as plt # type: ignore +from pyFAI.geometry import Geometry # type: ignore +from pyFAI.goniometer import SingleGeometry # type: ignore +from pyFAI.azimuthalIntegrator import AzimuthalIntegrator # type: ignore +from pyFAI.calibrant import CALIBRANT_FACTORY # type: ignore +from sklearn.gaussian_process import GaussianProcessRegressor # type: ignore +from sklearn.gaussian_process.kernels import RBF, ConstantKernel, WhiteKernel # type: ignore +from sklearn.utils._testing import ignore_warnings # type: ignore +from sklearn.exceptions import ConvergenceWarning # type: ignore +from scipy.stats import norm # type: ignore from mpi4py import MPI logger: logging.Logger = get_logger(__name__) From 1bc84983d785de7d0b9ea21533fd5eb9c5a17974 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sat, 7 Dec 2024 02:38:24 +0000 Subject: [PATCH 231/375] Auto-commit black formatting --- lute/io/models/validators.py | 4 ++-- lute/tasks/geom_opt.py | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lute/io/models/validators.py b/lute/io/models/validators.py index 685aadb7..170c6815 100644 --- a/lute/io/models/validators.py +++ b/lute/io/models/validators.py @@ -15,8 +15,8 @@ from lute.io.db import read_latest_db_entry -import psana # type: ignore -from PSCalib.CalibFileFinder import find_calib_file # type: ignore +import psana # type: ignore +from PSCalib.CalibFileFinder import find_calib_file # type: ignore def template_parameter_validator(template_params_name: str): diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 40d73048..14f41085 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -14,14 +14,14 @@ from lute.tasks.dataclasses import TaskStatus from lute.execution.logging import get_logger -import psana # type: ignore +import psana # type: ignore import os import logging from typing import Optional, Tuple import sys sys.path.append("/sdf/home/l/lconreux/LCLSGeom") -from LCLSGeom.swap_geom import ( # type: ignore +from LCLSGeom.swap_geom import ( # type: ignore PsanaToPyFAI, PyFAIToCrystFEL, CrystFELToPsana, @@ -31,16 +31,16 @@ import h5py # type: ignore import numpy as np import numpy.typing as npt -import matplotlib.pyplot as plt # type: ignore -from pyFAI.geometry import Geometry # type: ignore -from pyFAI.goniometer import SingleGeometry # type: ignore -from pyFAI.azimuthalIntegrator import AzimuthalIntegrator # type: ignore -from pyFAI.calibrant import CALIBRANT_FACTORY # type: ignore -from sklearn.gaussian_process import GaussianProcessRegressor # type: ignore -from sklearn.gaussian_process.kernels import RBF, ConstantKernel, WhiteKernel # type: ignore -from sklearn.utils._testing import ignore_warnings # type: ignore -from sklearn.exceptions import ConvergenceWarning # type: ignore -from scipy.stats import norm # type: ignore +import matplotlib.pyplot as plt # type: ignore +from pyFAI.geometry import Geometry # type: ignore +from pyFAI.goniometer import SingleGeometry # type: ignore +from pyFAI.azimuthalIntegrator import AzimuthalIntegrator # type: ignore +from pyFAI.calibrant import CALIBRANT_FACTORY # type: ignore +from sklearn.gaussian_process import GaussianProcessRegressor # type: ignore +from sklearn.gaussian_process.kernels import RBF, ConstantKernel, WhiteKernel # type: ignore +from sklearn.utils._testing import ignore_warnings # type: ignore +from sklearn.exceptions import ConvergenceWarning # type: ignore +from scipy.stats import norm # type: ignore from mpi4py import MPI logger: logging.Logger = get_logger(__name__) From 09d85bd9654ff097862c152c024ae6288a0936ca Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 9 Dec 2024 15:27:30 -0800 Subject: [PATCH 232/375] [edit] Fix mypy testing through asserting correct pydantic type --- lute/tasks/geom_opt.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 40d73048..1f1402e5 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -832,6 +832,7 @@ def _build_pyFAI_detector(self): """ Fetch the geometry data and build a pyFAI detector object. """ + assert isinstance(self._task_parameters, OptimizePyFAIGeometryParameters) in_file = self._task_parameters.in_file det_type = self._task_parameters.det_type ds_args = f"exp={self._task_parameters.exp}:run={self._task_parameters.run}:idx" @@ -915,6 +916,7 @@ def _extract_powder( Returns None if no powder could be extracted and no specific error was encountered. """ powder = None + assert isinstance(self._task_parameters, OptimizePyFAIGeometryParameters) if isinstance(powder_path, str): is_valid: bool dtype: Optional[str] @@ -955,6 +957,7 @@ def _update_geometry(self, optimizer): optimizer : BayesGeomOpt Optimizer object """ + assert isinstance(self._task_parameters, OptimizePyFAIGeometryParameters) PyFAIToCrystFEL( detector=optimizer.detector, params=optimizer.params, @@ -984,6 +987,7 @@ def _update_geometry(self, optimizer): return detector def _run(self) -> None: + assert isinstance(self._task_parameters, OptimizePyFAIGeometryParameters) detector = self._build_pyFAI_detector() powder = self._extract_powder(self._task_parameters.powder, detector.shape) if powder is None: From c5a71110e365f49f648d49106954906e25ed7569 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 9 Dec 2024 15:55:51 -0800 Subject: [PATCH 233/375] [edit] Fix mypy testing through asserting correct pydantic type --- lute/io/models/geom_opt.py | 27 ++++++++------------------- lute/tasks/geom_opt.py | 14 +++++++------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index 1071e526..ec4397ae 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -100,11 +100,6 @@ class BayesGeomOptParameters(BaseModel): description="Detector type. Currently supported: 'ePix10k2M', 'ePix10kaQuad', 'Rayonix', 'Jungfrau1M', 'Jungfrau4M'", ) - date: str = Field( - "", - description="Start date of analysis", - ) - work_dir: str = Field( "", description="Main working directory for LUTE.", @@ -139,33 +134,27 @@ class BayesGeomOptParameters(BaseModel): @validator("exp", always=True) def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: if not exp: - exp: str = values["lute_config"].experiment - return exp + exp_config: str = values["lute_config"].experiment + return exp_config @validator("run", always=True) def validate_run( cls, run: Union[str, int], values: Dict[str, Any] ) -> Union[str, int]: if not run: - run: Union[str, int] = values["lute_config"].run - return run - - @validator("date", always=True) - def validate_date(cls, date, values: Dict[str, Any]) -> str: - if not date: - date: str = values["lute_config"].date - return date + run_config: Union[str, int] = values["lute_config"].run + return run_config @validator("work_dir", always=True) def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: if not work_dir: - work_dir: str = values["lute_config"].work_dir - return work_dir + work_dir_config: str = values["lute_config"].work_dir + return work_dir_config @validator("out_file", always=True) def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: if not out_file: in_file = values["in_file"] run = values["run"] - out_file: str = in_file.replace("0-end.data", f"{run}-end.data") - return out_file + out_file_config: str = in_file.replace("0-end.data", f"{run}-end.data") + return out_file_config diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 3ce56460..5f0854fd 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -915,7 +915,7 @@ def _extract_powder( The extracted powder image. Returns None if no powder could be extracted and no specific error was encountered. """ - powder = None + powder: Optional[npt.NDArray[np.float64]] = None assert isinstance(self._task_parameters, OptimizePyFAIGeometryParameters) if isinstance(powder_path, str): is_valid: bool @@ -930,22 +930,22 @@ def _extract_powder( with h5py.File(powder_path) as h5: try: if self._task_parameters.det_type == "Rayonix": - powder: npt.NDArray[np.float64] = h5[ + powder = h5[ f"Sums/{self._task_parameters.det_type}_calib_skipFirst_max" ][()] else: - powder: npt.NDArray[np.float64] = h5[ + powder = h5[ f"Sums/{self._task_parameters.det_type}_calib_max" ][()] except KeyError: logger.warning( 'No "Max" powder found in SmallData. Using "Sum" powder.' ) - powder: npt.NDArray[np.float64] = h5[ + powder = h5[ f"Sums/{self._task_parameters.det_type}_calib" ][()] - if powder.shape != shape: - powder: npt.NDArray[np.float64] = np.reshape(powder, shape) + if powder is not None and powder.shape != shape: + powder = np.reshape(powder, shape) return powder def _update_geometry(self, optimizer): @@ -1019,7 +1019,7 @@ def _run(self) -> None: logger.info( f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" ) - logger.info(f"Final Residuals: {optimizer.residual:.2e}") + logger.info(f"Final Residual: {optimizer.residual:.2e}") fig_folder = os.path.join(self._task_parameters.work_dir, "figs") os.makedirs(fig_folder, exist_ok=True) plot = ( From a2d8b67aa753b60babc3d92a8298d1d0a0797d3d Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Mon, 9 Dec 2024 23:56:16 +0000 Subject: [PATCH 234/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 5f0854fd..9097f144 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -941,9 +941,7 @@ def _extract_powder( logger.warning( 'No "Max" powder found in SmallData. Using "Sum" powder.' ) - powder = h5[ - f"Sums/{self._task_parameters.det_type}_calib" - ][()] + powder = h5[f"Sums/{self._task_parameters.det_type}_calib"][()] if powder is not None and powder.shape != shape: powder = np.reshape(powder, shape) return powder From 6dc7a002ff5c971a82f06f5f6714b464514e70e7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 9 Dec 2024 16:03:27 -0800 Subject: [PATCH 235/375] [edit] Fix mypy testing through asserting correct pydantic type --- lute/io/models/validators.py | 2 +- lute/tasks/geom_opt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/io/models/validators.py b/lute/io/models/validators.py index 170c6815..f39305e4 100644 --- a/lute/io/models/validators.py +++ b/lute/io/models/validators.py @@ -83,7 +83,7 @@ def _validate_calib_path(cls, calib_path: str, values: Dict[str, Any]) -> str: try: det_type: str = values["det_type"] except KeyError: - det_type: str = values["detname"] + det_type = values["detname"] cdir = f"/sdf/data/lcls/ds/{exp[:3]}/{exp}/calib" ds_args = f"exp={exp}:run={run}:idx" ds = psana.DataSource(ds_args) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 9097f144..32b0360d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -923,7 +923,7 @@ def _extract_powder( is_valid, dtype = self._check_if_path_and_type(powder_path) if is_valid and dtype == "numpy": powder = np.load(powder_path) - if powder.shape != shape: + if powder is not None and powder.shape != shape: powder = np.reshape(powder, shape) elif is_valid and dtype == "smd": h5: h5py.File From 09aaa5afc7223b8b301927f4b08f0c9e277f3c46 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 9 Dec 2024 16:31:00 -0800 Subject: [PATCH 236/375] [edit] Fix mypy pydantic check --- lute/io/models/geom_opt.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lute/io/models/geom_opt.py b/lute/io/models/geom_opt.py index ec4397ae..9323222b 100644 --- a/lute/io/models/geom_opt.py +++ b/lute/io/models/geom_opt.py @@ -134,27 +134,27 @@ class BayesGeomOptParameters(BaseModel): @validator("exp", always=True) def validate_exp(cls, exp: str, values: Dict[str, Any]) -> str: if not exp: - exp_config: str = values["lute_config"].experiment - return exp_config + exp = values["lute_config"].experiment + return exp @validator("run", always=True) def validate_run( cls, run: Union[str, int], values: Dict[str, Any] ) -> Union[str, int]: if not run: - run_config: Union[str, int] = values["lute_config"].run - return run_config + run = values["lute_config"].run + return run @validator("work_dir", always=True) def validate_work_dir(cls, work_dir: str, values: Dict[str, Any]) -> str: if not work_dir: - work_dir_config: str = values["lute_config"].work_dir - return work_dir_config + work_dir = values["lute_config"].work_dir + return work_dir @validator("out_file", always=True) def validate_out_file(cls, out_file: str, values: Dict[str, Any]) -> str: if not out_file: in_file = values["in_file"] run = values["run"] - out_file_config: str = in_file.replace("0-end.data", f"{run}-end.data") - return out_file_config + out_file = in_file.replace("0-end.data", f"{run}-end.data") + return out_file From 5b12484ba3710ff311736c82a15a6d26d01d5a62 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 14:33:03 -0800 Subject: [PATCH 237/375] [refacto] Fix unnecessary logs and set back to default mapping of mpirun procs --- lute/execution/executor.py | 2 +- lute/execution/logging.py | 4 ++++ lute/tasks/geom_opt.py | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lute/execution/executor.py b/lute/execution/executor.py index 04dd5619..4535aa5e 100644 --- a/lute/execution/executor.py +++ b/lute/execution/executor.py @@ -1151,7 +1151,7 @@ def _submit_cmd(self, executable_path: str, params: str) -> str: nprocs: int = max( int(os.environ.get("SLURM_NPROCS", len(os.sched_getaffinity(0)))) - 1, 1 ) - mpi_cmd: str = f"mpirun -np {nprocs} --map-by core" + mpi_cmd: str = f"mpirun -np {nprocs}" if __debug__: py_cmd = f"python -B -u -m mpi4py.run {executable_path} {params}" else: diff --git a/lute/execution/logging.py b/lute/execution/logging.py index c1a24c51..7890b0c7 100644 --- a/lute/execution/logging.py +++ b/lute/execution/logging.py @@ -106,6 +106,10 @@ def get_logger(name: str, is_task: bool = True) -> logging.Logger: other_logger, logging.PlaceHolder ): other_logger.setLevel(logging.CRITICAL) + elif "numexpr.utils" in other_name and not isinstance( + other_logger, logging.PlaceHolder + ): + other_logger.setLevel(logging.WARNING) logger: logging.Logger = logging.getLogger(name) logger.propagate = False handler: logging.Handler diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 32b0360d..d62c822a 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -32,6 +32,8 @@ import numpy as np import numpy.typing as npt import matplotlib.pyplot as plt # type: ignore +import pyFAI # type: ignore +pyFAI.use_opencl = False # type: ignore from pyFAI.geometry import Geometry # type: ignore from pyFAI.goniometer import SingleGeometry # type: ignore from pyFAI.azimuthalIntegrator import AzimuthalIntegrator # type: ignore @@ -187,7 +189,8 @@ def min_intensity(self, powder): """ mean = np.mean(powder) threshold = mean + 5 * np.std(powder) - logger.info(f"Threshold for pixel outliers: {threshold:.2e}") + if self.rank == 0: + logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] Imins = np.arange(95, 100, 0.01) From 9aaacf455c41ca8efdbbe0e4bdd550e47da51b6f Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Tue, 10 Dec 2024 22:33:31 +0000 Subject: [PATCH 238/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d62c822a..42254027 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -32,8 +32,9 @@ import numpy as np import numpy.typing as npt import matplotlib.pyplot as plt # type: ignore -import pyFAI # type: ignore -pyFAI.use_opencl = False # type: ignore +import pyFAI # type: ignore + +pyFAI.use_opencl = False # type: ignore from pyFAI.geometry import Geometry # type: ignore from pyFAI.goniometer import SingleGeometry # type: ignore from pyFAI.azimuthalIntegrator import AzimuthalIntegrator # type: ignore From 5cab26d7b441d9c1626580e0c7791d864f5e01e8 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 14:55:12 -0800 Subject: [PATCH 239/375] [edit] Set --bind-to core inside mpi command for MPIExecutor --- lute/execution/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/execution/executor.py b/lute/execution/executor.py index 4535aa5e..a110ccd9 100644 --- a/lute/execution/executor.py +++ b/lute/execution/executor.py @@ -1151,7 +1151,7 @@ def _submit_cmd(self, executable_path: str, params: str) -> str: nprocs: int = max( int(os.environ.get("SLURM_NPROCS", len(os.sched_getaffinity(0)))) - 1, 1 ) - mpi_cmd: str = f"mpirun -np {nprocs}" + mpi_cmd: str = f"mpirun -np {nprocs} --bind-to core" if __debug__: py_cmd = f"python -B -u -m mpi4py.run {executable_path} {params}" else: From 5947427a8d978fce5c651dd7fe0343bffc4a7616 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 15:11:08 -0800 Subject: [PATCH 240/375] [edit] Set --map-by core inside mpi command for MPIExecutor --- lute/execution/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/execution/executor.py b/lute/execution/executor.py index a110ccd9..04dd5619 100644 --- a/lute/execution/executor.py +++ b/lute/execution/executor.py @@ -1151,7 +1151,7 @@ def _submit_cmd(self, executable_path: str, params: str) -> str: nprocs: int = max( int(os.environ.get("SLURM_NPROCS", len(os.sched_getaffinity(0)))) - 1, 1 ) - mpi_cmd: str = f"mpirun -np {nprocs} --bind-to core" + mpi_cmd: str = f"mpirun -np {nprocs} --map-by core" if __debug__: py_cmd = f"python -B -u -m mpi4py.run {executable_path} {params}" else: From 8f4778e87a0958a441ad64e87ac60ed1e33e3005 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 15:24:49 -0800 Subject: [PATCH 241/375] [refacto] Removed surrogate map and acquisition map from bayes opt history --- lute/tasks/geom_opt.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 42254027..e68ccf42 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -398,12 +398,9 @@ def bayes_opt_center( sg.extract_cp(max_rings=max_rings, pts_per_deg=1, Imin=Imin) score = len(sg.geometry_refinement.data) y = np.append(y, [score], axis=0) - ypred = gp_model.predict(X_norm, return_std=False) bo_history[f"iteration_{i+1}"] = { "param": X[new_idx], "score": score, - "pred": ypred, - "af": af_values, } X_samples = np.append(X_samples, [X[new_idx]], axis=0) X_norm_samples = np.append(X_norm_samples, [X_norm[new_idx]], axis=0) From 9f9436c9be457a94000dbdfdb1d4b7885d33a06d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 15:53:49 -0800 Subject: [PATCH 242/375] [edit] Fix pyOpenCL after importing modules --- lute/tasks/geom_opt.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index e68ccf42..8205ae63 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -33,8 +33,6 @@ import numpy.typing as npt import matplotlib.pyplot as plt # type: ignore import pyFAI # type: ignore - -pyFAI.use_opencl = False # type: ignore from pyFAI.geometry import Geometry # type: ignore from pyFAI.goniometer import SingleGeometry # type: ignore from pyFAI.azimuthalIntegrator import AzimuthalIntegrator # type: ignore @@ -45,6 +43,7 @@ from sklearn.exceptions import ConvergenceWarning # type: ignore from scipy.stats import norm # type: ignore from mpi4py import MPI +pyFAI.use_opencl = False # type: ignore logger: logging.Logger = get_logger(__name__) From dddab65be65427e6ef82f2f7069b49c0cab1a528 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Tue, 10 Dec 2024 23:54:17 +0000 Subject: [PATCH 243/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8205ae63..e6e4049d 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -43,6 +43,7 @@ from sklearn.exceptions import ConvergenceWarning # type: ignore from scipy.stats import norm # type: ignore from mpi4py import MPI + pyFAI.use_opencl = False # type: ignore logger: logging.Logger = get_logger(__name__) From a9ee8172328de929e6ea29f01b63245204317f00 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 16:34:39 -0800 Subject: [PATCH 244/375] [edit] Try to compute detector edge resolution --- lute/tasks/geom_opt.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 8205ae63..dc5a0f5f 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -11,7 +11,7 @@ from lute.io.models.geom_opt import OptimizePyFAIGeometryParameters from lute.tasks.task import Task -from lute.tasks.dataclasses import TaskStatus +from lute.tasks.dataclasses import TaskStatus, ElogSummaryPlots from lute.execution.logging import get_logger import psana # type: ignore @@ -1031,5 +1031,24 @@ def _run(self) -> None: params=optimizer.params, plot=plot, ) + p1, p2, _ = calib_detector.calc_cartesian_positions() + cx_pix = np.abs(cx - np.min(p1))/ calib_detector.pixel1 + cy_pix = np.abs(cy - np.min(p2)) / calib_detector.pixel2 + theta: float = np.arctan(cx / distance) + q: float = 2.0 * np.sin(theta / 2.0) / (optimizer.calibrant.wavelength * 1e10) + edge_resolution: float = 1.0 / q self._result.payload = plot + self._result.summary = [] + self._result.summary.append( + { + "Detector distance (m)": distance, + "Detector center (pixels)": (cx_pix, cy_pix), + "Detector edge resolution (A)": edge_resolution, + } + ) + logger.info(f"Beam center (pixels): ({cx_pix}, {cy_pix})") + logger.info(f"Detector edge resolution (A): {edge_resolution}") + self._result.summary.append( + ElogSummaryPlots(f"Geometry_Fit/r{self._task_parameters.run:0>4}", plot) + ) self._result.task_status = TaskStatus.COMPLETED From 1493b36ee88b861e2760f26da99f9c70c4289daf Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Wed, 11 Dec 2024 00:35:15 +0000 Subject: [PATCH 245/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 2500b904..3ea93193 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -1033,10 +1033,12 @@ def _run(self) -> None: plot=plot, ) p1, p2, _ = calib_detector.calc_cartesian_positions() - cx_pix = np.abs(cx - np.min(p1))/ calib_detector.pixel1 + cx_pix = np.abs(cx - np.min(p1)) / calib_detector.pixel1 cy_pix = np.abs(cy - np.min(p2)) / calib_detector.pixel2 theta: float = np.arctan(cx / distance) - q: float = 2.0 * np.sin(theta / 2.0) / (optimizer.calibrant.wavelength * 1e10) + q: float = ( + 2.0 * np.sin(theta / 2.0) / (optimizer.calibrant.wavelength * 1e10) + ) edge_resolution: float = 1.0 / q self._result.payload = plot self._result.summary = [] From 122f9fffb2af46151d7c740afe8c318a05f14b2d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 16:37:43 -0800 Subject: [PATCH 246/375] [edit] Set Imin lower bound to 98 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 2500b904..fbde6a99 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -194,7 +194,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(95, 100, 0.01) + Imins = np.arange(98, 100, 0.01) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From 5a4a1c7ff2f27bb0df2d7e0090a8574487cf0b4a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 16:47:22 -0800 Subject: [PATCH 247/375] [edit] Set Imin lower bound to 99 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index c7906ee4..3e496fec 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -194,7 +194,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(98, 100, 0.01) + Imins = np.arange(99, 100, 0.01) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From 5ea90a40376ccb9c0e54b8581697b04c501ae4e2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 16:51:28 -0800 Subject: [PATCH 248/375] [edit] Set Imin lower bound to 99 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 3e496fec..e3e0afbb 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -202,8 +202,8 @@ def min_intensity(self, powder): noise_pixels = powder[nice_pix][powder[nice_pix] <= threshold] noise = np.std(noise_pixels) SNRs.append(signal / noise) - self.q = round(Imins[np.argmax(SNRs)], 2) Imin = np.percentile(powder[nice_pix], self.q) + self.q = round(Imins[np.argmax(SNRs)], 2) self.Imin = Imin self.powder = powder return Imin From 55380dd5c771d77862bf21cdf910733b61c86ab6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 16:54:58 -0800 Subject: [PATCH 249/375] [edit] Set Imin lower bound to 99 --- lute/tasks/geom_opt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index e3e0afbb..47e074ae 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -202,8 +202,9 @@ def min_intensity(self, powder): noise_pixels = powder[nice_pix][powder[nice_pix] <= threshold] noise = np.std(noise_pixels) SNRs.append(signal / noise) - Imin = np.percentile(powder[nice_pix], self.q) - self.q = round(Imins[np.argmax(SNRs)], 2) + q = Imins[np.argmax(SNRs)] + Imin = np.percentile(powder[nice_pix], q) + self.q = round(q, 2) self.Imin = Imin self.powder = powder return Imin From 1e2d8feae3a0d66c2cf4fa9e9927f0928038446d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 17:05:40 -0800 Subject: [PATCH 250/375] [edit] Set Imin lower bound to 99.5 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 47e074ae..96839a5c 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -194,7 +194,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(99, 100, 0.01) + Imins = np.arange(99.5, 100, 0.01) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From a80de81d1570a0af9164f01b875f8ef0bdc76def Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 17:29:15 -0800 Subject: [PATCH 251/375] [edit] Set back to 95 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 96839a5c..d99068e3 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -194,7 +194,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(99.5, 100, 0.01) + Imins = np.arange(95, 100, 0.01) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From ee2ac0e47fde41938980ecaa5176c67ebee84fee Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 17:36:05 -0800 Subject: [PATCH 252/375] [edit] Add refined dist to label --- lute/tasks/geom_opt.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index d99068e3..0a34faac 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -656,7 +656,7 @@ def score_distance_scan(self, distances, ax): ax.set_ylabel("Score") ax.set_title("Number of Control Points vs Distance") - def residual_distance_scan(self, distances, ax): + def residual_distance_scan(self, distances, refined_dist, ax): """ Plot the residual scan over distance @@ -664,6 +664,8 @@ def residual_distance_scan(self, distances, ax): ---------- distances : np.array Array of distances + refined_dist : float + Refined distance ax : plt.Axes Matplotlib axes """ @@ -674,7 +676,13 @@ def residual_distance_scan(self, distances, ax): best_dist, color="green", linestyle="--", - label=f"Best distance: {best_dist:.2e}", + label=f"Best distance (m): {best_dist:.2e}", + ) + ax.axvline( + refined_dist, + color="red", + linestyle="--", + label=f"Refined distance (m): {refined_dist:.2e}", ) ax.legend(fontsize="x-small") ax.set_yscale("log") @@ -741,7 +749,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") ax.legend(fontsize="x-small") - def visualize_results(self, powder, bo_history, detector, params, plot=""): + def visualize_results(self, powder, bo_history, detector, params, refined_dist, plot=""): """ Visualize fit, plotting (1) the BO convergence, (2) the radial profile and (3) the powder image. @@ -755,6 +763,8 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): Corrected PyFAI detector object params : list List of parameters for the best fit + refined_dist : float + Refined distance plot : str Path to save plot """ @@ -816,7 +826,7 @@ def visualize_results(self, powder, bo_history, detector, params, plot=""): # Plotting residual scan over distance ax6 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - self.residual_distance_scan(self.distances, ax6) + self.residual_distance_scan(self.distances, refined_dist, ax6) if plot != "": fig.savefig(plot, dpi=180) @@ -1031,6 +1041,7 @@ def _run(self) -> None: bo_history=optimizer.bo_history, detector=calib_detector, params=optimizer.params, + refined_dist=distance, plot=plot, ) p1, p2, _ = calib_detector.calc_cartesian_positions() From e8d6f7b974441bd765441dc84fe70bd1f16c76e1 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Wed, 11 Dec 2024 01:36:32 +0000 Subject: [PATCH 253/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0a34faac..55c14acc 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -749,7 +749,9 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") ax.legend(fontsize="x-small") - def visualize_results(self, powder, bo_history, detector, params, refined_dist, plot=""): + def visualize_results( + self, powder, bo_history, detector, params, refined_dist, plot="" + ): """ Visualize fit, plotting (1) the BO convergence, (2) the radial profile and (3) the powder image. From 336b054af091092469e9a63395ae2b398ee1284c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 18:13:06 -0800 Subject: [PATCH 254/375] [edit] Fix plots --- lute/tasks/geom_opt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 0a34faac..6f6fb39b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -194,7 +194,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(95, 100, 0.01) + Imins = np.arange(95, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] @@ -204,7 +204,7 @@ def min_intensity(self, powder): SNRs.append(signal / noise) q = Imins[np.argmax(SNRs)] Imin = np.percentile(powder[nice_pix], q) - self.q = round(q, 2) + self.q = round(q, 1) self.Imin = Imin self.powder = powder return Imin @@ -779,7 +779,7 @@ def visualize_results(self, powder, bo_history, detector, params, refined_dist, ax1.set_xticks(np.arange(len(scores), step=20)) ax1.set_xlabel("Iteration") ax1.set_ylabel("Best score so far") - ax1.set_title(f"Convergence Plot, best score: {int(self.score)}") + ax1.set_title(f"Convergence Plot, best score: {self.scan['score'][self.index]}") icol += 1 # Plotting histogram of pixel intensities @@ -1057,7 +1057,7 @@ def _run(self) -> None: self._result.summary.append( { "Detector distance (m)": distance, - "Detector center (pixels)": (cx_pix, cy_pix), + "Detector center (m)": (cx, cy), "Detector edge resolution (A)": edge_resolution, } ) From 2bd18cb1e72e74ae68a33fed61f85d9dc027e92e Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 18:30:40 -0800 Subject: [PATCH 255/375] [edit] Fix output on eLog --- lute/tasks/geom_opt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 810ad844..c2ba8258 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -1058,9 +1058,9 @@ def _run(self) -> None: self._result.summary = [] self._result.summary.append( { - "Detector distance (m)": distance, - "Detector center (m)": (cx, cy), - "Detector edge resolution (A)": edge_resolution, + "Detector distance (m)": f"{distance:.3f}", + "Detector center (m)": (f"{cx:.6f}", f"{cy:.6f}"), + "Detector edge resolution (A)": f"{edge_resolution:.3f}", } ) logger.info(f"Beam center (pixels): ({cx_pix}, {cy_pix})") From 9618b1d9b4549a4b45f2090fa94dbc9e044a7fb7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 18:44:45 -0800 Subject: [PATCH 256/375] [edit] Fix plot for Elog summary --- lute/tasks/geom_opt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index c2ba8258..df316336 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -832,6 +832,7 @@ def visualize_results( if plot != "": fig.savefig(plot, dpi=180) + return fig class OptimizePyFAIGeometry(Task): @@ -1038,7 +1039,7 @@ def _run(self) -> None: f"{fig_folder}/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" ) calib_detector = self._update_geometry(optimizer) - optimizer.visualize_results( + fig = optimizer.visualize_results( powder=optimizer.powder, bo_history=optimizer.bo_history, detector=calib_detector, @@ -1066,6 +1067,6 @@ def _run(self) -> None: logger.info(f"Beam center (pixels): ({cx_pix}, {cy_pix})") logger.info(f"Detector edge resolution (A): {edge_resolution}") self._result.summary.append( - ElogSummaryPlots(f"Geometry_Fit/r{self._task_parameters.run:0>4}", plot) + ElogSummaryPlots(f"Geometry_Fit/r{self._task_parameters.run:0>4}", fig) ) self._result.task_status = TaskStatus.COMPLETED From c6365e804a8edb0b3cf4c3526c26426dc5114bd7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 10 Dec 2024 18:53:29 -0800 Subject: [PATCH 257/375] [edit] Include plots in the eLog summary --- lute/tasks/geom_opt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index df316336..58667cb8 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -29,6 +29,7 @@ ) import h5py # type: ignore +import panel as pn # type: ignore import numpy as np import numpy.typing as npt import matplotlib.pyplot as plt # type: ignore @@ -1047,6 +1048,7 @@ def _run(self) -> None: refined_dist=distance, plot=plot, ) + plots = pn.Tabs(fig) p1, p2, _ = calib_detector.calc_cartesian_positions() cx_pix = np.abs(cx - np.min(p1)) / calib_detector.pixel1 cy_pix = np.abs(cy - np.min(p2)) / calib_detector.pixel2 @@ -1055,7 +1057,6 @@ def _run(self) -> None: 2.0 * np.sin(theta / 2.0) / (optimizer.calibrant.wavelength * 1e10) ) edge_resolution: float = 1.0 / q - self._result.payload = plot self._result.summary = [] self._result.summary.append( { @@ -1067,6 +1068,6 @@ def _run(self) -> None: logger.info(f"Beam center (pixels): ({cx_pix}, {cy_pix})") logger.info(f"Detector edge resolution (A): {edge_resolution}") self._result.summary.append( - ElogSummaryPlots(f"Geometry_Fit/r{self._task_parameters.run:0>4}", fig) + ElogSummaryPlots(f"Geometry_Fit/r{self._task_parameters.run:0>4}", plots) ) self._result.task_status = TaskStatus.COMPLETED From 27ea664b456cc3c0dd172601818bb0f00d5471e7 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Wed, 11 Dec 2024 02:53:53 +0000 Subject: [PATCH 258/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 58667cb8..dce01fd7 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -29,7 +29,7 @@ ) import h5py # type: ignore -import panel as pn # type: ignore +import panel as pn # type: ignore import numpy as np import numpy.typing as npt import matplotlib.pyplot as plt # type: ignore @@ -1068,6 +1068,8 @@ def _run(self) -> None: logger.info(f"Beam center (pixels): ({cx_pix}, {cy_pix})") logger.info(f"Detector edge resolution (A): {edge_resolution}") self._result.summary.append( - ElogSummaryPlots(f"Geometry_Fit/r{self._task_parameters.run:0>4}", plots) + ElogSummaryPlots( + f"Geometry_Fit/r{self._task_parameters.run:0>4}", plots + ) ) self._result.task_status = TaskStatus.COMPLETED From dca2ccd2e0e9156bf834f5e9b8c8d6e8f488d75c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 11 Dec 2024 16:32:17 -0800 Subject: [PATCH 259/375] [edit] Set back to 98 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 58667cb8..693d340b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -195,7 +195,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(95, 100, 0.1) + Imins = np.arange(98, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From 5e9ac779fc5e92dede15166f83c107682da9e9e6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 11 Dec 2024 17:07:55 -0800 Subject: [PATCH 260/375] [edit] Refine not on rotation first, only on dist cx cy, then finally apply rotations --- lute/tasks/geom_opt.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index b219b825..7c919c87 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -436,7 +436,7 @@ def bayes_opt_center( score = len(sg.geometry_refinement.data) residual = 0 if score != 0: - residual = sg.geometry_refinement.refine3(fix=["wavelength"]) + residual = sg.geometry_refinement.refine3(fix=["rot1", "rot2", "rot3", "wavelength"]) params = sg.geometry_refinement.param result = { "bo_history": bo_history, @@ -541,9 +541,7 @@ def bayes_opt_geom( self.scan["residual"] = self.comm.gather(results["residual"], root=0) self.scan["score"] = self.comm.gather(results["score"], root=0) self.scan["best_idx"] = self.comm.gather(results["best_idx"], root=0) - self.finalize() - def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) @@ -554,6 +552,33 @@ def finalize(self): self.residual = self.scan["residual"][index] self.score = self.scan["score"][index] self.best_idx = self.scan["best_idx"][index] + dist, poni1, poni2, rot1, rot2, rot3 = self.params + geom_initial = Geometry( + dist=dist, + poni1=poni1, + poni2=poni2, + rot1=rot1, + rot2=rot2, + rot3=rot3, + detector=self.detector, + wavelength=self.calibrant.wavelength, + ) + sg = SingleGeometry( + "extract_cp", + powder, + calibrant=self.calibrant, + detector=self.detector, + geometry=geom_initial, + ) + sg.extract_cp(max_rings=max_rings, pts_per_deg=1, Imin=Imin) + self.sg = sg + score = len(sg.geometry_refinement.data) + residual = 0 + if score != 0: + residual = sg.geometry_refinement.refine3(fix=["wavelength"]) + params = sg.geometry_refinement.param + self.params = params + self.residual = residual def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): """ From d12e2396df54d19fa56c7713ded77664d9277329 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 12 Dec 2024 01:08:24 +0000 Subject: [PATCH 261/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7c919c87..575e6f6b 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -436,7 +436,9 @@ def bayes_opt_center( score = len(sg.geometry_refinement.data) residual = 0 if score != 0: - residual = sg.geometry_refinement.refine3(fix=["rot1", "rot2", "rot3", "wavelength"]) + residual = sg.geometry_refinement.refine3( + fix=["rot1", "rot2", "rot3", "wavelength"] + ) params = sg.geometry_refinement.param result = { "bo_history": bo_history, From f75e4a628f2cdaea9b44c8cc4277be71f776ebc5 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 11 Dec 2024 17:16:55 -0800 Subject: [PATCH 262/375] [edit] Set back to 95 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7c919c87..ade1b579 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -195,7 +195,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(98, 100, 0.1) + Imins = np.arange(95, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From d868e5754bcf741bfbe1a1d084720162b37555a7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 11 Dec 2024 17:28:18 -0800 Subject: [PATCH 263/375] [edit] Set back normal --- lute/tasks/geom_opt.py | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index edc68cbf..e3a82e16 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -437,7 +437,7 @@ def bayes_opt_center( residual = 0 if score != 0: residual = sg.geometry_refinement.refine3( - fix=["rot1", "rot2", "rot3", "wavelength"] + fix=["wavelength"] ) params = sg.geometry_refinement.param result = { @@ -543,7 +543,9 @@ def bayes_opt_geom( self.scan["residual"] = self.comm.gather(results["residual"], root=0) self.scan["score"] = self.comm.gather(results["score"], root=0) self.scan["best_idx"] = self.comm.gather(results["best_idx"], root=0) + self.finalize() + def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) @@ -554,33 +556,6 @@ def bayes_opt_geom( self.residual = self.scan["residual"][index] self.score = self.scan["score"][index] self.best_idx = self.scan["best_idx"][index] - dist, poni1, poni2, rot1, rot2, rot3 = self.params - geom_initial = Geometry( - dist=dist, - poni1=poni1, - poni2=poni2, - rot1=rot1, - rot2=rot2, - rot3=rot3, - detector=self.detector, - wavelength=self.calibrant.wavelength, - ) - sg = SingleGeometry( - "extract_cp", - powder, - calibrant=self.calibrant, - detector=self.detector, - geometry=geom_initial, - ) - sg.extract_cp(max_rings=max_rings, pts_per_deg=1, Imin=Imin) - self.sg = sg - score = len(sg.geometry_refinement.data) - residual = 0 - if score != 0: - residual = sg.geometry_refinement.refine3(fix=["wavelength"]) - params = sg.geometry_refinement.param - self.params = params - self.residual = residual def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): """ From 5f90f33809ea8ce710102261585bde7224f4c1a9 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 12 Dec 2024 01:28:48 +0000 Subject: [PATCH 264/375] Auto-commit black formatting --- lute/tasks/geom_opt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index e3a82e16..dce01fd7 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -436,9 +436,7 @@ def bayes_opt_center( score = len(sg.geometry_refinement.data) residual = 0 if score != 0: - residual = sg.geometry_refinement.refine3( - fix=["wavelength"] - ) + residual = sg.geometry_refinement.refine3(fix=["wavelength"]) params = sg.geometry_refinement.param result = { "bo_history": bo_history, From 09b9ef6a4ca1eb996a2a19d325908a30a87501ef Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 11 Dec 2024 17:42:49 -0800 Subject: [PATCH 265/375] [edit] Set to 99 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index e3a82e16..925beb86 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -195,7 +195,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(95, 100, 0.1) + Imins = np.arange(99, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From 90920fef1d41b117642d234e4dbb66e79c8b0d98 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 11 Dec 2024 17:51:35 -0800 Subject: [PATCH 266/375] [edit] Set back to 95 --- lute/tasks/geom_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/geom_opt.py b/lute/tasks/geom_opt.py index 7e0b4158..dce01fd7 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/geom_opt.py @@ -195,7 +195,7 @@ def min_intensity(self, powder): logger.info(f"Threshold for pixel outliers: {threshold:.2e}") nice_pix = powder < threshold SNRs = [] - Imins = np.arange(99, 100, 0.1) + Imins = np.arange(95, 100, 0.1) for Imin in Imins: threshold = np.percentile(powder[nice_pix], Imin) signal_pixels = powder[nice_pix][powder[nice_pix] > threshold] From 2c9f6c5699a49ba56b76ee701879b69847549109 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Tue, 17 Dec 2024 01:41:51 -0800 Subject: [PATCH 267/375] [edit] Rename python files into bayfai to match workflow name BayFAI --- lute/io/models/__init__.py | 2 +- lute/io/models/{geom_opt.py => bayfai.py} | 0 lute/tasks/__init__.py | 2 +- lute/tasks/{geom_opt.py => bayfai.py} | 2 +- workflows/airflow/{geom_opt_pyfai.py => bayfai.py} | 0 5 files changed, 3 insertions(+), 3 deletions(-) rename lute/io/models/{geom_opt.py => bayfai.py} (100%) rename lute/tasks/{geom_opt.py => bayfai.py} (99%) rename workflows/airflow/{geom_opt_pyfai.py => bayfai.py} (100%) diff --git a/lute/io/models/__init__.py b/lute/io/models/__init__.py index f98322a4..6c8db115 100644 --- a/lute/io/models/__init__.py +++ b/lute/io/models/__init__.py @@ -11,4 +11,4 @@ from .tests import * from .mpi_tests import * from .geometry import * -from .geom_opt import * +from .bayfai import * diff --git a/lute/io/models/geom_opt.py b/lute/io/models/bayfai.py similarity index 100% rename from lute/io/models/geom_opt.py rename to lute/io/models/bayfai.py diff --git a/lute/tasks/__init__.py b/lute/tasks/__init__.py index 6fd6b8ee..b0b06037 100644 --- a/lute/tasks/__init__.py +++ b/lute/tasks/__init__.py @@ -87,7 +87,7 @@ def import_task(task_name: str) -> Type[Task]: return TestMultiNodeCommunication if task_name == "OptimizePyFAIGeometry": - from .geom_opt import OptimizePyFAIGeometry + from .bayfai import OptimizePyFAIGeometry return OptimizePyFAIGeometry diff --git a/lute/tasks/geom_opt.py b/lute/tasks/bayfai.py similarity index 99% rename from lute/tasks/geom_opt.py rename to lute/tasks/bayfai.py index dce01fd7..37d99d12 100644 --- a/lute/tasks/geom_opt.py +++ b/lute/tasks/bayfai.py @@ -9,7 +9,7 @@ __all__ = ["OptimizePyFAIGeometry"] __author__ = "Louis Conreux" -from lute.io.models.geom_opt import OptimizePyFAIGeometryParameters +from lute.io.models.bayfai import OptimizePyFAIGeometryParameters from lute.tasks.task import Task from lute.tasks.dataclasses import TaskStatus, ElogSummaryPlots from lute.execution.logging import get_logger diff --git a/workflows/airflow/geom_opt_pyfai.py b/workflows/airflow/bayfai.py similarity index 100% rename from workflows/airflow/geom_opt_pyfai.py rename to workflows/airflow/bayfai.py From bc229f0f9e5ccb57a61093aa83d37e04b45b52e7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Wed, 18 Dec 2024 00:55:34 -0800 Subject: [PATCH 268/375] [edit] Smalldata producer template change to output max of the maxs, not sums of max --- config/templates/smd_producer_template.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/templates/smd_producer_template.py b/config/templates/smd_producer_template.py index 7e8a26d3..89630951 100644 --- a/config/templates/smd_producer_template.py +++ b/config/templates/smd_producer_template.py @@ -991,7 +991,10 @@ def get_sd_file(write_dir, exp, hutch): sumDict={'Sums': {}} for det in dets: for key in det.storeSum().keys(): - sumData=small_data.sum(det.storeSum()[key]) + if 'max' in key: + sumData=small_data.max(det.storeSum()[key]) + else: + sumData=small_data.sum(det.storeSum()[key]) sumDict['Sums']['%s_%s'%(det._name, key)]=sumData if len(sumDict['Sums'].keys())>0: # print(sumDict) From 91a2c8cce5f82c8b0e07ece44abfeb2fc4b8cbf9 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 01:18:25 -0800 Subject: [PATCH 269/375] [edit] Add a preprocessing layer before launching optimization --- lute/io/models/bayfai.py | 5 ++++ lute/tasks/bayfai.py | 50 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/lute/io/models/bayfai.py b/lute/io/models/bayfai.py index 9323222b..4b7b4797 100644 --- a/lute/io/models/bayfai.py +++ b/lute/io/models/bayfai.py @@ -115,6 +115,11 @@ class BayesGeomOptParameters(BaseModel): description="Powder diffraction pattern to be used for the calibration.", ) + preprocess: Optional[str] = Field( + None, + description="Preprocessing method to be used for the calibration.", + ) + calibrant: str = Field( "", description="Calibrant used for the calibration supported by pyFAI: https://github.com/silx-kit/pyFAI/tree/main/src/pyFAI/resources/calibration", diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 37d99d12..ca991d06 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -43,6 +43,8 @@ from sklearn.utils._testing import ignore_warnings # type: ignore from sklearn.exceptions import ConvergenceWarning # type: ignore from scipy.stats import norm # type: ignore +from scipy.ndimage import gaussian_filter # type: ignore +from scipy.signal import convolve2d # type: ignore from mpi4py import MPI pyFAI.use_opencl = False # type: ignore @@ -962,6 +964,53 @@ def _extract_powder( powder = np.reshape(powder, shape) return powder + def _preprocess_powder(self, powder: npt.NDArray[np.float64], preprocess: str = None) -> npt.NDArray[np.float64]: + """ + Preprocess extracted powder for enhancing optimization + + Parameters + ---------- + powder : npt.NDArray[np.float64] + Powder image to use for calibration + preprocess : str + Type of preprocessing technique + Available preprocessing: gradient "magnitude" powder, "gradient" sigmoid powder, + "high-pass" filtering, "CAE" convolutional autoencoding (later) + """ + def sigmoid(x): + return 1 / (1 + np.exp(-x)) + + if preprocess == None: + return powder + elif preprocess == "magnitude": + sigma = 1 + calib = gaussian_filter(powder, sigma=sigma) + gradx_calib = np.zeros_like(powder) + grady_calib = np.zeros_like(powder) + gradx_calib[:-1, :-1] = (calib[1:, :-1] - calib[:-1, :-1] + calib[1:, 1:] - calib[:-1, 1:]) / 2 + grady_calib[:-1, :-1] = (calib[:-1, 1:] - calib[:-1, :-1] + calib[1:, 1:] - calib[1:, :-1]) / 2 + powder = np.sqrt(gradx_calib**2 + grady_calib**2) + return powder + elif preprocess == "gradient": + sigma = 1 + calib = gaussian_filter(powder, sigma=sigma) + gradx_calib = np.zeros_like(powder) + grady_calib = np.zeros_like(powder) + gradx_calib[:-1, :-1] = (calib[1:, :-1] - calib[:-1, :-1] + calib[1:, 1:] - calib[:-1, 1:]) / 2 + grady_calib[:-1, :-1] = (calib[:-1, 1:] - calib[:-1, :-1] + calib[1:, 1:] - calib[1:, :-1]) / 2 + powder = gradx_calib + grady_calib + powder = sigmoid(powder) + return powder + elif preprocess == "high-pass": + kernel = np.array([ + [-1, -1, -1], + [-1, 8, -1], + [-1, -1, -1] + ]) + powder = convolve2d(powder, kernel, mode='same', boundary='symm') + powder = sigmoid(powder) + return powder + def _update_geometry(self, optimizer): """ Update the geometry and write a new .geom file and .data file @@ -1004,6 +1053,7 @@ def _run(self) -> None: assert isinstance(self._task_parameters, OptimizePyFAIGeometryParameters) detector = self._build_pyFAI_detector() powder = self._extract_powder(self._task_parameters.powder, detector.shape) + powder = self._preprocess_powder(powder, self._task_parameters.preprocess) if powder is None: raise RuntimeError("Unable to extract powder. Cannot continue.") optimizer = BayesGeomOpt( From 5d6548eba73939afc623871a359ae6a61fc09a93 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 19 Dec 2024 09:18:56 +0000 Subject: [PATCH 270/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index ca991d06..66a48a15 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -43,8 +43,8 @@ from sklearn.utils._testing import ignore_warnings # type: ignore from sklearn.exceptions import ConvergenceWarning # type: ignore from scipy.stats import norm # type: ignore -from scipy.ndimage import gaussian_filter # type: ignore -from scipy.signal import convolve2d # type: ignore +from scipy.ndimage import gaussian_filter # type: ignore +from scipy.signal import convolve2d # type: ignore from mpi4py import MPI pyFAI.use_opencl = False # type: ignore @@ -964,7 +964,9 @@ def _extract_powder( powder = np.reshape(powder, shape) return powder - def _preprocess_powder(self, powder: npt.NDArray[np.float64], preprocess: str = None) -> npt.NDArray[np.float64]: + def _preprocess_powder( + self, powder: npt.NDArray[np.float64], preprocess: str = None + ) -> npt.NDArray[np.float64]: """ Preprocess extracted powder for enhancing optimization @@ -975,8 +977,9 @@ def _preprocess_powder(self, powder: npt.NDArray[np.float64], preprocess: str = preprocess : str Type of preprocessing technique Available preprocessing: gradient "magnitude" powder, "gradient" sigmoid powder, - "high-pass" filtering, "CAE" convolutional autoencoding (later) + "high-pass" filtering, "CAE" convolutional autoencoding (later) """ + def sigmoid(x): return 1 / (1 + np.exp(-x)) @@ -987,8 +990,12 @@ def sigmoid(x): calib = gaussian_filter(powder, sigma=sigma) gradx_calib = np.zeros_like(powder) grady_calib = np.zeros_like(powder) - gradx_calib[:-1, :-1] = (calib[1:, :-1] - calib[:-1, :-1] + calib[1:, 1:] - calib[:-1, 1:]) / 2 - grady_calib[:-1, :-1] = (calib[:-1, 1:] - calib[:-1, :-1] + calib[1:, 1:] - calib[1:, :-1]) / 2 + gradx_calib[:-1, :-1] = ( + calib[1:, :-1] - calib[:-1, :-1] + calib[1:, 1:] - calib[:-1, 1:] + ) / 2 + grady_calib[:-1, :-1] = ( + calib[:-1, 1:] - calib[:-1, :-1] + calib[1:, 1:] - calib[1:, :-1] + ) / 2 powder = np.sqrt(gradx_calib**2 + grady_calib**2) return powder elif preprocess == "gradient": @@ -996,18 +1003,18 @@ def sigmoid(x): calib = gaussian_filter(powder, sigma=sigma) gradx_calib = np.zeros_like(powder) grady_calib = np.zeros_like(powder) - gradx_calib[:-1, :-1] = (calib[1:, :-1] - calib[:-1, :-1] + calib[1:, 1:] - calib[:-1, 1:]) / 2 - grady_calib[:-1, :-1] = (calib[:-1, 1:] - calib[:-1, :-1] + calib[1:, 1:] - calib[1:, :-1]) / 2 + gradx_calib[:-1, :-1] = ( + calib[1:, :-1] - calib[:-1, :-1] + calib[1:, 1:] - calib[:-1, 1:] + ) / 2 + grady_calib[:-1, :-1] = ( + calib[:-1, 1:] - calib[:-1, :-1] + calib[1:, 1:] - calib[1:, :-1] + ) / 2 powder = gradx_calib + grady_calib powder = sigmoid(powder) return powder elif preprocess == "high-pass": - kernel = np.array([ - [-1, -1, -1], - [-1, 8, -1], - [-1, -1, -1] - ]) - powder = convolve2d(powder, kernel, mode='same', boundary='symm') + kernel = np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]]) + powder = convolve2d(powder, kernel, mode="same", boundary="symm") powder = sigmoid(powder) return powder From 1e866bbe934526cff0782a2ac5b8d180adbf8ecd Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 01:54:20 -0800 Subject: [PATCH 271/375] [edit] Filter out bad scores in taking the minimal residual --- lute/tasks/bayfai.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index ca991d06..c3243a3e 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -549,7 +549,10 @@ def finalize(self): if self.rank == 0: for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) - index = np.argmin(self.scan["residual"]) + percentile_10 = np.percentile(self.scan["score"], 10) + score_indices= np.where(self.scan["score"] > percentile_10)[0] + shift_index = np.argmin(self.scan["residual"][score_indices]) + index = score_indices[shift_index] self.index = index self.bo_history = self.scan["bo_history"][index] self.params = self.scan["params"][index] @@ -872,7 +875,7 @@ def _build_pyFAI_detector(self): detector = psana_to_pyfai.detector return detector - def _check_if_path_and_type(self, string: str) -> Tuple[bool, Optional[str]]: + def _check_path_and_type(self, string: str) -> Tuple[bool, Optional[str]]: """ Check if a string is a valid path and determine the filetype. @@ -938,7 +941,7 @@ def _extract_powder( if isinstance(powder_path, str): is_valid: bool dtype: Optional[str] - is_valid, dtype = self._check_if_path_and_type(powder_path) + is_valid, dtype = self._check_path_and_type(powder_path) if is_valid and dtype == "numpy": powder = np.load(powder_path) if powder is not None and powder.shape != shape: From 6258b7ac82e8320e46aca17d84d0acee55be5aea Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 19 Dec 2024 09:55:31 +0000 Subject: [PATCH 272/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index fd3b9fba..6b020e41 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -550,7 +550,7 @@ def finalize(self): for key in self.scan.keys(): self.scan[key] = np.array([item for item in self.scan[key]]) percentile_10 = np.percentile(self.scan["score"], 10) - score_indices= np.where(self.scan["score"] > percentile_10)[0] + score_indices = np.where(self.scan["score"] > percentile_10)[0] shift_index = np.argmin(self.scan["residual"][score_indices]) index = score_indices[shift_index] self.index = index From 479f6adeac21b6e1202798193af7bc66ba7bcc84 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 02:08:40 -0800 Subject: [PATCH 273/375] [edit] Filter out bad scores in taking the minimal residual + plot --- lute/tasks/bayfai.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index fd3b9fba..ba820ee2 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -657,7 +657,14 @@ def score_distance_scan(self, distances, ax): Matplotlib axes """ scores = self.scan["score"] + percentile_10 = np.percentile(scores, 10) ax.plot(distances, scores) + ax.axhline( + percentile_10, + color='red', + linestyle="--", + label=f"Minimal score: {percentile_10}" + ) ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.set_title("Number of Control Points vs Distance") From 576bfaf9e266b00c889d625be334fd35eb17e7f7 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 19 Dec 2024 10:09:25 +0000 Subject: [PATCH 274/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index afbb761f..9895806d 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -661,9 +661,9 @@ def score_distance_scan(self, distances, ax): ax.plot(distances, scores) ax.axhline( percentile_10, - color='red', + color="red", linestyle="--", - label=f"Minimal score: {percentile_10}" + label=f"Minimal score: {percentile_10}", ) ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") From 4c94ba0cbfc42fa0735c978f1fc294de4315ef61 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 06:21:52 -0800 Subject: [PATCH 275/375] [edit] Able to reconstruct 2D detector plot for visualizing powder --- lute/tasks/bayfai.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index afbb761f..3dd2283a 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -194,7 +194,7 @@ def min_intensity(self, powder): mean = np.mean(powder) threshold = mean + 5 * np.std(powder) if self.rank == 0: - logger.info(f"Threshold for pixel outliers: {threshold:.2e}") + logger.info(f"Threshold for pixel outliers: {threshold}") nice_pix = powder < threshold SNRs = [] Imins = np.arange(95, 100, 0.1) @@ -206,8 +206,8 @@ def min_intensity(self, powder): noise = np.std(noise_pixels) SNRs.append(signal / noise) q = Imins[np.argmax(SNRs)] - Imin = np.percentile(powder[nice_pix], q) - self.q = round(q, 1) + Imin = np.percentile(powder[nice_pix], 95) + self.q = 95 self.Imin = Imin self.powder = powder return Imin @@ -579,20 +579,20 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ai = sg.geometry_refinement if label is None: label = sg.label - img = ax.imshow( - powder.T, - origin="lower", - cmap="viridis", - vmin=np.percentile(powder, 5), - vmax=np.percentile(powder, 95), - ) + detector = sg.detector + y, x, z = detector.calc_cartesian_positions() + img = ax.scatter(x.flatten(), y.flatten(), c=powder.flatten(), s=1, edgecolors=None, linewidth=0, vmin=np.percentile(powder, 5), vmax=np.percentile(powder, 95)) cbar = plt.colorbar(img, ax=ax, orientation="vertical") cbar.set_label("Intensity") - if ai is not None and cp.calibrant is not None: - tth = cp.calibrant.get_2th() - ttha = ai.twoThetaArray() + x = np.reshape(x, detector.raw_shape) + y = np.reshape(y, detector.raw_shape) + z = np.reshape(z, detector.raw_shape) + z += ai.dist + ttha = np.arctan2(np.sqrt(x*x+y*y), z) + tth = cp.calibrant.get_2th() + for i in range(x.shape[0]): ax.contour( - ttha.T, levels=tth, cmap="autumn", linewidths=0.5, linestyles="dashed" + x[i], y[i], ttha[i], levels=tth, cmap="autumn", linewidths=1, linestyles="dashed" ) return ax @@ -665,6 +665,7 @@ def score_distance_scan(self, distances, ax): linestyle="--", label=f"Minimal score: {percentile_10}" ) + ax.legend() ax.set_xlabel("Distance (m)") ax.set_ylabel("Score") ax.set_title("Number of Control Points vs Distance") From 5f0b373e98c42214b3c2dce235693c03d1aea624 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 19 Dec 2024 14:22:32 +0000 Subject: [PATCH 276/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index c2924c2d..0ba01e85 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -581,18 +581,33 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): label = sg.label detector = sg.detector y, x, z = detector.calc_cartesian_positions() - img = ax.scatter(x.flatten(), y.flatten(), c=powder.flatten(), s=1, edgecolors=None, linewidth=0, vmin=np.percentile(powder, 5), vmax=np.percentile(powder, 95)) + img = ax.scatter( + x.flatten(), + y.flatten(), + c=powder.flatten(), + s=1, + edgecolors=None, + linewidth=0, + vmin=np.percentile(powder, 5), + vmax=np.percentile(powder, 95), + ) cbar = plt.colorbar(img, ax=ax, orientation="vertical") cbar.set_label("Intensity") x = np.reshape(x, detector.raw_shape) y = np.reshape(y, detector.raw_shape) z = np.reshape(z, detector.raw_shape) z += ai.dist - ttha = np.arctan2(np.sqrt(x*x+y*y), z) - tth = cp.calibrant.get_2th() + ttha = np.arctan2(np.sqrt(x * x + y * y), z) + tth = cp.calibrant.get_2th() for i in range(x.shape[0]): ax.contour( - x[i], y[i], ttha[i], levels=tth, cmap="autumn", linewidths=1, linestyles="dashed" + x[i], + y[i], + ttha[i], + levels=tth, + cmap="autumn", + linewidths=1, + linestyles="dashed", ) return ax From 82db0f5d7f99beb73130074fcc4a5341740ce7fa Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 06:27:39 -0800 Subject: [PATCH 277/375] [edit] Handled rayonix case where shape is then (1, det.shape) --- lute/tasks/bayfai.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index c2924c2d..1e5391e0 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -869,6 +869,7 @@ def _build_pyFAI_detector(self): self.det = psana.Detector(det_type, self.ds.env()) self.shape = self.det.shape() if det_type.lower() == "rayonix": + self.shape = (1, self.shape[0], self.shape[1]) env = self.ds.env() cfg = env.configStore() self.pixel_size = cfg.get(psana.Rayonix.ConfigV2).pixelWidth() * 1e-6 From 7d342d302bd01fce75dfc805b82d562466d70268 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 06:31:32 -0800 Subject: [PATCH 278/375] [edit] Add label on assembled powder --- lute/tasks/bayfai.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 939e7fff..c4cfec41 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -609,6 +609,8 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): linewidths=1, linestyles="dashed", ) + ax.set_xlabel('X-axis (m)') + ax.set_ylabel('Y-axis (m)') return ax def radial_integration(self, result, calibrant=None, label=None, ax=None): From d023489c15a862e6f21868dffc94fb77c3f159f4 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 19 Dec 2024 14:32:02 +0000 Subject: [PATCH 279/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index c4cfec41..a3e8fdfb 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -609,8 +609,8 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): linewidths=1, linestyles="dashed", ) - ax.set_xlabel('X-axis (m)') - ax.set_ylabel('Y-axis (m)') + ax.set_xlabel("X-axis (m)") + ax.set_ylabel("Y-axis (m)") return ax def radial_integration(self, result, calibrant=None, label=None, ax=None): From f51e242839087bb211395aa90577c95437a49002 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 07:03:31 -0800 Subject: [PATCH 280/375] [edit] Fix plot for Rayonix... --- lute/tasks/bayfai.py | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index c4cfec41..de7956d1 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -581,6 +581,7 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): label = sg.label detector = sg.detector y, x, z = detector.calc_cartesian_positions() + z += ai.dist img = ax.scatter( x.flatten(), y.flatten(), @@ -593,22 +594,33 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) cbar = plt.colorbar(img, ax=ax, orientation="vertical") cbar.set_label("Intensity") - x = np.reshape(x, detector.raw_shape) - y = np.reshape(y, detector.raw_shape) - z = np.reshape(z, detector.raw_shape) - z += ai.dist - ttha = np.arctan2(np.sqrt(x * x + y * y), z) - tth = cp.calibrant.get_2th() - for i in range(x.shape[0]): + if self.det_type.lower() != 'rayonix': + x = np.reshape(x, detector.raw_shape) + y = np.reshape(y, detector.raw_shape) + z = np.reshape(z, detector.raw_shape) + ttha = np.arctan2(np.sqrt(x * x + y * y), z) + tth = cp.calibrant.get_2th() + for i in range(x.shape[0]): + ax.contour( + x[i], + y[i], + ttha[i], + levels=tth, + cmap="autumn", + linewidths=0.5, + linestyles="dashed", + ) + else: + ttha = np.arctan2(np.sqrt(x * x + y * y), z) ax.contour( - x[i], - y[i], - ttha[i], - levels=tth, - cmap="autumn", - linewidths=1, - linestyles="dashed", - ) + x, + y, + ttha, + levels=tth, + cmap="autumn", + linewidths=0.5, + linestyles="dashed", + ) ax.set_xlabel('X-axis (m)') ax.set_ylabel('Y-axis (m)') return ax From 8e20d3d4b5c336fc84df6ab07548e2c615f9c6c1 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 19 Dec 2024 15:04:41 +0000 Subject: [PATCH 281/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index de7956d1..9bbbb08a 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -594,7 +594,7 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) cbar = plt.colorbar(img, ax=ax, orientation="vertical") cbar.set_label("Intensity") - if self.det_type.lower() != 'rayonix': + if self.det_type.lower() != "rayonix": x = np.reshape(x, detector.raw_shape) y = np.reshape(y, detector.raw_shape) z = np.reshape(z, detector.raw_shape) @@ -613,16 +613,16 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): else: ttha = np.arctan2(np.sqrt(x * x + y * y), z) ax.contour( - x, - y, - ttha, - levels=tth, - cmap="autumn", - linewidths=0.5, - linestyles="dashed", - ) - ax.set_xlabel('X-axis (m)') - ax.set_ylabel('Y-axis (m)') + x, + y, + ttha, + levels=tth, + cmap="autumn", + linewidths=0.5, + linestyles="dashed", + ) + ax.set_xlabel("X-axis (m)") + ax.set_ylabel("Y-axis (m)") return ax def radial_integration(self, result, calibrant=None, label=None, ax=None): From 506c9cffa10885be2b77ef41c192f9d59ec90ff6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 07:19:35 -0800 Subject: [PATCH 282/375] [edit] Fix Z coords case where Nonetype --- lute/tasks/bayfai.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index de7956d1..2d99b537 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -581,6 +581,8 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): label = sg.label detector = sg.detector y, x, z = detector.calc_cartesian_positions() + if z is None: + z = np.zeros_like(x) z += ai.dist img = ax.scatter( x.flatten(), From 8436385e302b8a87c681b4089482475f36fb01a6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 07:26:10 -0800 Subject: [PATCH 283/375] [edit] Remove sigmoid in gradient powder computing --- lute/tasks/bayfai.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 860d51e4..1f0952d7 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -1022,10 +1022,6 @@ def _preprocess_powder( Available preprocessing: gradient "magnitude" powder, "gradient" sigmoid powder, "high-pass" filtering, "CAE" convolutional autoencoding (later) """ - - def sigmoid(x): - return 1 / (1 + np.exp(-x)) - if preprocess == None: return powder elif preprocess == "magnitude": @@ -1053,12 +1049,10 @@ def sigmoid(x): calib[:-1, 1:] - calib[:-1, :-1] + calib[1:, 1:] - calib[1:, :-1] ) / 2 powder = gradx_calib + grady_calib - powder = sigmoid(powder) return powder elif preprocess == "high-pass": kernel = np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]]) powder = convolve2d(powder, kernel, mode="same", boundary="symm") - powder = sigmoid(powder) return powder def _update_geometry(self, optimizer): From 239ca4db63b364db3fc4aa46c6abae16dee88c78 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 07:31:44 -0800 Subject: [PATCH 284/375] [edit] Fix plotting for Rayonix, I hate Rayonix --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 1f0952d7..b13c1347 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -596,12 +596,12 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) cbar = plt.colorbar(img, ax=ax, orientation="vertical") cbar.set_label("Intensity") + tth = cp.calibrant.get_2th() if self.det_type.lower() != "rayonix": x = np.reshape(x, detector.raw_shape) y = np.reshape(y, detector.raw_shape) z = np.reshape(z, detector.raw_shape) ttha = np.arctan2(np.sqrt(x * x + y * y), z) - tth = cp.calibrant.get_2th() for i in range(x.shape[0]): ax.contour( x[i], From b1803dea7368fc5bd766e1197b773d5e2cbc44c5 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 07:50:41 -0800 Subject: [PATCH 285/375] [edit] Fix plotting for Rayonix, I hate Rayonix even more --- lute/tasks/bayfai.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index b13c1347..33f21dcb 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -596,28 +596,17 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) cbar = plt.colorbar(img, ax=ax, orientation="vertical") cbar.set_label("Intensity") - tth = cp.calibrant.get_2th() if self.det_type.lower() != "rayonix": x = np.reshape(x, detector.raw_shape) y = np.reshape(y, detector.raw_shape) z = np.reshape(z, detector.raw_shape) - ttha = np.arctan2(np.sqrt(x * x + y * y), z) - for i in range(x.shape[0]): - ax.contour( - x[i], - y[i], - ttha[i], - levels=tth, - cmap="autumn", - linewidths=0.5, - linestyles="dashed", - ) - else: - ttha = np.arctan2(np.sqrt(x * x + y * y), z) + tth = cp.calibrant.get_2th() + ttha = np.arctan2(np.sqrt(x * x + y * y), z) + for i in range(detector.n_modules): ax.contour( - x, - y, - ttha, + x[i], + y[i], + ttha[i], levels=tth, cmap="autumn", linewidths=0.5, @@ -625,6 +614,7 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") + ax.set_title(label) return ax def radial_integration(self, result, calibrant=None, label=None, ax=None): @@ -900,7 +890,6 @@ def _build_pyFAI_detector(self): self.det = psana.Detector(det_type, self.ds.env()) self.shape = self.det.shape() if det_type.lower() == "rayonix": - self.shape = (1, self.shape[0], self.shape[1]) env = self.ds.env() cfg = env.configStore() self.pixel_size = cfg.get(psana.Rayonix.ConfigV2).pixelWidth() * 1e-6 From 02ce3873320c8b886bce361e1e478f183ebdb720 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 08:08:15 -0800 Subject: [PATCH 286/375] [edit] Fix plotting for Rayonix, I hate Rayonix even more --- lute/tasks/bayfai.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 33f21dcb..2e02a17c 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -596,17 +596,28 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) cbar = plt.colorbar(img, ax=ax, orientation="vertical") cbar.set_label("Intensity") + tth = cp.calibrant.get_2th() if self.det_type.lower() != "rayonix": x = np.reshape(x, detector.raw_shape) y = np.reshape(y, detector.raw_shape) z = np.reshape(z, detector.raw_shape) - tth = cp.calibrant.get_2th() - ttha = np.arctan2(np.sqrt(x * x + y * y), z) - for i in range(detector.n_modules): + ttha = np.arctan2(np.sqrt(x * x + y * y), z) + for i in range(detector.n_modules): + ax.contour( + x[i], + y[i], + ttha[i], + levels=tth, + cmap="autumn", + linewidths=0.5, + linestyles="dashed", + ) + else: + ttha = np.arctan2(np.sqrt(x * x + y * y), z) ax.contour( - x[i], - y[i], - ttha[i], + x, + y, + ttha, levels=tth, cmap="autumn", linewidths=0.5, From af57c31dade470fc192fac61c72c6e30633417cb Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 09:18:24 -0800 Subject: [PATCH 287/375] [edit] Correct mypy and ruff error tests --- lute/tasks/bayfai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 2e02a17c..e28f6d1a 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -205,7 +205,7 @@ def min_intensity(self, powder): noise_pixels = powder[nice_pix][powder[nice_pix] <= threshold] noise = np.std(noise_pixels) SNRs.append(signal / noise) - q = Imins[np.argmax(SNRs)] + #q = Imins[np.argmax(SNRs)] Imin = np.percentile(powder[nice_pix], 95) self.q = 95 self.Imin = Imin @@ -1022,7 +1022,7 @@ def _preprocess_powder( Available preprocessing: gradient "magnitude" powder, "gradient" sigmoid powder, "high-pass" filtering, "CAE" convolutional autoencoding (later) """ - if preprocess == None: + if preprocess is None: return powder elif preprocess == "magnitude": sigma = 1 From e6b4bceebdf037af9219abd47043c3c1e62d44a9 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 19 Dec 2024 17:18:53 +0000 Subject: [PATCH 288/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index e28f6d1a..579bf0ca 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -205,7 +205,7 @@ def min_intensity(self, powder): noise_pixels = powder[nice_pix][powder[nice_pix] <= threshold] noise = np.std(noise_pixels) SNRs.append(signal / noise) - #q = Imins[np.argmax(SNRs)] + # q = Imins[np.argmax(SNRs)] Imin = np.percentile(powder[nice_pix], 95) self.q = 95 self.Imin = Imin From b768de24548b4ecaf373ca67ca02177958c1ed58 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 19 Dec 2024 09:23:00 -0800 Subject: [PATCH 289/375] [edit] Correct mypy and ruff error tests --- lute/tasks/bayfai.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index e28f6d1a..a3981fca 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -1008,8 +1008,8 @@ def _extract_powder( return powder def _preprocess_powder( - self, powder: npt.NDArray[np.float64], preprocess: str = None - ) -> npt.NDArray[np.float64]: + self, powder: Optional[npt.NDArray[np.float64]], preprocess: Optional[str] = None + ) -> Optional[npt.NDArray[np.float64]]: """ Preprocess extracted powder for enhancing optimization @@ -1054,6 +1054,9 @@ def _preprocess_powder( kernel = np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]]) powder = convolve2d(powder, kernel, mode="same", boundary="symm") return powder + else: + logger.warning(f"Preprocessing technique {preprocess} not recognized.") + return None def _update_geometry(self, optimizer): """ From 650229c3d356d1dc2badb7e561ccc26597705c20 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 19 Dec 2024 17:23:31 +0000 Subject: [PATCH 290/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7fa42d7e..e5350ce8 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -1008,7 +1008,9 @@ def _extract_powder( return powder def _preprocess_powder( - self, powder: Optional[npt.NDArray[np.float64]], preprocess: Optional[str] = None + self, + powder: Optional[npt.NDArray[np.float64]], + preprocess: Optional[str] = None, ) -> Optional[npt.NDArray[np.float64]]: """ Preprocess extracted powder for enhancing optimization From 8d8bfb6ae5e0e9adeecf192e0f98c9b8ae8f1d66 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 09:22:38 -0800 Subject: [PATCH 291/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7fa42d7e..d05a19dc 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -372,6 +372,7 @@ def bayes_opt_center( for i in range(n_iterations): # 1. Generate the Acquisition Function values using the Gaussian Process Regressor + y_pred = gp_model.predict(X_norm, return_std=False) af_values = af(X_norm, gp_model, best_score, hyperparam) af_values[visited_idx] = -np.inf @@ -405,6 +406,7 @@ def bayes_opt_center( bo_history[f"iteration_{i+1}"] = { "param": X[new_idx], "score": score, + "pred": y_pred, } X_samples = np.append(X_samples, [X[new_idx]], axis=0) X_norm_samples = np.append(X_norm_samples, [X_norm[new_idx]], axis=0) @@ -560,6 +562,54 @@ def finalize(self): self.score = self.scan["score"][index] self.best_idx = self.scan["best_idx"][index] + def bayes_opt_animation(self, bo_history, bounds, res, dist): + import numpy as np + import matplotlib.pyplot as plt + from matplotlib.animation import FuncAnimation + + num_frames = len(bo_history) + + poni1 = np.arange(bounds["poni1"][0], bounds["poni1"][1] + res, res) + poni2 = np.arange(bounds["poni2"][0], bounds["poni2"][1] + res, res) + X, Y = np.meshgrid(poni2, poni1) + + sample_points = [bo_history[f"init_sample_{i+1}"]["param"] for i in range(num_frames)] + + fig, ax = plt.subplots() + score_plot = ax.pcolormesh(X, Y, bo_history['iteration_1']['pred'], cmap='viridis', shading='auto') + colorbar = plt.colorbar(score_plot, ax=ax) + first_point = bo_history['iteration_1']['param'] + points_red, = ax.plot([first_point[2]], [first_point[1]], 'ro', label='Next Sampled Point') + points_green, = ax.plot([p[2] for p in sample_points], [p[1] for p in sample_points], 'o', color='green', label='Sampled Points') + points_orange, = ax.plot([], [], 'o', color='green', label='Sampled Points') + + def update(frame): + iteration_key = f"iteration_{frame + 1}" + + pred = bo_history[iteration_key]['pred'] + score_plot.set_array(pred.ravel()) + score_plot.set_clim(vmin=pred.min(), vmax=pred.max()) + colorbar.update_normal(score_plot) + + current_point = bo_history[iteration_key]['param'] + previous_points = [bo_history[f"iteration_{i + 1}"]['param'] for i in range(frame)] + + points_red.set_data([current_point[2]], [current_point[1]]) + if previous_points: + points_orange.set_data( + [p[2] for p in previous_points], + [p[1] for p in previous_points] + ) + + return score_plot, points_red, points_orange + + anim = FuncAnimation( + fig, update, frames=num_frames, interval=500, blit=True + ) + filename = f"bayes_opt_{self.exp}_r{self.run}_dist_{int(dist * 1000):03d}mm.gif" + anim.save(filename, writer="imagemagick") + # anim.save("bayesian_optimization.mp4", writer="ffmpeg") + def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): """ Display an image with the control points and the calibrated rings From d4e2f146cbfa0635638233e50bdac941096875ba Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 17:23:17 +0000 Subject: [PATCH 292/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index eb1b8641..8e8b92de 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -573,39 +573,50 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): poni2 = np.arange(bounds["poni2"][0], bounds["poni2"][1] + res, res) X, Y = np.meshgrid(poni2, poni1) - sample_points = [bo_history[f"init_sample_{i+1}"]["param"] for i in range(num_frames)] + sample_points = [ + bo_history[f"init_sample_{i+1}"]["param"] for i in range(num_frames) + ] fig, ax = plt.subplots() - score_plot = ax.pcolormesh(X, Y, bo_history['iteration_1']['pred'], cmap='viridis', shading='auto') + score_plot = ax.pcolormesh( + X, Y, bo_history["iteration_1"]["pred"], cmap="viridis", shading="auto" + ) colorbar = plt.colorbar(score_plot, ax=ax) - first_point = bo_history['iteration_1']['param'] - points_red, = ax.plot([first_point[2]], [first_point[1]], 'ro', label='Next Sampled Point') - points_green, = ax.plot([p[2] for p in sample_points], [p[1] for p in sample_points], 'o', color='green', label='Sampled Points') - points_orange, = ax.plot([], [], 'o', color='green', label='Sampled Points') + first_point = bo_history["iteration_1"]["param"] + (points_red,) = ax.plot( + [first_point[2]], [first_point[1]], "ro", label="Next Sampled Point" + ) + (points_green,) = ax.plot( + [p[2] for p in sample_points], + [p[1] for p in sample_points], + "o", + color="green", + label="Sampled Points", + ) + (points_orange,) = ax.plot([], [], "o", color="green", label="Sampled Points") def update(frame): iteration_key = f"iteration_{frame + 1}" - pred = bo_history[iteration_key]['pred'] + pred = bo_history[iteration_key]["pred"] score_plot.set_array(pred.ravel()) score_plot.set_clim(vmin=pred.min(), vmax=pred.max()) colorbar.update_normal(score_plot) - current_point = bo_history[iteration_key]['param'] - previous_points = [bo_history[f"iteration_{i + 1}"]['param'] for i in range(frame)] + current_point = bo_history[iteration_key]["param"] + previous_points = [ + bo_history[f"iteration_{i + 1}"]["param"] for i in range(frame) + ] points_red.set_data([current_point[2]], [current_point[1]]) if previous_points: points_orange.set_data( - [p[2] for p in previous_points], - [p[1] for p in previous_points] + [p[2] for p in previous_points], [p[1] for p in previous_points] ) return score_plot, points_red, points_orange - - anim = FuncAnimation( - fig, update, frames=num_frames, interval=500, blit=True - ) + + anim = FuncAnimation(fig, update, frames=num_frames, interval=500, blit=True) filename = f"bayes_opt_{self.exp}_r{self.run}_dist_{int(dist * 1000):03d}mm.gif" anim.save(filename, writer="imagemagick") # anim.save("bayesian_optimization.mp4", writer="ffmpeg") From 7969cd9929e5138b0f968002a52511dec2d85339 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 09:24:12 -0800 Subject: [PATCH 293/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index eb1b8641..203646b1 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -539,6 +539,10 @@ def bayes_opt_geom( ) self.comm.Barrier() + self.bayes_opt_animation(results["bo_history"], bounds, res, dist) + + self.comm.Barrier() + self.scan = {} self.scan["bo_history"] = self.comm.gather(results["bo_history"], root=0) self.scan["params"] = self.comm.gather(results["params"], root=0) From 4c9ddbfa4e620355c3dff00eaafd1fc7f2cd8751 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 09:33:40 -0800 Subject: [PATCH 294/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index db9e092c..0666a71a 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -571,7 +571,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): import matplotlib.pyplot as plt from matplotlib.animation import FuncAnimation - num_frames = len(bo_history) + num_frames = len(bo_history) // 2 poni1 = np.arange(bounds["poni1"][0], bounds["poni1"][1] + res, res) poni2 = np.arange(bounds["poni2"][0], bounds["poni2"][1] + res, res) From 6a9d1c888c338f6b0fe3e858a52ddcce176a85a2 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 17:34:02 +0000 Subject: [PATCH 295/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 0666a71a..12b40ded 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -571,7 +571,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): import matplotlib.pyplot as plt from matplotlib.animation import FuncAnimation - num_frames = len(bo_history) // 2 + num_frames = len(bo_history) // 2 poni1 = np.arange(bounds["poni1"][0], bounds["poni1"][1] + res, res) poni2 = np.arange(bounds["poni2"][0], bounds["poni2"][1] + res, res) From d51ade0d6fb9b2093b313253620f1956ea877b32 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 09:43:36 -0800 Subject: [PATCH 296/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 0666a71a..50d18325 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -581,9 +581,11 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): bo_history[f"init_sample_{i+1}"]["param"] for i in range(num_frames) ] + pred = bo_history["iteration_1"]["pred"] + pred = np.reshape(pred, X.shape) fig, ax = plt.subplots() score_plot = ax.pcolormesh( - X, Y, bo_history["iteration_1"]["pred"], cmap="viridis", shading="auto" + X, Y, pred, cmap="viridis", shading="auto" ) colorbar = plt.colorbar(score_plot, ax=ax) first_point = bo_history["iteration_1"]["param"] @@ -602,7 +604,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): def update(frame): iteration_key = f"iteration_{frame + 1}" - pred = bo_history[iteration_key]["pred"] + pred = np.reshape(bo_history[iteration_key]["pred"], X.shape) score_plot.set_array(pred.ravel()) score_plot.set_clim(vmin=pred.min(), vmax=pred.max()) colorbar.update_normal(score_plot) From b1683ca49d28ab33efd3a4bc0116d945da94ed1f Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 17:44:07 +0000 Subject: [PATCH 297/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7d2859f1..d968bab8 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -584,9 +584,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): pred = bo_history["iteration_1"]["pred"] pred = np.reshape(pred, X.shape) fig, ax = plt.subplots() - score_plot = ax.pcolormesh( - X, Y, pred, cmap="viridis", shading="auto" - ) + score_plot = ax.pcolormesh(X, Y, pred, cmap="viridis", shading="auto") colorbar = plt.colorbar(score_plot, ax=ax) first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( From 0cd7a3749515a0159880ce581874ec8a38c42cbb Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 09:58:40 -0800 Subject: [PATCH 298/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7d2859f1..526200e6 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,17 +589,22 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): ) colorbar = plt.colorbar(score_plot, ax=ax) first_point = bo_history["iteration_1"]["param"] - (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], "ro", label="Next Sampled Point" + (points_red,) = ax.scatter( + [first_point[2]], [first_point[1]], "ro", s=2, label="Next Sampled Point" ) - (points_green,) = ax.plot( + (points_green,) = ax.scatter( [p[2] for p in sample_points], [p[1] for p in sample_points], "o", + s=1, color="green", label="Sampled Points", ) - (points_orange,) = ax.plot([], [], "o", color="green", label="Sampled Points") + (points_orange,) = ax.scatter([], [], "o", s=1, color="orange", label="Previous Points") + ax.set_title(f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m") + ax.set_xlabel('Y-axis (m)') + ax.set_ylabel('X-axis (m)') + ax.legend() def update(frame): iteration_key = f"iteration_{frame + 1}" From 271fc9fe1458c6d9737a612769913008d1230509 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 17:59:11 +0000 Subject: [PATCH 299/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 07784ff1..13499352 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -598,10 +598,14 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): color="green", label="Sampled Points", ) - (points_orange,) = ax.scatter([], [], "o", s=1, color="orange", label="Previous Points") - ax.set_title(f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m") - ax.set_xlabel('Y-axis (m)') - ax.set_ylabel('X-axis (m)') + (points_orange,) = ax.scatter( + [], [], "o", s=1, color="orange", label="Previous Points" + ) + ax.set_title( + f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" + ) + ax.set_xlabel("Y-axis (m)") + ax.set_ylabel("X-axis (m)") ax.legend() def update(frame): From 86dd4d1c0a09644a33db51c84a0ddec599387306 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 10:12:11 -0800 Subject: [PATCH 300/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 07784ff1..3f18c167 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -585,7 +585,8 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): pred = np.reshape(pred, X.shape) fig, ax = plt.subplots() score_plot = ax.pcolormesh(X, Y, pred, cmap="viridis", shading="auto") - colorbar = plt.colorbar(score_plot, ax=ax) + colorbar = plt.colorbar(score_plot, ax=ax, orientation='vertical') + colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.scatter( [first_point[2]], [first_point[1]], "ro", s=2, label="Next Sampled Point" @@ -626,7 +627,7 @@ def update(frame): return score_plot, points_red, points_orange anim = FuncAnimation(fig, update, frames=num_frames, interval=500, blit=True) - filename = f"bayes_opt_{self.exp}_r{self.run}_dist_{int(dist * 1000):03d}mm.gif" + filename = f"../tests/animation/bayes_opt_{self.exp}_r{self.run}_dist_{int(dist * 1000):03d}mm.gif" anim.save(filename, writer="imagemagick") # anim.save("bayesian_optimization.mp4", writer="ffmpeg") From c589cb369262c7eb2327bb77fb22c53bf4a3aea5 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 18:12:47 +0000 Subject: [PATCH 301/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index cdff4348..9b2dad68 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -585,7 +585,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): pred = np.reshape(pred, X.shape) fig, ax = plt.subplots() score_plot = ax.pcolormesh(X, Y, pred, cmap="viridis", shading="auto") - colorbar = plt.colorbar(score_plot, ax=ax, orientation='vertical') + colorbar = plt.colorbar(score_plot, ax=ax, orientation="vertical") colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.scatter( From 80ad520ff7ded0d456b8c566a4efb096d54761cd Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 10:17:59 -0800 Subject: [PATCH 302/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index cdff4348..57b58322 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,18 +589,17 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.scatter( - [first_point[2]], [first_point[1]], "ro", s=2, label="Next Sampled Point" + [first_point[2]], [first_point[1]], s=2, label="Next Sampled Point" ) (points_green,) = ax.scatter( [p[2] for p in sample_points], [p[1] for p in sample_points], - "o", s=1, color="green", label="Sampled Points", ) (points_orange,) = ax.scatter( - [], [], "o", s=1, color="orange", label="Previous Points" + [], [], s=1, color="orange", label="Previous Points" ) ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" From 0397ba3e82e829a1bd9142ea3e03cb06fa481c30 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 10:25:03 -0800 Subject: [PATCH 303/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index a3e8eca6..12578ece 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,7 +589,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.scatter( - [first_point[2]], [first_point[1]], s=2, label="Next Sampled Point" + first_point[2], first_point[1], s=2, color='red', label="Next Sampled Point" ) (points_green,) = ax.scatter( [p[2] for p in sample_points], From 4afcd47873f7fd056222d8b072086b7129628204 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 18:25:28 +0000 Subject: [PATCH 304/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 12578ece..0c93f47a 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,7 +589,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.scatter( - first_point[2], first_point[1], s=2, color='red', label="Next Sampled Point" + first_point[2], first_point[1], s=2, color="red", label="Next Sampled Point" ) (points_green,) = ax.scatter( [p[2] for p in sample_points], From 5eefc840e7e737d8ee998ba88a636f63097fc759 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 10:31:31 -0800 Subject: [PATCH 305/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 12578ece..651fa26f 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,7 +589,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.scatter( - first_point[2], first_point[1], s=2, color='red', label="Next Sampled Point" + [first_point[2]], [first_point[1]], s=2, color='red', label="Next Sampled Point" ) (points_green,) = ax.scatter( [p[2] for p in sample_points], From 5230c510388d6a7f94d8eb6f8932d9fb07bf2182 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 10:32:19 -0800 Subject: [PATCH 306/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 76c32cb7..651fa26f 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,11 +589,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.scatter( -<<<<<<< HEAD [first_point[2]], [first_point[1]], s=2, color='red', label="Next Sampled Point" -======= - first_point[2], first_point[1], s=2, color="red", label="Next Sampled Point" ->>>>>>> 4afcd47873f7fd056222d8b072086b7129628204 ) (points_green,) = ax.scatter( [p[2] for p in sample_points], From 0d7eebffdf384aa658bf3b049f6cc95b3611d935 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 18:32:42 +0000 Subject: [PATCH 307/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 651fa26f..a9a3571e 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,7 +589,11 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.scatter( - [first_point[2]], [first_point[1]], s=2, color='red', label="Next Sampled Point" + [first_point[2]], + [first_point[1]], + s=2, + color="red", + label="Next Sampled Point", ) (points_green,) = ax.scatter( [p[2] for p in sample_points], From 106823ba80581c44bf6d5613176a28bf1de37763 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 10:39:35 -0800 Subject: [PATCH 308/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 651fa26f..9dda6e71 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -588,18 +588,17 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar = plt.colorbar(score_plot, ax=ax, orientation="vertical") colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] - (points_red,) = ax.scatter( - [first_point[2]], [first_point[1]], s=2, color='red', label="Next Sampled Point" + (points_red,) = ax.plot( + [first_point[2]], [first_point[1]], color='red', label="Next Sampled Point" ) - (points_green,) = ax.scatter( + (points_green,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - s=1, color="green", label="Sampled Points", ) - (points_orange,) = ax.scatter( - [], [], s=1, color="orange", label="Previous Points" + (points_orange,) = ax.plot( + [], [], color="orange", label="Previous Points" ) ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" From f1a887beae88235e855662f5ceefb62bd3a32d10 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 18:40:22 +0000 Subject: [PATCH 309/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 9dda6e71..1f8264f2 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,7 +589,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], color='red', label="Next Sampled Point" + [first_point[2]], [first_point[1]], color="red", label="Next Sampled Point" ) (points_green,) = ax.plot( [p[2] for p in sample_points], @@ -597,9 +597,7 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): color="green", label="Sampled Points", ) - (points_orange,) = ax.plot( - [], [], color="orange", label="Previous Points" - ) + (points_orange,) = ax.plot([], [], color="orange", label="Previous Points") ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" ) From 986ae37fa7b75b5706c70939151e1ac2f430c7f0 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 10:50:32 -0800 Subject: [PATCH 310/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 9dda6e71..7d4bdd07 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,16 +589,18 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], color='red', label="Next Sampled Point" + [first_point[2]], [first_point[1]], marker="ro", markersize=2, label="Next Sampled Point" ) - (points_green,) = ax.plot( + (points_blue,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - color="green", + marker = "o", + color = "green", + markersize=1, label="Sampled Points", ) (points_orange,) = ax.plot( - [], [], color="orange", label="Previous Points" + [], [], marker="o", color="orange", markersize=1, label="Previous Points" ) ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" From cd6214d47041486dc03769a41029972a80471e9e Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 18:51:18 +0000 Subject: [PATCH 311/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7d4bdd07..fafc76b7 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,13 +589,17 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], marker="ro", markersize=2, label="Next Sampled Point" + [first_point[2]], + [first_point[1]], + marker="ro", + markersize=2, + label="Next Sampled Point", ) (points_blue,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - marker = "o", - color = "green", + marker="o", + color="green", markersize=1, label="Sampled Points", ) From b978ce804bb73f54fe07c97aa48fe3a3206b49a3 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 10:59:24 -0800 Subject: [PATCH 312/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7d4bdd07..1e012e37 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,18 +589,18 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], marker="ro", markersize=2, label="Next Sampled Point" + [first_point[2]], [first_point[1]], marker=".", color="red", markersize=2, label="Next Sampled Point" ) (points_blue,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - marker = "o", + marker = ".", color = "green", markersize=1, label="Sampled Points", ) (points_orange,) = ax.plot( - [], [], marker="o", color="orange", markersize=1, label="Previous Points" + [], [], marker=".", color="orange", markersize=1, label="Previous Points" ) ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" From f410add2178fde7fa4a36c5d46714a84a7f9f89c Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 19:00:09 +0000 Subject: [PATCH 313/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 1e012e37..a381a1ce 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,13 +589,18 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], marker=".", color="red", markersize=2, label="Next Sampled Point" + [first_point[2]], + [first_point[1]], + marker=".", + color="red", + markersize=2, + label="Next Sampled Point", ) (points_blue,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - marker = ".", - color = "green", + marker=".", + color="green", markersize=1, label="Sampled Points", ) From d714502886dee4bb9763ed821317eb18d67a6f7f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 11:06:59 -0800 Subject: [PATCH 314/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 1e012e37..ef3235f8 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,18 +589,18 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], marker=".", color="red", markersize=2, label="Next Sampled Point" + [first_point[2]], [first_point[1]], marker="o", color="red", markersize=2, label="Next Sampled Point" ) (points_blue,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - marker = ".", + marker = "o", color = "green", markersize=1, label="Sampled Points", ) (points_orange,) = ax.plot( - [], [], marker=".", color="orange", markersize=1, label="Previous Points" + [], [], marker="o", color="orange", markersize=1, label="Previous Points" ) ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" From 4cca08f45a43ded8824ec4861a6d09287f46e7bb Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 19:07:47 +0000 Subject: [PATCH 315/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index ef3235f8..2fcd1200 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,13 +589,18 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], marker="o", color="red", markersize=2, label="Next Sampled Point" + [first_point[2]], + [first_point[1]], + marker="o", + color="red", + markersize=2, + label="Next Sampled Point", ) (points_blue,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - marker = "o", - color = "green", + marker="o", + color="green", markersize=1, label="Sampled Points", ) From 8cf669832842e94b6d72b298a987e0145e8d6dcf Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 11:59:25 -0800 Subject: [PATCH 316/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index ef3235f8..7850a591 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,18 +589,17 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], marker="o", color="red", markersize=2, label="Next Sampled Point" + [first_point[2]], [first_point[1]], "o", color="red", label="Next Sampled Point" ) (points_blue,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], marker = "o", color = "green", - markersize=1, label="Sampled Points", ) (points_orange,) = ax.plot( - [], [], marker="o", color="orange", markersize=1, label="Previous Points" + [], [], marker="o", color="orange", label="Previous Points" ) ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" From 23f26c4d4b8fbb95bac0e52164bc5e026772e06c Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 20:00:18 +0000 Subject: [PATCH 317/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7850a591..d339bfeb 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,13 +589,17 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], "o", color="red", label="Next Sampled Point" + [first_point[2]], + [first_point[1]], + "o", + color="red", + label="Next Sampled Point", ) (points_blue,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - marker = "o", - color = "green", + marker="o", + color="green", label="Sampled Points", ) (points_orange,) = ax.plot( From af233ecac61947084afd16156ad5bdb249c0d9c9 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 20 Dec 2024 12:08:39 -0800 Subject: [PATCH 318/375] [edit] Add animation for website description of BayFAI --- lute/tasks/bayfai.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7850a591..58ec7a33 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -589,17 +589,17 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] (points_red,) = ax.plot( - [first_point[2]], [first_point[1]], "o", color="red", label="Next Sampled Point" + [first_point[2]], [first_point[1]], "ro", label="Next Sampled Point" ) (points_blue,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - marker = "o", - color = "green", + marker = "bo", label="Sampled Points", + linestyle=None ) (points_orange,) = ax.plot( - [], [], marker="o", color="orange", label="Previous Points" + [], [], marker="o", color="orange", label="Previous Points", linestyle=None ) ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" From 47f7b13cdedd86d66f2849a034f90113d7a7c5c3 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 20 Dec 2024 20:11:39 +0000 Subject: [PATCH 319/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7f86e9c2..ccbb3dff 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -603,7 +603,14 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): linestyle=None, ) (points_orange,) = ax.plot( - [], [], marker="o", color="orange", markersize=1, alpha=0.7, label="Previous Points", linestyle=None + [], + [], + marker="o", + color="orange", + markersize=1, + alpha=0.7, + label="Previous Points", + linestyle=None, ) ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" From b3ce866fcccc483e50b193a1be18e1ec15a46880 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 02:14:52 -0800 Subject: [PATCH 320/375] [edit] Change n_init and n_iter to 20 80 + trying to fix animation --- lute/io/models/bayfai.py | 4 ++-- lute/tasks/bayfai.py | 33 ++++++++++++++++----------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/lute/io/models/bayfai.py b/lute/io/models/bayfai.py index 4b7b4797..e1e86a3d 100644 --- a/lute/io/models/bayfai.py +++ b/lute/io/models/bayfai.py @@ -49,12 +49,12 @@ class BayesGeomOptParameters(BaseModel): ) n_samples: Optional[int] = Field( - 50, + 20, description="Number of random starts to initialize the Bayesian optimization.", ) n_iterations: Optional[int] = Field( - 50, + 80, description="Number of iterations to run the Bayesian optimization.", ) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7f86e9c2..e8d268d8 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -219,10 +219,10 @@ def bayes_opt_center( dist, bounds, res, - Imin=90, + Imin, max_rings=5, - n_samples=50, - n_iterations=50, + n_samples=20, + n_iterations=80, af="ucb", hyperparam=None, prior=True, @@ -456,10 +456,9 @@ def bayes_opt_geom( powder, bounds, res, - Imin=99, max_rings=5, - n_samples=50, - n_iterations=50, + n_samples=20, + n_iterations=80, af="ucb", hyperparam=None, prior=True, @@ -476,8 +475,6 @@ def bayes_opt_geom( Dictionary of bounds and resolution for search parameters res : float Resolution of the grid used to discretize the parameter search space - Imin : float - Minimum intensity threshold for control point extraction based on intensity distribution percentile max_rings : int Maximum number of rings to use for control point extraction n_samples : int @@ -539,7 +536,7 @@ def bayes_opt_geom( ) self.comm.Barrier() - self.bayes_opt_animation(results["bo_history"], bounds, res, dist) + self.bayes_opt_animation(results["bo_history"], n_samples, n_iterations, bounds, res, dist) self.comm.Barrier() @@ -566,19 +563,19 @@ def finalize(self): self.score = self.scan["score"][index] self.best_idx = self.scan["best_idx"][index] - def bayes_opt_animation(self, bo_history, bounds, res, dist): + def bayes_opt_animation(self, bo_history, n_samples, n_iterations, bounds, res, dist): import numpy as np import matplotlib.pyplot as plt from matplotlib.animation import FuncAnimation - num_frames = len(bo_history) // 2 + num_frames = n_iterations poni1 = np.arange(bounds["poni1"][0], bounds["poni1"][1] + res, res) poni2 = np.arange(bounds["poni2"][0], bounds["poni2"][1] + res, res) X, Y = np.meshgrid(poni2, poni1) sample_points = [ - bo_history[f"init_sample_{i+1}"]["param"] for i in range(num_frames) + bo_history[f"init_sample_{i+1}"]["param"] for i in range(n_samples) ] pred = bo_history["iteration_1"]["pred"] @@ -594,16 +591,18 @@ def bayes_opt_animation(self, bo_history, bounds, res, dist): "ro", label="Next Sampled Point", ) - (points_blue,) = ax.plot( + (points_green,) = ax.plot( [p[2] for p in sample_points], [p[1] for p in sample_points], - "bo", + marker="o", + color="green", + markersize=2, label="Sampled Points", - alpha=0.7, - linestyle=None, + alpha=0.5, + linestyle="", ) (points_orange,) = ax.plot( - [], [], marker="o", color="orange", markersize=1, alpha=0.7, label="Previous Points", linestyle=None + [], [], marker="o", color="orange", markersize=2, alpha=0.5, label="Previous Points", linestyle="", ) ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" From 5345ece84a773710e684ed686cd3db36aba001b4 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 10:15:55 +0000 Subject: [PATCH 321/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index dd14855b..2ae8b2c6 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -536,7 +536,9 @@ def bayes_opt_geom( ) self.comm.Barrier() - self.bayes_opt_animation(results["bo_history"], n_samples, n_iterations, bounds, res, dist) + self.bayes_opt_animation( + results["bo_history"], n_samples, n_iterations, bounds, res, dist + ) self.comm.Barrier() @@ -563,7 +565,9 @@ def finalize(self): self.score = self.scan["score"][index] self.best_idx = self.scan["best_idx"][index] - def bayes_opt_animation(self, bo_history, n_samples, n_iterations, bounds, res, dist): + def bayes_opt_animation( + self, bo_history, n_samples, n_iterations, bounds, res, dist + ): import numpy as np import matplotlib.pyplot as plt from matplotlib.animation import FuncAnimation From 026aa9f0c20fab7992590fbeb451462321e3c16f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 02:22:29 -0800 Subject: [PATCH 322/375] [edit] Finally fixed animation --- lute/tasks/bayfai.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index dd14855b..cfd631c4 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -596,9 +596,9 @@ def bayes_opt_animation(self, bo_history, n_samples, n_iterations, bounds, res, [p[1] for p in sample_points], marker="o", color="green", - markersize=2, + markersize=5, label="Sampled Points", - alpha=0.5, + alpha=0.2, linestyle="", ) (points_orange,) = ax.plot( @@ -606,8 +606,8 @@ def bayes_opt_animation(self, bo_history, n_samples, n_iterations, bounds, res, [], marker="o", color="orange", - markersize=2, - alpha=0.5, + markersize=5, + alpha=0.2, label="Previous Points", linestyle="", ) From 38d057deceb0bf6d942254ff6bd2ce3cab9a739d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 02:26:21 -0800 Subject: [PATCH 323/375] [edit] Change convergence plot for more visualization --- lute/tasks/bayfai.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 2881a692..52028421 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -911,10 +911,16 @@ def visualize_results( # Plotting BO convergence ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) scores = [bo_history[key]["score"] for key in bo_history.keys()] - ax1.plot(np.maximum.accumulate(scores)) + ax1.plot(scores) ax1.set_xticks(np.arange(len(scores), step=20)) + ax1.axvline( + self.index, + color="green", + linestyle="--", + label=f"Best score reached at iteration {self.index}", + ) ax1.set_xlabel("Iteration") - ax1.set_ylabel("Best score so far") + ax1.set_ylabel("Number of Control Points") ax1.set_title(f"Convergence Plot, best score: {self.scan['score'][self.index]}") icol += 1 From 0226180085b888bcb8d39aeac2c731173efff105 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 02:34:28 -0800 Subject: [PATCH 324/375] [edit] Format ticks on animation --- lute/tasks/bayfai.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 52028421..7d9f602d 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -602,7 +602,7 @@ def bayes_opt_animation( color="green", markersize=5, label="Sampled Points", - alpha=0.2, + alpha=0.3, linestyle="", ) (points_orange,) = ax.plot( @@ -611,7 +611,7 @@ def bayes_opt_animation( marker="o", color="orange", markersize=5, - alpha=0.2, + alpha=0.3, label="Previous Points", linestyle="", ) @@ -621,6 +621,8 @@ def bayes_opt_animation( ax.set_xlabel("Y-axis (m)") ax.set_ylabel("X-axis (m)") ax.legend() + ax.tick_params(axis='x', labelrotation=45, labelsize=8) + ax.tick_params(axis='y', labelrotation=45, labelsize=8) def update(frame): iteration_key = f"iteration_{frame + 1}" From e9de7756a35890775651d0fdc5744b2521e6147e Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 10:34:52 +0000 Subject: [PATCH 325/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7d9f602d..0abdc9cd 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -621,8 +621,8 @@ def bayes_opt_animation( ax.set_xlabel("Y-axis (m)") ax.set_ylabel("X-axis (m)") ax.legend() - ax.tick_params(axis='x', labelrotation=45, labelsize=8) - ax.tick_params(axis='y', labelrotation=45, labelsize=8) + ax.tick_params(axis="x", labelrotation=45, labelsize=8) + ax.tick_params(axis="y", labelrotation=45, labelsize=8) def update(frame): iteration_key = f"iteration_{frame + 1}" From 654c06a41b019f925f0d59c50db2770ee39872cf Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 02:35:27 -0800 Subject: [PATCH 326/375] [edit] Format ticks on animation --- lute/tasks/bayfai.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 7d9f602d..616a1179 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -715,6 +715,8 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") + ax.tick_params(axis='x', labelrotation=45, labelsize=8) + ax.tick_params(axis='y', labelrotation=45, labelsize=8) ax.set_title(label) return ax From f317aa94a46a6058aaca09d839908e62489bb418 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 10:35:58 +0000 Subject: [PATCH 327/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index e938cb72..9df9cd26 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -715,8 +715,8 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") - ax.tick_params(axis='x', labelrotation=45, labelsize=8) - ax.tick_params(axis='y', labelrotation=45, labelsize=8) + ax.tick_params(axis="x", labelrotation=45, labelsize=8) + ax.tick_params(axis="y", labelrotation=45, labelsize=8) ax.set_title(label) return ax From 716fd5b56c8e23847aec4293941993a008b59d1c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 02:40:27 -0800 Subject: [PATCH 328/375] [edit] Format ticks on animation --- lute/tasks/bayfai.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index e938cb72..242d0e8b 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -621,8 +621,8 @@ def bayes_opt_animation( ax.set_xlabel("Y-axis (m)") ax.set_ylabel("X-axis (m)") ax.legend() - ax.tick_params(axis="x", labelrotation=45, labelsize=8) - ax.tick_params(axis="y", labelrotation=45, labelsize=8) + ax.tick_params(axis="x", labelsize=8) + ax.tick_params(axis="y", labelsize=8) def update(frame): iteration_key = f"iteration_{frame + 1}" @@ -715,8 +715,8 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") - ax.tick_params(axis='x', labelrotation=45, labelsize=8) - ax.tick_params(axis='y', labelrotation=45, labelsize=8) + ax.tick_params(axis='x', labelsize=8) + ax.tick_params(axis='y', labelsize=8) ax.set_title(label) return ax From 69ecbb771978a9a1df84207e3c1be0a05dd6da3f Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 10:41:13 +0000 Subject: [PATCH 329/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 242d0e8b..1a5c2947 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -715,8 +715,8 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): ) ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") - ax.tick_params(axis='x', labelsize=8) - ax.tick_params(axis='y', labelsize=8) + ax.tick_params(axis="x", labelsize=8) + ax.tick_params(axis="y", labelsize=8) ax.set_title(label) return ax From d1dd3090da4fd6abe7156f0a41745078437d8192 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 02:43:43 -0800 Subject: [PATCH 330/375] [edit] Fix error in legend plotting for convergence plot --- lute/tasks/bayfai.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 242d0e8b..5e87df15 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -918,13 +918,14 @@ def visualize_results( ax1.plot(scores) ax1.set_xticks(np.arange(len(scores), step=20)) ax1.axvline( - self.index, + self.scan["best_idx"][self.index], color="green", linestyle="--", - label=f"Best score reached at iteration {self.index}", + label=f"Best score reached at iteration {self.scan["best_idx"][self.index]}", ) ax1.set_xlabel("Iteration") ax1.set_ylabel("Number of Control Points") + ax1.legend() ax1.set_title(f"Convergence Plot, best score: {self.scan['score'][self.index]}") icol += 1 From d13ffe23276e9efbd94e9ed1d4817f3a17afcb0a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 04:23:11 -0800 Subject: [PATCH 331/375] [edit] Computation of detector resolution with plots --- lute/tasks/bayfai.py | 110 ++++++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 34 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index de4dc003..dd4585a1 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -650,30 +650,31 @@ def update(frame): anim.save(filename, writer="imagemagick") # anim.save("bayesian_optimization.mp4", writer="ffmpeg") - def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): + def powder_and_resolution(self, sg, distance, beam_center, ax=None): """ - Display an image with the control points and the calibrated rings + Display an image with the control points and the calibrated rings as well as detector resolutions Parameters ---------- - powder : np.ndarray + sg : SingleGeometry + SingleGeometry object containing powder and geometry data + distance : float + Distance of the detector + beam_center : Tuple(float) + Beam center coordinates """ if ax is None: _fig, ax = plt.subplots() if sg is not None: - if powder is None: - powder = sg.image - if cp is None: - cp = sg.control_points - if ai is None: - ai = sg.geometry_refinement - if label is None: - label = sg.label + powder = sg.image + cp = sg.control_points + ai = sg.geometry_refinement + label = sg.label detector = sg.detector y, x, z = detector.calc_cartesian_positions() if z is None: z = np.zeros_like(x) - z += ai.dist + z += distance img = ax.scatter( x.flatten(), y.flatten(), @@ -713,12 +714,55 @@ def display(self, powder=None, cp=None, ai=None, label=None, sg=None, ax=None): linewidths=0.5, linestyles="dashed", ) + + cx, cy = beam_center + d = np.sqrt((x - cx) ** 2 + (y - cy) ** 2) + + closest_pixel_index = np.argmin(d) + closest_pixel = d.flatten()[closest_pixel_index] + closest_q = (4 * np.pi * np.sin(np.arctan2(closest_pixel / distance)) / self.wavelength) * 1e10 + closest_resol = 2 * np.pi / closest_q + + furthest_pixel_index = np.argmax(d) + furthest_pixel = d.flatten()[furthest_pixel_index] + furthest_q = (4 * np.pi * np.sin(np.arctan2(furthest_pixel / distance)) / self.wavelength) * 1e10 + furthest_resol = 2 * np.pi / furthest_q + + xmin, xmax = x.min(), x.max() + ymin, ymax = y.min(), y.max() + d_left = abs(cx - xmin) + d_right = abs(cx - xmax) + d_bottom = abs(cy - ymin) + d_top = abs(cy - ymax) + border_distances = [d_left, d_right, d_bottom, d_top] + border_pixel = min(border_distances) + border_q = (4 * np.pi * np.sin(np.arctan2(border_pixel / distance)) / self.wavelength) * 1e10 + border_resol = 2 * np.pi / border_q + border_2_q = (4 * np.pi * np.sin(np.arctan2(border_pixel / 2 * distance)) / self.wavelength) * 1e10 + border_2_resol = 2 * np.pi / border_2_q + + circle_closest = plt.Circle((cx, cy), closest_pixel, color='green', linestyle='dashed', fill=False) + ax.add_artist(circle_closest) + ax.text(cx + closest_pixel / np.sqrt(2), cy + closest_pixel / np.sqrt(2), f'{closest_resol:.2f} \u00c5', fontsize=10, ha='center') + + circle_furthest = plt.Circle((cx, cy), furthest_pixel, color='green', linestyle='dashed', fill=False) + ax.add_artist(circle_furthest) + ax.text(cx + furthest_pixel / np.sqrt(2), cy + furthest_pixel / np.sqrt(2), f'{furthest_resol:.2f} \u00c5', fontsize=10, ha='center') + + circle_border = plt.Circle((cx, cy), border_pixel, color='green', linestyle='dashed', fill=False) + ax.add_artist(circle_border) + ax.text(cx + border_pixel / np.sqrt(2), cy + border_pixel / np.sqrt(2), f'{border_resol} \u00c5', fontsize=10, ha='center') + + circle_border_2 = plt.Circle((cx, cy), border_pixel / 2, color='green', linestyle='dashed', fill=False) + ax.add_artist(circle_border_2) + ax.text(cx + border_pixel / 2 * np.sqrt(2), cy + border_pixel / 2 * np.sqrt(2), f'{border_2_resol} \u00c5', fontsize=10, ha='center') + ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") ax.tick_params(axis="x", labelsize=8) ax.tick_params(axis="y", labelsize=8) ax.set_title(label) - return ax + return closest_q, closest_resol, furthest_q, furthest_resol, border_q, border_resol def radial_integration(self, result, calibrant=None, label=None, ax=None): """ @@ -888,7 +932,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax.legend(fontsize="x-small") def visualize_results( - self, powder, bo_history, detector, params, refined_dist, plot="" + self, powder, bo_history, detector, params, distance, beam_center, plot="" ): """ Visualize fit, plotting (1) the BO convergence, (2) the radial profile and (3) the powder image. @@ -903,8 +947,10 @@ def visualize_results( Corrected PyFAI detector object params : list List of parameters for the best fit - refined_dist : float + distance : float Refined distance + beam_center : Tuple(float) + Refined beam center plot : str Path to save plot """ @@ -921,7 +967,7 @@ def visualize_results( self.scan["best_idx"][self.index], color="green", linestyle="--", - label=f"Best score reached at iteration {self.scan["best_idx"][self.index]}", + label=f"Best score reached at iteration {self.scan['best_idx'][self.index]}", ) ax1.set_xlabel("Iteration") ax1.set_ylabel("Number of Control Points") @@ -951,9 +997,9 @@ def visualize_results( self.radial_integration(res, calibrant=self.calibrant, ax=ax3) icol += 1 - # Plotting stacked powder + # Plotting assembled powder with resolutions ax4 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - geometry = Geometry(dist=params[0]) + geometry = Geometry(dist=distance) sg = SingleGeometry( f"Max {self.calibrant_name}", powder, @@ -962,7 +1008,7 @@ def visualize_results( geometry=geometry, ) sg.extract_cp(max_rings=self.max_rings, pts_per_deg=1, Imin=self.Imin) - self.display(sg=sg, ax=ax4) + low_q, low_res, high_q, high_res, border_q, border_res = self.powder_and_resolution(sg=sg, distance=distance, beam_center=beam_center, ax=ax4) irow += 1 icol = 0 @@ -973,11 +1019,11 @@ def visualize_results( # Plotting residual scan over distance ax6 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - self.residual_distance_scan(self.distances, refined_dist, ax6) + self.residual_distance_scan(self.distances, distance, ax6) if plot != "": fig.savefig(plot, dpi=180) - return fig + return fig, low_q, low_res, high_q, high_res, border_q, border_res class OptimizePyFAIGeometry(Task): @@ -1238,33 +1284,29 @@ def _run(self) -> None: f"{fig_folder}/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" ) calib_detector = self._update_geometry(optimizer) - fig = optimizer.visualize_results( + fig, low_q, low_res, high_q, high_res, border_q, border_res = optimizer.visualize_results( powder=optimizer.powder, bo_history=optimizer.bo_history, detector=calib_detector, params=optimizer.params, - refined_dist=distance, + distance=distance, + beam_center=(cx, cy), plot=plot, ) plots = pn.Tabs(fig) - p1, p2, _ = calib_detector.calc_cartesian_positions() - cx_pix = np.abs(cx - np.min(p1)) / calib_detector.pixel1 - cy_pix = np.abs(cy - np.min(p2)) / calib_detector.pixel2 - theta: float = np.arctan(cx / distance) - q: float = ( - 2.0 * np.sin(theta / 2.0) / (optimizer.calibrant.wavelength * 1e10) - ) - edge_resolution: float = 1.0 / q self._result.summary = [] self._result.summary.append( { "Detector distance (m)": f"{distance:.3f}", "Detector center (m)": (f"{cx:.6f}", f"{cy:.6f}"), - "Detector edge resolution (A)": f"{edge_resolution:.3f}", + "Low q": f"{low_q:.2f} \u00c5-1 | {low_res:.2f} \u00c5", + "High q": f"{border_q:.2f} \u00c5-1 | {border_res:.2f} \u00c5 (detector edge)", + "Highest q": f"{high_q:.2f} \u00c5-1 | {high_res:.2f} \u00c5 (detector corner)", } ) - logger.info(f"Beam center (pixels): ({cx_pix}, {cy_pix})") - logger.info(f"Detector edge resolution (A): {edge_resolution}") + logger.info(f">>> Low q : {low_q:.2f} \u00c5-1 | {low_res:.2f} \u00c5") + logger.info(f">>> High q : {border_q:.2f} \u00c5-1 | {border_res:.2f} \u00c5 (detector edge)") + logger.info(f">>> Highest q : {high_q:.2f} \u00c5-1 | {high_res:.2f} \u00c5 (detector corner)") self._result.summary.append( ElogSummaryPlots( f"Geometry_Fit/r{self._task_parameters.run:0>4}", plots From 205c1276513061b84ced2930bc1654cc48079af8 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 12:23:42 +0000 Subject: [PATCH 332/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 110 +++++++++++++++++++++++++++++++++---------- 1 file changed, 85 insertions(+), 25 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index dd4585a1..56e98ba1 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -720,14 +720,18 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): closest_pixel_index = np.argmin(d) closest_pixel = d.flatten()[closest_pixel_index] - closest_q = (4 * np.pi * np.sin(np.arctan2(closest_pixel / distance)) / self.wavelength) * 1e10 + closest_q = ( + 4 * np.pi * np.sin(np.arctan2(closest_pixel / distance)) / self.wavelength + ) * 1e10 closest_resol = 2 * np.pi / closest_q furthest_pixel_index = np.argmax(d) furthest_pixel = d.flatten()[furthest_pixel_index] - furthest_q = (4 * np.pi * np.sin(np.arctan2(furthest_pixel / distance)) / self.wavelength) * 1e10 + furthest_q = ( + 4 * np.pi * np.sin(np.arctan2(furthest_pixel / distance)) / self.wavelength + ) * 1e10 furthest_resol = 2 * np.pi / furthest_q - + xmin, xmax = x.min(), x.max() ymin, ymax = y.min(), y.max() d_left = abs(cx - xmin) @@ -736,33 +740,79 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): d_top = abs(cy - ymax) border_distances = [d_left, d_right, d_bottom, d_top] border_pixel = min(border_distances) - border_q = (4 * np.pi * np.sin(np.arctan2(border_pixel / distance)) / self.wavelength) * 1e10 + border_q = ( + 4 * np.pi * np.sin(np.arctan2(border_pixel / distance)) / self.wavelength + ) * 1e10 border_resol = 2 * np.pi / border_q - border_2_q = (4 * np.pi * np.sin(np.arctan2(border_pixel / 2 * distance)) / self.wavelength) * 1e10 + border_2_q = ( + 4 + * np.pi + * np.sin(np.arctan2(border_pixel / 2 * distance)) + / self.wavelength + ) * 1e10 border_2_resol = 2 * np.pi / border_2_q - circle_closest = plt.Circle((cx, cy), closest_pixel, color='green', linestyle='dashed', fill=False) + circle_closest = plt.Circle( + (cx, cy), closest_pixel, color="green", linestyle="dashed", fill=False + ) ax.add_artist(circle_closest) - ax.text(cx + closest_pixel / np.sqrt(2), cy + closest_pixel / np.sqrt(2), f'{closest_resol:.2f} \u00c5', fontsize=10, ha='center') + ax.text( + cx + closest_pixel / np.sqrt(2), + cy + closest_pixel / np.sqrt(2), + f"{closest_resol:.2f} \u00c5", + fontsize=10, + ha="center", + ) - circle_furthest = plt.Circle((cx, cy), furthest_pixel, color='green', linestyle='dashed', fill=False) + circle_furthest = plt.Circle( + (cx, cy), furthest_pixel, color="green", linestyle="dashed", fill=False + ) ax.add_artist(circle_furthest) - ax.text(cx + furthest_pixel / np.sqrt(2), cy + furthest_pixel / np.sqrt(2), f'{furthest_resol:.2f} \u00c5', fontsize=10, ha='center') + ax.text( + cx + furthest_pixel / np.sqrt(2), + cy + furthest_pixel / np.sqrt(2), + f"{furthest_resol:.2f} \u00c5", + fontsize=10, + ha="center", + ) - circle_border = plt.Circle((cx, cy), border_pixel, color='green', linestyle='dashed', fill=False) + circle_border = plt.Circle( + (cx, cy), border_pixel, color="green", linestyle="dashed", fill=False + ) ax.add_artist(circle_border) - ax.text(cx + border_pixel / np.sqrt(2), cy + border_pixel / np.sqrt(2), f'{border_resol} \u00c5', fontsize=10, ha='center') + ax.text( + cx + border_pixel / np.sqrt(2), + cy + border_pixel / np.sqrt(2), + f"{border_resol} \u00c5", + fontsize=10, + ha="center", + ) - circle_border_2 = plt.Circle((cx, cy), border_pixel / 2, color='green', linestyle='dashed', fill=False) + circle_border_2 = plt.Circle( + (cx, cy), border_pixel / 2, color="green", linestyle="dashed", fill=False + ) ax.add_artist(circle_border_2) - ax.text(cx + border_pixel / 2 * np.sqrt(2), cy + border_pixel / 2 * np.sqrt(2), f'{border_2_resol} \u00c5', fontsize=10, ha='center') + ax.text( + cx + border_pixel / 2 * np.sqrt(2), + cy + border_pixel / 2 * np.sqrt(2), + f"{border_2_resol} \u00c5", + fontsize=10, + ha="center", + ) ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") ax.tick_params(axis="x", labelsize=8) ax.tick_params(axis="y", labelsize=8) ax.set_title(label) - return closest_q, closest_resol, furthest_q, furthest_resol, border_q, border_resol + return ( + closest_q, + closest_resol, + furthest_q, + furthest_resol, + border_q, + border_resol, + ) def radial_integration(self, result, calibrant=None, label=None, ax=None): """ @@ -1008,7 +1058,11 @@ def visualize_results( geometry=geometry, ) sg.extract_cp(max_rings=self.max_rings, pts_per_deg=1, Imin=self.Imin) - low_q, low_res, high_q, high_res, border_q, border_res = self.powder_and_resolution(sg=sg, distance=distance, beam_center=beam_center, ax=ax4) + low_q, low_res, high_q, high_res, border_q, border_res = ( + self.powder_and_resolution( + sg=sg, distance=distance, beam_center=beam_center, ax=ax4 + ) + ) irow += 1 icol = 0 @@ -1284,14 +1338,16 @@ def _run(self) -> None: f"{fig_folder}/bayes_opt_geom_{optimizer.exp}_r{optimizer.run:0>4}.png" ) calib_detector = self._update_geometry(optimizer) - fig, low_q, low_res, high_q, high_res, border_q, border_res = optimizer.visualize_results( - powder=optimizer.powder, - bo_history=optimizer.bo_history, - detector=calib_detector, - params=optimizer.params, - distance=distance, - beam_center=(cx, cy), - plot=plot, + fig, low_q, low_res, high_q, high_res, border_q, border_res = ( + optimizer.visualize_results( + powder=optimizer.powder, + bo_history=optimizer.bo_history, + detector=calib_detector, + params=optimizer.params, + distance=distance, + beam_center=(cx, cy), + plot=plot, + ) ) plots = pn.Tabs(fig) self._result.summary = [] @@ -1305,8 +1361,12 @@ def _run(self) -> None: } ) logger.info(f">>> Low q : {low_q:.2f} \u00c5-1 | {low_res:.2f} \u00c5") - logger.info(f">>> High q : {border_q:.2f} \u00c5-1 | {border_res:.2f} \u00c5 (detector edge)") - logger.info(f">>> Highest q : {high_q:.2f} \u00c5-1 | {high_res:.2f} \u00c5 (detector corner)") + logger.info( + f">>> High q : {border_q:.2f} \u00c5-1 | {border_res:.2f} \u00c5 (detector edge)" + ) + logger.info( + f">>> Highest q : {high_q:.2f} \u00c5-1 | {high_res:.2f} \u00c5 (detector corner)" + ) self._result.summary.append( ElogSummaryPlots( f"Geometry_Fit/r{self._task_parameters.run:0>4}", plots From 6b3d5c9722c93519d8f5d43382d1238fb0338e1f Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 04:35:22 -0800 Subject: [PATCH 333/375] [edit] Computation of detector resolution with plots --- lute/tasks/bayfai.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index dd4585a1..3f782165 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -720,12 +720,12 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): closest_pixel_index = np.argmin(d) closest_pixel = d.flatten()[closest_pixel_index] - closest_q = (4 * np.pi * np.sin(np.arctan2(closest_pixel / distance)) / self.wavelength) * 1e10 + closest_q = (4 * np.pi * np.sin(np.arctan2(closest_pixel, distance)) / self.wavelength) * 1e10 closest_resol = 2 * np.pi / closest_q furthest_pixel_index = np.argmax(d) furthest_pixel = d.flatten()[furthest_pixel_index] - furthest_q = (4 * np.pi * np.sin(np.arctan2(furthest_pixel / distance)) / self.wavelength) * 1e10 + furthest_q = (4 * np.pi * np.sin(np.arctan2(furthest_pixel, distance)) / self.wavelength) * 1e10 furthest_resol = 2 * np.pi / furthest_q xmin, xmax = x.min(), x.max() @@ -736,9 +736,9 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): d_top = abs(cy - ymax) border_distances = [d_left, d_right, d_bottom, d_top] border_pixel = min(border_distances) - border_q = (4 * np.pi * np.sin(np.arctan2(border_pixel / distance)) / self.wavelength) * 1e10 + border_q = (4 * np.pi * np.sin(np.arctan2(border_pixel, distance)) / self.wavelength) * 1e10 border_resol = 2 * np.pi / border_q - border_2_q = (4 * np.pi * np.sin(np.arctan2(border_pixel / 2 * distance)) / self.wavelength) * 1e10 + border_2_q = (4 * np.pi * np.sin(np.arctan2(border_pixel / 2, distance)) / self.wavelength) * 1e10 border_2_resol = 2 * np.pi / border_2_q circle_closest = plt.Circle((cx, cy), closest_pixel, color='green', linestyle='dashed', fill=False) From 7b71c63f3eec5f95b2b175d4706233c74a46cc92 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 04:40:26 -0800 Subject: [PATCH 334/375] [edit] Computation of detector resolution with plots --- lute/tasks/bayfai.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 56e98ba1..18afc88c 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -721,14 +721,14 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): closest_pixel_index = np.argmin(d) closest_pixel = d.flatten()[closest_pixel_index] closest_q = ( - 4 * np.pi * np.sin(np.arctan2(closest_pixel / distance)) / self.wavelength + 4 * np.pi * np.sin(np.arctan2(closest_pixel, distance)) / self.wavelength ) * 1e10 closest_resol = 2 * np.pi / closest_q furthest_pixel_index = np.argmax(d) furthest_pixel = d.flatten()[furthest_pixel_index] furthest_q = ( - 4 * np.pi * np.sin(np.arctan2(furthest_pixel / distance)) / self.wavelength + 4 * np.pi * np.sin(np.arctan2(furthest_pixel, distance)) / self.wavelength ) * 1e10 furthest_resol = 2 * np.pi / furthest_q @@ -741,13 +741,13 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): border_distances = [d_left, d_right, d_bottom, d_top] border_pixel = min(border_distances) border_q = ( - 4 * np.pi * np.sin(np.arctan2(border_pixel / distance)) / self.wavelength + 4 * np.pi * np.sin(np.arctan2(border_pixel, distance)) / self.wavelength ) * 1e10 border_resol = 2 * np.pi / border_q border_2_q = ( 4 * np.pi - * np.sin(np.arctan2(border_pixel / 2 * distance)) + * np.sin(np.arctan2(border_pixel / 2, distance)) / self.wavelength ) * 1e10 border_2_resol = 2 * np.pi / border_2_q From c1d15223092465d4c7e6df73ff22b95b6621159d Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 12:40:53 +0000 Subject: [PATCH 335/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 18afc88c..476ff289 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -745,10 +745,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ) * 1e10 border_resol = 2 * np.pi / border_q border_2_q = ( - 4 - * np.pi - * np.sin(np.arctan2(border_pixel / 2, distance)) - / self.wavelength + 4 * np.pi * np.sin(np.arctan2(border_pixel / 2, distance)) / self.wavelength ) * 1e10 border_2_resol = 2 * np.pi / border_2_q From 239c2259b1c65812e467fb85d7b336c980da0c49 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 05:11:54 -0800 Subject: [PATCH 336/375] [edit] Computation of detector resolution with plots --- lute/tasks/bayfai.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 18afc88c..c32d18d8 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -722,14 +722,14 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): closest_pixel = d.flatten()[closest_pixel_index] closest_q = ( 4 * np.pi * np.sin(np.arctan2(closest_pixel, distance)) / self.wavelength - ) * 1e10 + ) closest_resol = 2 * np.pi / closest_q furthest_pixel_index = np.argmax(d) furthest_pixel = d.flatten()[furthest_pixel_index] furthest_q = ( 4 * np.pi * np.sin(np.arctan2(furthest_pixel, distance)) / self.wavelength - ) * 1e10 + ) furthest_resol = 2 * np.pi / furthest_q xmin, xmax = x.min(), x.max() @@ -742,14 +742,14 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): border_pixel = min(border_distances) border_q = ( 4 * np.pi * np.sin(np.arctan2(border_pixel, distance)) / self.wavelength - ) * 1e10 + ) border_resol = 2 * np.pi / border_q border_2_q = ( 4 * np.pi * np.sin(np.arctan2(border_pixel / 2, distance)) / self.wavelength - ) * 1e10 + ) border_2_resol = 2 * np.pi / border_2_q circle_closest = plt.Circle( @@ -760,7 +760,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): cx + closest_pixel / np.sqrt(2), cy + closest_pixel / np.sqrt(2), f"{closest_resol:.2f} \u00c5", - fontsize=10, + fontsize=4, ha="center", ) @@ -772,7 +772,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): cx + furthest_pixel / np.sqrt(2), cy + furthest_pixel / np.sqrt(2), f"{furthest_resol:.2f} \u00c5", - fontsize=10, + fontsize=4, ha="center", ) @@ -784,7 +784,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): cx + border_pixel / np.sqrt(2), cy + border_pixel / np.sqrt(2), f"{border_resol} \u00c5", - fontsize=10, + fontsize=4, ha="center", ) @@ -796,7 +796,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): cx + border_pixel / 2 * np.sqrt(2), cy + border_pixel / 2 * np.sqrt(2), f"{border_2_resol} \u00c5", - fontsize=10, + fontsize=4, ha="center", ) From 29d8c1e9fb789cb88f9e38ca87f2545393ebac1a Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 13:13:49 +0000 Subject: [PATCH 337/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index c32d18d8..fb92c7d0 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -745,10 +745,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ) border_resol = 2 * np.pi / border_q border_2_q = ( - 4 - * np.pi - * np.sin(np.arctan2(border_pixel / 2, distance)) - / self.wavelength + 4 * np.pi * np.sin(np.arctan2(border_pixel / 2, distance)) / self.wavelength ) border_2_resol = 2 * np.pi / border_2_q From 6a935e216c52260da7e73d490f41a6f044e39486 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 05:29:08 -0800 Subject: [PATCH 338/375] [edit] Fix powder plot with resolution visualization --- lute/tasks/bayfai.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index c32d18d8..3f673f7a 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -675,6 +675,12 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): if z is None: z = np.zeros_like(x) z += distance + + xmin, xmax = x.min(), x.max() + ymin, ymax = y.min(), y.max() + ax.set_xlim(xmin * np.sqrt(2), xmax * np.sqrt(2)) + ax.set_ylim(ymin * np.sqrt(2), ymax * np.sqrt(2)) + img = ax.scatter( x.flatten(), y.flatten(), @@ -721,19 +727,17 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): closest_pixel_index = np.argmin(d) closest_pixel = d.flatten()[closest_pixel_index] closest_q = ( - 4 * np.pi * np.sin(np.arctan2(closest_pixel, distance)) / self.wavelength + 4 * np.pi * np.sin(np.arctan2(closest_pixel, distance)) / (self.wavelength * 1e10) ) closest_resol = 2 * np.pi / closest_q furthest_pixel_index = np.argmax(d) furthest_pixel = d.flatten()[furthest_pixel_index] furthest_q = ( - 4 * np.pi * np.sin(np.arctan2(furthest_pixel, distance)) / self.wavelength + 4 * np.pi * np.sin(np.arctan2(furthest_pixel, distance)) / (self.wavelength * 1e10) ) furthest_resol = 2 * np.pi / furthest_q - xmin, xmax = x.min(), x.max() - ymin, ymax = y.min(), y.max() d_left = abs(cx - xmin) d_right = abs(cx - xmax) d_bottom = abs(cy - ymin) @@ -741,14 +745,14 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): border_distances = [d_left, d_right, d_bottom, d_top] border_pixel = min(border_distances) border_q = ( - 4 * np.pi * np.sin(np.arctan2(border_pixel, distance)) / self.wavelength + 4 * np.pi * np.sin(np.arctan2(border_pixel, distance)) / (self.wavelength * 1e10) ) border_resol = 2 * np.pi / border_q border_2_q = ( 4 * np.pi * np.sin(np.arctan2(border_pixel / 2, distance)) - / self.wavelength + / (self.wavelength * 1e10) ) border_2_resol = 2 * np.pi / border_2_q @@ -760,8 +764,9 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): cx + closest_pixel / np.sqrt(2), cy + closest_pixel / np.sqrt(2), f"{closest_resol:.2f} \u00c5", - fontsize=4, - ha="center", + color="red", + fontsize=6, + ha="right", ) circle_furthest = plt.Circle( @@ -772,8 +777,9 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): cx + furthest_pixel / np.sqrt(2), cy + furthest_pixel / np.sqrt(2), f"{furthest_resol:.2f} \u00c5", - fontsize=4, - ha="center", + color="red", + fontsize=6, + ha="right", ) circle_border = plt.Circle( @@ -784,8 +790,9 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): cx + border_pixel / np.sqrt(2), cy + border_pixel / np.sqrt(2), f"{border_resol} \u00c5", - fontsize=4, - ha="center", + color="red", + fontsize=6, + ha="right", ) circle_border_2 = plt.Circle( @@ -796,8 +803,9 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): cx + border_pixel / 2 * np.sqrt(2), cy + border_pixel / 2 * np.sqrt(2), f"{border_2_resol} \u00c5", - fontsize=4, - ha="center", + color="red", + fontsize=6, + ha="right", ) ax.set_xlabel("X-axis (m)") From 92efc3e8480b4a32775005acaa953181fccfaf89 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 13:30:29 +0000 Subject: [PATCH 339/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 75b4566d..15066f01 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -727,14 +727,20 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): closest_pixel_index = np.argmin(d) closest_pixel = d.flatten()[closest_pixel_index] closest_q = ( - 4 * np.pi * np.sin(np.arctan2(closest_pixel, distance)) / (self.wavelength * 1e10) + 4 + * np.pi + * np.sin(np.arctan2(closest_pixel, distance)) + / (self.wavelength * 1e10) ) closest_resol = 2 * np.pi / closest_q furthest_pixel_index = np.argmax(d) furthest_pixel = d.flatten()[furthest_pixel_index] furthest_q = ( - 4 * np.pi * np.sin(np.arctan2(furthest_pixel, distance)) / (self.wavelength * 1e10) + 4 + * np.pi + * np.sin(np.arctan2(furthest_pixel, distance)) + / (self.wavelength * 1e10) ) furthest_resol = 2 * np.pi / furthest_q @@ -745,11 +751,17 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): border_distances = [d_left, d_right, d_bottom, d_top] border_pixel = min(border_distances) border_q = ( - 4 * np.pi * np.sin(np.arctan2(border_pixel, distance)) / (self.wavelength * 1e10) + 4 + * np.pi + * np.sin(np.arctan2(border_pixel, distance)) + / (self.wavelength * 1e10) ) border_resol = 2 * np.pi / border_q border_2_q = ( - 4 * np.pi * np.sin(np.arctan2(border_pixel / 2, distance)) / (self.wavelength * 1e10) + 4 + * np.pi + * np.sin(np.arctan2(border_pixel / 2, distance)) + / (self.wavelength * 1e10) ) border_2_resol = 2 * np.pi / border_2_q From 1741bedac1a71d3b81bcb29f0838dfddf1f73a64 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 05:45:20 -0800 Subject: [PATCH 340/375] [edit] Fix powder plot with resolution visualization --- lute/tasks/bayfai.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 75b4566d..efad2baa 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -763,7 +763,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): f"{closest_resol:.2f} \u00c5", color="red", fontsize=6, - ha="right", + ha="left", ) circle_furthest = plt.Circle( @@ -776,7 +776,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): f"{furthest_resol:.2f} \u00c5", color="red", fontsize=6, - ha="right", + ha="left", ) circle_border = plt.Circle( @@ -786,10 +786,10 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ax.text( cx + border_pixel / np.sqrt(2), cy + border_pixel / np.sqrt(2), - f"{border_resol} \u00c5", + f"{border_resol:.2f} \u00c5", color="red", fontsize=6, - ha="right", + ha="left", ) circle_border_2 = plt.Circle( @@ -797,12 +797,12 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ) ax.add_artist(circle_border_2) ax.text( - cx + border_pixel / 2 * np.sqrt(2), - cy + border_pixel / 2 * np.sqrt(2), - f"{border_2_resol} \u00c5", + cx + (border_pixel / 2) / np.sqrt(2), + cy + (border_pixel / 2) / np.sqrt(2), + f"{border_2_resol:.2f} \u00c5", color="red", fontsize=6, - ha="right", + ha="left", ) ax.set_xlabel("X-axis (m)") From d602fb69ac3c3bd90827d5693677a32e131b344d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 05:53:12 -0800 Subject: [PATCH 341/375] [edit] Fix powder plot with resolution visualization --- lute/tasks/bayfai.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index d92b15fe..863b68fc 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -678,8 +678,8 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): xmin, xmax = x.min(), x.max() ymin, ymax = y.min(), y.max() - ax.set_xlim(xmin * np.sqrt(2), xmax * np.sqrt(2)) - ax.set_ylim(ymin * np.sqrt(2), ymax * np.sqrt(2)) + ax.set_xlim(xmin * 1.1, xmax * 1.1) + ax.set_ylim(ymin * 1.1, ymax * 1.1) img = ax.scatter( x.flatten(), @@ -706,7 +706,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ttha[i], levels=tth, cmap="autumn", - linewidths=0.5, + linewidths=1, linestyles="dashed", ) else: @@ -717,7 +717,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ttha, levels=tth, cmap="autumn", - linewidths=0.5, + linewidths=1, linestyles="dashed", ) @@ -1034,7 +1034,7 @@ def visualize_results( self.scan["best_idx"][self.index], color="green", linestyle="--", - label=f"Best score reached at iteration {self.scan['best_idx'][self.index]}", + label=f"Best score at n={self.scan['best_idx'][self.index]}", ) ax1.set_xlabel("Iteration") ax1.set_ylabel("Number of Control Points") From e4353935219fb6ded8d1b9c5e62e34e1d36fde61 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 06:12:42 -0800 Subject: [PATCH 342/375] [edit] Fix powder plot with resolution visualization --- lute/tasks/bayfai.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 863b68fc..63899e8b 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -576,7 +576,7 @@ def bayes_opt_animation( poni1 = np.arange(bounds["poni1"][0], bounds["poni1"][1] + res, res) poni2 = np.arange(bounds["poni2"][0], bounds["poni2"][1] + res, res) - X, Y = np.meshgrid(poni2, poni1) + X, Y = np.meshgrid(poni2, poni1, indexing='ij') sample_points = [ bo_history[f"init_sample_{i+1}"]["param"] for i in range(n_samples) @@ -618,8 +618,8 @@ def bayes_opt_animation( ax.set_title( f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" ) - ax.set_xlabel("Y-axis (m)") - ax.set_ylabel("X-axis (m)") + ax.set_xlabel("X-axis (m)") + ax.set_ylabel("Y-axis (m)") ax.legend() ax.tick_params(axis="x", labelsize=8) ax.tick_params(axis="y", labelsize=8) @@ -640,7 +640,7 @@ def update(frame): points_red.set_data([current_point[2]], [current_point[1]]) if previous_points: points_orange.set_data( - [p[2] for p in previous_points], [p[1] for p in previous_points] + [p[1] for p in previous_points], [p[2] for p in previous_points] ) return score_plot, points_red, points_orange From 7066d0ff6d12e503b75625ff5f86b252f2bc28bd Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 14:13:07 +0000 Subject: [PATCH 343/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 63899e8b..c95f2cba 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -576,7 +576,7 @@ def bayes_opt_animation( poni1 = np.arange(bounds["poni1"][0], bounds["poni1"][1] + res, res) poni2 = np.arange(bounds["poni2"][0], bounds["poni2"][1] + res, res) - X, Y = np.meshgrid(poni2, poni1, indexing='ij') + X, Y = np.meshgrid(poni2, poni1, indexing="ij") sample_points = [ bo_history[f"init_sample_{i+1}"]["param"] for i in range(n_samples) From ca36c486fc18d5b519c11141497287146f7c6f00 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 06:21:40 -0800 Subject: [PATCH 344/375] [edit] Fix powder plot with resolution visualization --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 63899e8b..abeb5276 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -640,7 +640,7 @@ def update(frame): points_red.set_data([current_point[2]], [current_point[1]]) if previous_points: points_orange.set_data( - [p[1] for p in previous_points], [p[2] for p in previous_points] + [p[2] for p in previous_points], [p[1] for p in previous_points] ) return score_plot, points_red, points_orange From eff13a3661ee566475f8971e456638819ff970ed Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 06:28:45 -0800 Subject: [PATCH 345/375] [edit] Fix powder plot with resolution visualization --- lute/tasks/bayfai.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 0ecf9097..53e1f721 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -593,7 +593,7 @@ def bayes_opt_animation( [first_point[2]], [first_point[1]], "ro", - label="Next Sampled Point", + label="Next Sampled", ) (points_green,) = ax.plot( [p[2] for p in sample_points], @@ -601,7 +601,7 @@ def bayes_opt_animation( marker="o", color="green", markersize=5, - label="Sampled Points", + label="Sampled", alpha=0.3, linestyle="", ) @@ -612,7 +612,7 @@ def bayes_opt_animation( color="orange", markersize=5, alpha=0.3, - label="Previous Points", + label="Visited", linestyle="", ) ax.set_title( @@ -620,7 +620,7 @@ def bayes_opt_animation( ) ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") - ax.legend() + ax.legend(loc='upper right') ax.tick_params(axis="x", labelsize=8) ax.tick_params(axis="y", labelsize=8) From e5e7a9b76b9c883c11b3c0d38d3fe88574dc571e Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Sun, 22 Dec 2024 14:29:13 +0000 Subject: [PATCH 346/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 53e1f721..3f93aec8 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -620,7 +620,7 @@ def bayes_opt_animation( ) ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") - ax.legend(loc='upper right') + ax.legend(loc="upper right") ax.tick_params(axis="x", labelsize=8) ax.tick_params(axis="y", labelsize=8) From 58d85d426ce6aaef8a9e7ded619847ff4acb52d0 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 06:34:19 -0800 Subject: [PATCH 347/375] [edit] Fix powder plot with resolution visualization --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 53e1f721..4164e0bd 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -616,7 +616,7 @@ def bayes_opt_animation( linestyle="", ) ax.set_title( - f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.2f}m" + f"Bayesian Optimization on {self.exp} \n run {self.run} for distance {dist:.3f}m" ) ax.set_xlabel("X-axis (m)") ax.set_ylabel("Y-axis (m)") From 4e8e5a476ca8d363011987655eaf3b38464b1a9c Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 06:42:15 -0800 Subject: [PATCH 348/375] [edit] Final edit of visualize results --- lute/tasks/bayfai.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 4a0fdb8f..5337f6a4 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -772,7 +772,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ax.text( cx + closest_pixel / np.sqrt(2), cy + closest_pixel / np.sqrt(2), - f"{closest_resol:.2f} \u00c5", + f"{closest_resol:.3f} \u00c5", color="red", fontsize=6, ha="left", @@ -785,7 +785,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ax.text( cx + furthest_pixel / np.sqrt(2), cy + furthest_pixel / np.sqrt(2), - f"{furthest_resol:.2f} \u00c5", + f"{furthest_resol:.3f} \u00c5", color="red", fontsize=6, ha="left", @@ -798,7 +798,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ax.text( cx + border_pixel / np.sqrt(2), cy + border_pixel / np.sqrt(2), - f"{border_resol:.2f} \u00c5", + f"{border_resol:.3f} \u00c5", color="red", fontsize=6, ha="left", @@ -811,7 +811,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ax.text( cx + (border_pixel / 2) / np.sqrt(2), cy + (border_pixel / 2) / np.sqrt(2), - f"{border_2_resol:.2f} \u00c5", + f"{border_2_resol:.3f} \u00c5", color="red", fontsize=6, ha="left", @@ -925,13 +925,13 @@ def residual_distance_scan(self, distances, refined_dist, ax): best_dist, color="green", linestyle="--", - label=f"Best distance (m): {best_dist:.2e}", + label=f"Best distance (m): {best_dist:.3f}", ) ax.axvline( refined_dist, color="red", linestyle="--", - label=f"Refined distance (m): {refined_dist:.2e}", + label=f"Refined distance (m): {refined_dist:.3f}", ) ax.legend(fontsize="x-small") ax.set_yscale("log") @@ -1343,8 +1343,8 @@ def _run(self) -> None: if optimizer.rank == 0: logger.info("Optimization complete") distance, cx, cy = get_beam_center(optimizer.params) - logger.info(f"Detector Distance to Sample: {distance:.2e}") - logger.info(f"Beam center: ({cx:.2e}, {cy:.2e})") + logger.info(f"Detector Distance to Sample: {distance:.6f}") + logger.info(f"Beam center: ({cx:.6f}, {cy:.6f})") logger.info( f"Rotations: \u03B8x = ({optimizer.params[3]:.2e}, \u03B8y = {optimizer.params[4]:.2e}, \u03B8z = {optimizer.params[5]:.2e})" ) @@ -1370,19 +1370,19 @@ def _run(self) -> None: self._result.summary = [] self._result.summary.append( { - "Detector distance (m)": f"{distance:.3f}", + "Detector distance (m)": f"{distance:.6f}", "Detector center (m)": (f"{cx:.6f}", f"{cy:.6f}"), - "Low q": f"{low_q:.2f} \u00c5-1 | {low_res:.2f} \u00c5", - "High q": f"{border_q:.2f} \u00c5-1 | {border_res:.2f} \u00c5 (detector edge)", - "Highest q": f"{high_q:.2f} \u00c5-1 | {high_res:.2f} \u00c5 (detector corner)", + "Low q": f"{low_q:.3f} \u00c5-1 | {low_res:.3f} \u00c5", + "High q": f"{border_q:.3f} \u00c5-1 | {border_res:.3f} \u00c5 (detector edge)", + "Highest q": f"{high_q:.3f} \u00c5-1 | {high_res:.3f} \u00c5 (detector corner)", } ) - logger.info(f">>> Low q : {low_q:.2f} \u00c5-1 | {low_res:.2f} \u00c5") + logger.info(f">>> Low q : {low_q:.3f} \u00c5-1 | {low_res:.3f} \u00c5") logger.info( - f">>> High q : {border_q:.2f} \u00c5-1 | {border_res:.2f} \u00c5 (detector edge)" + f">>> High q : {border_q:.3f} \u00c5-1 | {border_res:.3f} \u00c5 (detector edge)" ) logger.info( - f">>> Highest q : {high_q:.2f} \u00c5-1 | {high_res:.2f} \u00c5 (detector corner)" + f">>> Highest q : {high_q:.3f} \u00c5-1 | {high_res:.3f} \u00c5 (detector corner)" ) self._result.summary.append( ElogSummaryPlots( From 2b30be7d6d8088485413ced63b520b70da93f500 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Sun, 22 Dec 2024 06:50:47 -0800 Subject: [PATCH 349/375] [edit] Final edit of visualize results --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 5337f6a4..61abee30 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -593,7 +593,7 @@ def bayes_opt_animation( [first_point[2]], [first_point[1]], "ro", - label="Next Sampled", + label="Next", ) (points_green,) = ax.plot( [p[2] for p in sample_points], From 109f03c2877081574c777883fdb8965aa2f02789 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 6 Jan 2025 03:42:13 -0800 Subject: [PATCH 350/375] [edit] Fix colorbar limits for better visualization --- lute/tasks/bayfai.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 61abee30..5d08e319 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -536,12 +536,6 @@ def bayes_opt_geom( ) self.comm.Barrier() - self.bayes_opt_animation( - results["bo_history"], n_samples, n_iterations, bounds, res, dist - ) - - self.comm.Barrier() - self.scan = {} self.scan["bo_history"] = self.comm.gather(results["bo_history"], root=0) self.scan["params"] = self.comm.gather(results["params"], root=0) @@ -570,7 +564,7 @@ def bayes_opt_animation( ): import numpy as np import matplotlib.pyplot as plt - from matplotlib.animation import FuncAnimation + from matplotlib.animation import FuncAnimation # type: ignore num_frames = n_iterations @@ -585,7 +579,9 @@ def bayes_opt_animation( pred = bo_history["iteration_1"]["pred"] pred = np.reshape(pred, X.shape) fig, ax = plt.subplots() - score_plot = ax.pcolormesh(X, Y, pred, cmap="viridis", shading="auto") + vmin = min(np.min(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)) + vmax = max(np.max(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)) + score_plot = ax.pcolormesh(X, Y, pred, cmap="viridis", shading="auto", vmin=vmin, vmax=vmax) colorbar = plt.colorbar(score_plot, ax=ax, orientation="vertical") colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] @@ -601,7 +597,7 @@ def bayes_opt_animation( marker="o", color="green", markersize=5, - label="Sampled", + label="Start", alpha=0.3, linestyle="", ) @@ -612,7 +608,7 @@ def bayes_opt_animation( color="orange", markersize=5, alpha=0.3, - label="Visited", + label="Sampled", linestyle="", ) ax.set_title( @@ -629,8 +625,6 @@ def update(frame): pred = np.reshape(bo_history[iteration_key]["pred"], X.shape) score_plot.set_array(pred.ravel()) - score_plot.set_clim(vmin=pred.min(), vmax=pred.max()) - colorbar.update_normal(score_plot) current_point = bo_history[iteration_key]["param"] previous_points = [ @@ -668,7 +662,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): if sg is not None: powder = sg.image cp = sg.control_points - ai = sg.geometry_refinement + #ai = sg.geometry_refinement label = sg.label detector = sg.detector y, x, z = detector.calc_cartesian_positions() From 5923e1cf7f8439649ac9037cd5eaad9d54177be1 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Mon, 6 Jan 2025 11:42:39 +0000 Subject: [PATCH 351/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 5d08e319..71b8721e 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -564,7 +564,7 @@ def bayes_opt_animation( ): import numpy as np import matplotlib.pyplot as plt - from matplotlib.animation import FuncAnimation # type: ignore + from matplotlib.animation import FuncAnimation # type: ignore num_frames = n_iterations @@ -579,9 +579,15 @@ def bayes_opt_animation( pred = bo_history["iteration_1"]["pred"] pred = np.reshape(pred, X.shape) fig, ax = plt.subplots() - vmin = min(np.min(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)) - vmax = max(np.max(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)) - score_plot = ax.pcolormesh(X, Y, pred, cmap="viridis", shading="auto", vmin=vmin, vmax=vmax) + vmin = min( + np.min(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames) + ) + vmax = max( + np.max(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames) + ) + score_plot = ax.pcolormesh( + X, Y, pred, cmap="viridis", shading="auto", vmin=vmin, vmax=vmax + ) colorbar = plt.colorbar(score_plot, ax=ax, orientation="vertical") colorbar.set_label("Normalized Score") first_point = bo_history["iteration_1"]["param"] @@ -662,7 +668,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): if sg is not None: powder = sg.image cp = sg.control_points - #ai = sg.geometry_refinement + # ai = sg.geometry_refinement label = sg.label detector = sg.detector y, x, z = detector.calc_cartesian_positions() From e3eeb2f0a69a11838356da3e640aa3b410f58845 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 6 Jan 2025 04:48:26 -0800 Subject: [PATCH 352/375] [edit] Redo bayesian opt animation with correction --- lute/tasks/bayfai.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 5d08e319..c85c1393 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -536,6 +536,10 @@ def bayes_opt_geom( ) self.comm.Barrier() + self.bayes_opt_animation(results["bo_history"], n_samples, n_iterations, bounds, res, dist) + + self.comm.Barrier() + self.scan = {} self.scan["bo_history"] = self.comm.gather(results["bo_history"], root=0) self.scan["params"] = self.comm.gather(results["params"], root=0) From cc857326ac8d5c7e7ca87cdc65bab19e56773320 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Mon, 6 Jan 2025 12:49:01 +0000 Subject: [PATCH 353/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 4855f1ce..0af751e9 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -536,7 +536,9 @@ def bayes_opt_geom( ) self.comm.Barrier() - self.bayes_opt_animation(results["bo_history"], n_samples, n_iterations, bounds, res, dist) + self.bayes_opt_animation( + results["bo_history"], n_samples, n_iterations, bounds, res, dist + ) self.comm.Barrier() From 60fd06c4e7882224ee07c7f7429a4066e78f295a Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 6 Jan 2025 05:14:24 -0800 Subject: [PATCH 354/375] [edit] Redo bayesian opt animation with correction --- lute/tasks/bayfai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 4855f1ce..7d420c29 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -585,10 +585,10 @@ def bayes_opt_animation( fig, ax = plt.subplots() vmin = min( np.min(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames) - ) + ) / 2 vmax = max( np.max(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames) - ) + ) / 2 score_plot = ax.pcolormesh( X, Y, pred, cmap="viridis", shading="auto", vmin=vmin, vmax=vmax ) From 5358d4e5d1b064322d9a99360e1d2d2a58654e6f Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Mon, 6 Jan 2025 13:15:02 +0000 Subject: [PATCH 355/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 8bccea28..9c2d8053 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -585,12 +585,20 @@ def bayes_opt_animation( pred = bo_history["iteration_1"]["pred"] pred = np.reshape(pred, X.shape) fig, ax = plt.subplots() - vmin = min( - np.min(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames) - ) / 2 - vmax = max( - np.max(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames) - ) / 2 + vmin = ( + min( + np.min(bo_history[f"iteration_{i+1}"]["pred"]) + for i in range(num_frames) + ) + / 2 + ) + vmax = ( + max( + np.max(bo_history[f"iteration_{i+1}"]["pred"]) + for i in range(num_frames) + ) + / 2 + ) score_plot = ax.pcolormesh( X, Y, pred, cmap="viridis", shading="auto", vmin=vmin, vmax=vmax ) From 731b511179c293779d394f9c5973ec2f3b058966 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 6 Jan 2025 05:26:47 -0800 Subject: [PATCH 356/375] [edit] Comment animation --- lute/tasks/bayfai.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 8bccea28..afa2c4bf 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -536,11 +536,9 @@ def bayes_opt_geom( ) self.comm.Barrier() - self.bayes_opt_animation( - results["bo_history"], n_samples, n_iterations, bounds, res, dist - ) - - self.comm.Barrier() + # self.bayes_opt_animation( + # results["bo_history"], n_samples, n_iterations, bounds, res, dist + # ) self.scan = {} self.scan["bo_history"] = self.comm.gather(results["bo_history"], root=0) From c4c64539c8d70c0d055e6cbb1d9d763841e1de23 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 9 Jan 2025 02:24:41 -0800 Subject: [PATCH 357/375] [edit] Change diagnostic output plot design --- lute/tasks/bayfai.py | 93 ++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index f6bc8c4e..952708ce 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -583,20 +583,8 @@ def bayes_opt_animation( pred = bo_history["iteration_1"]["pred"] pred = np.reshape(pred, X.shape) fig, ax = plt.subplots() - vmin = ( - min( - np.min(bo_history[f"iteration_{i+1}"]["pred"]) - for i in range(num_frames) - ) - / 2 - ) - vmax = ( - max( - np.max(bo_history[f"iteration_{i+1}"]["pred"]) - for i in range(num_frames) - ) - / 2 - ) + vmin = np.percentile([np.min(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)], 25) + vmax = np.percentile([np.max(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)], 75) score_plot = ax.pcolormesh( X, Y, pred, cmap="viridis", shading="auto", vmin=vmin, vmax=vmax ) @@ -1009,6 +997,10 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax.set_ylabel("Frequency") ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") ax.legend(fontsize="x-small") + from matplotlib.transforms import Affine2D # type: ignore + rotation = Affine2D().rotate_deg(90) + ax.set_transform(rotation + ax.transAxes) + def visualize_results( self, powder, bo_history, detector, params, distance, beam_center, plot="" @@ -1034,34 +1026,20 @@ def visualize_results( Path to save plot """ fig = plt.figure(figsize=(12, 16), dpi=180) - nrow, ncol = 3, 2 + nrow, ncol = 4, 3 irow, icol = 0, 0 - # Plotting BO convergence + # Labelling experiment and run number ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) - scores = [bo_history[key]["score"] for key in bo_history.keys()] - ax1.plot(scores) - ax1.set_xticks(np.arange(len(scores), step=20)) - ax1.axvline( - self.scan["best_idx"][self.index], - color="green", - linestyle="--", - label=f"Best score at n={self.scan['best_idx'][self.index]}", - ) - ax1.set_xlabel("Iteration") - ax1.set_ylabel("Number of Control Points") - ax1.legend() - ax1.set_title(f"Convergence Plot, best score: {self.scan['score'][self.index]}") + ax1.text(0.5, 0.8, f"Experiment Number: {self.exp}", ha="center", va="center", fontsize=12) + ax1.text(0.5, 0.6, f"Run : {self.run}", ha="center", va="center", fontsize=12) + ax1.text(0.5, 0.4, f"Calibrant: {self.calibrant_name}", ha="center", va="center", fontsize=12) + ax1.text(0.5, 0.2, f"{self.det_type} Distance: {distance:.3f} m", ha="center", va="center", fontsize=12) + ax1.axis("off") icol += 1 - # Plotting histogram of pixel intensities - ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - self.hist_and_compute_stats(powder, self.exp, self.run, ax2) - irow += 1 - icol = 0 - # Plotting radial profiles with peaks - ax3 = plt.subplot2grid((nrow, ncol), (irow, icol)) + ax2 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) ai = AzimuthalIntegrator( dist=params[0], detector=detector, wavelength=self.calibrant.wavelength ) @@ -1073,14 +1051,15 @@ def visualize_results( mask = ((row - center[0]) ** 2 + (col - center[1]) ** 2) <= radius**2 masked_powder = powder * mask res = ai.integrate1d(masked_powder, 1000) - self.radial_integration(res, calibrant=self.calibrant, ax=ax3) - icol += 1 + self.radial_integration(res, calibrant=self.calibrant, ax=ax2) + irow += 1 + icol = 0 # Plotting assembled powder with resolutions - ax4 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) + ax3 = plt.subplot2grid((nrow, ncol), (irow, icol), rowspan=2, colspan=2) geometry = Geometry(dist=distance) sg = SingleGeometry( - f"Max {self.calibrant_name}", + f"Run {self.run} {self.calibrant_name}", powder, calibrant=self.calibrant, detector=detector, @@ -1089,20 +1068,42 @@ def visualize_results( sg.extract_cp(max_rings=self.max_rings, pts_per_deg=1, Imin=self.Imin) low_q, low_res, high_q, high_res, border_q, border_res = ( self.powder_and_resolution( - sg=sg, distance=distance, beam_center=beam_center, ax=ax4 + sg=sg, distance=distance, beam_center=beam_center, ax=ax3 ) ) - irow += 1 + icol = +2 + + # Plotting histogram of pixel intensities + ax4 = plt.subplot2grid((nrow, ncol), (irow, icol), rowspan=2) + self.hist_and_compute_stats(powder, self.exp, self.run, ax4) + irow += 2 icol = 0 - # Plotting score scan over distance + # Plotting BO convergence ax5 = plt.subplot2grid((nrow, ncol), (irow, icol)) - self.score_distance_scan(self.distances, ax5) + scores = [bo_history[key]["score"] for key in bo_history.keys()] + ax5.plot(scores) + ax5.set_xticks(np.arange(len(scores), step=20)) + ax5.axvline( + self.scan["best_idx"][self.index], + color="green", + linestyle="--", + label=f"Best score at n={self.scan['best_idx'][self.index]}", + ) + ax5.set_xlabel("Iteration") + ax5.set_ylabel("Number of Control Points") + ax5.legend() + ax5.set_title(f"Convergence Plot, best score: {self.scan['score'][self.index]}") + icol += 1 + + # Plotting score scan over distance + ax6 = plt.subplot2grid((nrow, ncol), (irow, icol)) + self.score_distance_scan(self.distances, ax6) icol += 1 # Plotting residual scan over distance - ax6 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) - self.residual_distance_scan(self.distances, distance, ax6) + ax7 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) + self.residual_distance_scan(self.distances, distance, ax7) if plot != "": fig.savefig(plot, dpi=180) From ccd353a5767552334ab32b3086166cdff71f02b8 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 9 Jan 2025 10:25:10 +0000 Subject: [PATCH 358/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 952708ce..ad61626c 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -583,8 +583,14 @@ def bayes_opt_animation( pred = bo_history["iteration_1"]["pred"] pred = np.reshape(pred, X.shape) fig, ax = plt.subplots() - vmin = np.percentile([np.min(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)], 25) - vmax = np.percentile([np.max(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)], 75) + vmin = np.percentile( + [np.min(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)], + 25, + ) + vmax = np.percentile( + [np.max(bo_history[f"iteration_{i+1}"]["pred"]) for i in range(num_frames)], + 75, + ) score_plot = ax.pcolormesh( X, Y, pred, cmap="viridis", shading="auto", vmin=vmin, vmax=vmax ) @@ -997,11 +1003,11 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax.set_ylabel("Frequency") ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") ax.legend(fontsize="x-small") - from matplotlib.transforms import Affine2D # type: ignore + from matplotlib.transforms import Affine2D # type: ignore + rotation = Affine2D().rotate_deg(90) ax.set_transform(rotation + ax.transAxes) - def visualize_results( self, powder, bo_history, detector, params, distance, beam_center, plot="" ): @@ -1031,10 +1037,31 @@ def visualize_results( # Labelling experiment and run number ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) - ax1.text(0.5, 0.8, f"Experiment Number: {self.exp}", ha="center", va="center", fontsize=12) + ax1.text( + 0.5, + 0.8, + f"Experiment Number: {self.exp}", + ha="center", + va="center", + fontsize=12, + ) ax1.text(0.5, 0.6, f"Run : {self.run}", ha="center", va="center", fontsize=12) - ax1.text(0.5, 0.4, f"Calibrant: {self.calibrant_name}", ha="center", va="center", fontsize=12) - ax1.text(0.5, 0.2, f"{self.det_type} Distance: {distance:.3f} m", ha="center", va="center", fontsize=12) + ax1.text( + 0.5, + 0.4, + f"Calibrant: {self.calibrant_name}", + ha="center", + va="center", + fontsize=12, + ) + ax1.text( + 0.5, + 0.2, + f"{self.det_type} Distance: {distance:.3f} m", + ha="center", + va="center", + fontsize=12, + ) ax1.axis("off") icol += 1 From d1d8a5582141959159c0cda7225e137d1f3ca5d5 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 9 Jan 2025 02:25:33 -0800 Subject: [PATCH 359/375] [edit] Change diagnostic output plot design + animation --- lute/tasks/bayfai.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 952708ce..94b6da4c 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -536,9 +536,9 @@ def bayes_opt_geom( ) self.comm.Barrier() - # self.bayes_opt_animation( - # results["bo_history"], n_samples, n_iterations, bounds, res, dist - # ) + self.bayes_opt_animation( + results["bo_history"], n_samples, n_iterations, bounds, res, dist + ) self.scan = {} self.scan["bo_history"] = self.comm.gather(results["bo_history"], root=0) From f1faee2ed17b398fe99c12a114dde9f89f2ef378 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 9 Jan 2025 02:51:10 -0800 Subject: [PATCH 360/375] [edit] Change diagnostic output plot design + animation --- lute/tasks/bayfai.py | 50 +++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index bd1f97c2..dc8ac6f3 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -33,6 +33,8 @@ import numpy as np import numpy.typing as npt import matplotlib.pyplot as plt # type: ignore +from matplotlib.transforms import Affine2D # type: ignore +import matplotlib.patches as patches # type: ignore import pyFAI # type: ignore from pyFAI.geometry import Geometry # type: ignore from pyFAI.goniometer import SingleGeometry # type: ignore @@ -836,6 +838,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): border_q, border_resol, ) + ax.set_aspect("equal") def radial_integration(self, result, calibrant=None, label=None, ax=None): """ @@ -965,13 +968,16 @@ def hist_and_compute_stats(self, powder, exp, run, ax): mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) nice_pix = powder < threshold - _ = ax.hist( + base = plt.gca().transData + rot = Affine2D().rotate_deg(90) + hist = ax.hist( powder[nice_pix], bins=1000, color="skyblue", edgecolor="black", alpha=0.7, label="Pixel Intensities", + transform=rot + base, ) ax.axvline( mean, @@ -984,12 +990,14 @@ def hist_and_compute_stats(self, powder, exp, run, ax): color="orange", linestyle="--", label=f"Mean + Std Dev ({mean + std_dev:.2f})", + transform=rot + base, ) ax.axvline( mean + 2 * std_dev, color="green", linestyle="--", label=f"Mean + 2 Std Dev ({mean + 2 * std_dev:.2f})", + transform=rot + base, ) ax.axvline( self.Imin, @@ -997,16 +1005,13 @@ def hist_and_compute_stats(self, powder, exp, run, ax): linestyle=":", linewidth=1.5, label=f"{self.q} th Percentile ({self.Imin:.2f})", + transform=rot + base, ) ax.set_xlim([0, mean + 5 * std_dev]) ax.set_xlabel("Pixel Intensity") ax.set_ylabel("Frequency") ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") ax.legend(fontsize="x-small") - from matplotlib.transforms import Affine2D # type: ignore - - rotation = Affine2D().rotate_deg(90) - ax.set_transform(rotation + ax.transAxes) def visualize_results( self, powder, bo_history, detector, params, distance, beam_center, plot="" @@ -1034,30 +1039,39 @@ def visualize_results( fig = plt.figure(figsize=(12, 16), dpi=180) nrow, ncol = 4, 3 irow, icol = 0, 0 + plt.subplots_adjust(hspace=0.5, wspace=0.4) # Labelling experiment and run number ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) + rect = patches.Rectangle( + (0, 0), + 1, 1, + transform=ax1.transAxes, + color="lightgrey", + alpha=0.3, + ) + ax1.add_patch(rect) ax1.text( - 0.5, - 0.8, - f"Experiment Number: {self.exp}", - ha="center", + 0.05, + 0.9, + f"Experiment {self.exp}", + ha="left", va="center", fontsize=12, ) - ax1.text(0.5, 0.6, f"Run : {self.run}", ha="center", va="center", fontsize=12) + ax1.text(0.05, 0.8, f"Run {self.run}", ha="left", va="center", fontsize=12) ax1.text( - 0.5, - 0.4, - f"Calibrant: {self.calibrant_name}", - ha="center", + 0.05, + 0.7, + f"Calibrant {self.calibrant_name}", + ha="left", va="center", fontsize=12, ) ax1.text( - 0.5, - 0.2, - f"{self.det_type} Distance: {distance:.3f} m", + 0.05, + 0.6, + f"{self.det_type} distance = {distance:.4f} m", ha="center", va="center", fontsize=12, @@ -1132,6 +1146,8 @@ def visualize_results( ax7 = plt.subplot2grid((nrow, ncol), (irow, icol), colspan=ncol - icol) self.residual_distance_scan(self.distances, distance, ax7) + fig.tight_layout() + if plot != "": fig.savefig(plot, dpi=180) return fig, low_q, low_res, high_q, high_res, border_q, border_res From b23255102cb8d69859e7bbdedac284506b27b71b Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 9 Jan 2025 10:51:39 +0000 Subject: [PATCH 361/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index dc8ac6f3..f694f8c8 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -34,7 +34,7 @@ import numpy.typing as npt import matplotlib.pyplot as plt # type: ignore from matplotlib.transforms import Affine2D # type: ignore -import matplotlib.patches as patches # type: ignore +import matplotlib.patches as patches # type: ignore import pyFAI # type: ignore from pyFAI.geometry import Geometry # type: ignore from pyFAI.goniometer import SingleGeometry # type: ignore @@ -1044,11 +1044,12 @@ def visualize_results( # Labelling experiment and run number ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) rect = patches.Rectangle( - (0, 0), - 1, 1, - transform=ax1.transAxes, - color="lightgrey", - alpha=0.3, + (0, 0), + 1, + 1, + transform=ax1.transAxes, + color="lightgrey", + alpha=0.3, ) ax1.add_patch(rect) ax1.text( From add2230b90b041f4102d729570b8c8f9a42f2eaa Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 9 Jan 2025 02:57:31 -0800 Subject: [PATCH 362/375] [edit] Change diagnostic output plot design + animation --- lute/tasks/bayfai.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index dc8ac6f3..f4758324 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -830,6 +830,7 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ax.tick_params(axis="x", labelsize=8) ax.tick_params(axis="y", labelsize=8) ax.set_title(label) + ax.set_aspect("equal") return ( closest_q, closest_resol, @@ -838,7 +839,6 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): border_q, border_resol, ) - ax.set_aspect("equal") def radial_integration(self, result, calibrant=None, label=None, ax=None): """ @@ -968,16 +968,14 @@ def hist_and_compute_stats(self, powder, exp, run, ax): mean = np.mean(powder[nice_pix]) std_dev = np.std(powder[nice_pix]) nice_pix = powder < threshold - base = plt.gca().transData - rot = Affine2D().rotate_deg(90) - hist = ax.hist( + _ = ax.hist( powder[nice_pix], bins=1000, color="skyblue", edgecolor="black", alpha=0.7, label="Pixel Intensities", - transform=rot + base, + orientation="vertical", ) ax.axvline( mean, @@ -990,14 +988,12 @@ def hist_and_compute_stats(self, powder, exp, run, ax): color="orange", linestyle="--", label=f"Mean + Std Dev ({mean + std_dev:.2f})", - transform=rot + base, ) ax.axvline( mean + 2 * std_dev, color="green", linestyle="--", label=f"Mean + 2 Std Dev ({mean + 2 * std_dev:.2f})", - transform=rot + base, ) ax.axvline( self.Imin, @@ -1005,7 +1001,6 @@ def hist_and_compute_stats(self, powder, exp, run, ax): linestyle=":", linewidth=1.5, label=f"{self.q} th Percentile ({self.Imin:.2f})", - transform=rot + base, ) ax.set_xlim([0, mean + 5 * std_dev]) ax.set_xlabel("Pixel Intensity") From ccdfba40b368e2b20c4c09bff597172aaac3f168 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 9 Jan 2025 03:07:45 -0800 Subject: [PATCH 363/375] [edit] Change diagnostic output plot design + animation --- lute/tasks/bayfai.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 11a120db..0cab06b4 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -975,7 +975,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): edgecolor="black", alpha=0.7, label="Pixel Intensities", - orientation="vertical", + orientation="horizontal", ) ax.axvline( mean, @@ -1034,7 +1034,7 @@ def visualize_results( fig = plt.figure(figsize=(12, 16), dpi=180) nrow, ncol = 4, 3 irow, icol = 0, 0 - plt.subplots_adjust(hspace=0.5, wspace=0.4) + plt.subplots_adjust(hspace=0.3, wspace=0.3) # Labelling experiment and run number ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) @@ -1056,9 +1056,10 @@ def visualize_results( fontsize=12, ) ax1.text(0.05, 0.8, f"Run {self.run}", ha="left", va="center", fontsize=12) + ax1.text(0.05, 0.7, f"Detector {self.det_type}", ha="left", va="center", fontsize=12) ax1.text( 0.05, - 0.7, + 0.6, f"Calibrant {self.calibrant_name}", ha="left", va="center", @@ -1066,9 +1067,9 @@ def visualize_results( ) ax1.text( 0.05, - 0.6, - f"{self.det_type} distance = {distance:.4f} m", - ha="center", + 0.5, + f"Distance = {distance:.4f} m", + ha="left", va="center", fontsize=12, ) From 3a9566e8705a71898a419f3a5268918d27d2be26 Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Thu, 9 Jan 2025 11:08:15 +0000 Subject: [PATCH 364/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 0cab06b4..bc1d2ee8 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -1056,7 +1056,9 @@ def visualize_results( fontsize=12, ) ax1.text(0.05, 0.8, f"Run {self.run}", ha="left", va="center", fontsize=12) - ax1.text(0.05, 0.7, f"Detector {self.det_type}", ha="left", va="center", fontsize=12) + ax1.text( + 0.05, 0.7, f"Detector {self.det_type}", ha="left", va="center", fontsize=12 + ) ax1.text( 0.05, 0.6, From 22e09e744ef59bf14b8b88fea9a5d7edae5b7aa0 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 9 Jan 2025 03:14:12 -0800 Subject: [PATCH 365/375] [edit] Change diagnostic output plot design + animation --- lute/tasks/bayfai.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 0cab06b4..db2a3278 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -977,25 +977,25 @@ def hist_and_compute_stats(self, powder, exp, run, ax): label="Pixel Intensities", orientation="horizontal", ) - ax.axvline( + ax.axhline( mean, color="red", linestyle="--", label=f"Mean ({mean:.2f})", ) - ax.axvline( + ax.axhline( mean + std_dev, color="orange", linestyle="--", label=f"Mean + Std Dev ({mean + std_dev:.2f})", ) - ax.axvline( + ax.axhline( mean + 2 * std_dev, color="green", linestyle="--", label=f"Mean + 2 Std Dev ({mean + 2 * std_dev:.2f})", ) - ax.axvline( + ax.axhline( self.Imin, color="purple", linestyle=":", @@ -1034,7 +1034,6 @@ def visualize_results( fig = plt.figure(figsize=(12, 16), dpi=180) nrow, ncol = 4, 3 irow, icol = 0, 0 - plt.subplots_adjust(hspace=0.3, wspace=0.3) # Labelling experiment and run number ax1 = plt.subplot2grid((nrow, ncol), (irow, icol)) From feac75c4b8aaee5e5037965d40b21d0eace87dd7 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Thu, 9 Jan 2025 04:34:00 -0800 Subject: [PATCH 366/375] [edit] Change diagnostic output plot design + animation --- lute/tasks/bayfai.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 29365e8f..dd120704 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -1002,9 +1002,9 @@ def hist_and_compute_stats(self, powder, exp, run, ax): linewidth=1.5, label=f"{self.q} th Percentile ({self.Imin:.2f})", ) - ax.set_xlim([0, mean + 5 * std_dev]) - ax.set_xlabel("Pixel Intensity") - ax.set_ylabel("Frequency") + ax.set_ylim([0, mean + 5 * std_dev]) + ax.set_ylabel("Pixel Intensity") + ax.set_xlabel("Frequency") ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") ax.legend(fontsize="x-small") From b7099a390176f0d47d5575044450ca0a164d9175 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 10 Jan 2025 01:19:48 -0800 Subject: [PATCH 367/375] [edit] Change diagnostic output plot design + animation --- lute/tasks/bayfai.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index dd120704..36c6709a 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -33,7 +33,6 @@ import numpy as np import numpy.typing as npt import matplotlib.pyplot as plt # type: ignore -from matplotlib.transforms import Affine2D # type: ignore import matplotlib.patches as patches # type: ignore import pyFAI # type: ignore from pyFAI.geometry import Geometry # type: ignore @@ -1005,6 +1004,8 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax.set_ylim([0, mean + 5 * std_dev]) ax.set_ylabel("Pixel Intensity") ax.set_xlabel("Frequency") + ax.set_xticks([]) + ax.set_xticklabels([]) ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") ax.legend(fontsize="x-small") From 58edbe471204a6c89eb56511e4c92a4925faeaef Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 10 Jan 2025 09:20:18 +0000 Subject: [PATCH 368/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 36c6709a..aa5fc629 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -1004,7 +1004,7 @@ def hist_and_compute_stats(self, powder, exp, run, ax): ax.set_ylim([0, mean + 5 * std_dev]) ax.set_ylabel("Pixel Intensity") ax.set_xlabel("Frequency") - ax.set_xticks([]) + ax.set_xticks([]) ax.set_xticklabels([]) ax.set_title(f"Histogram of Pixel Intensities \n for {exp} run {run}") ax.legend(fontsize="x-small") From 2916197e641ceeb36f1684324a819029cfd174a6 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 10 Jan 2025 01:36:27 -0800 Subject: [PATCH 369/375] [edit] Comment out animation for testing --- lute/tasks/bayfai.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 36c6709a..29862337 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -537,10 +537,6 @@ def bayes_opt_geom( ) self.comm.Barrier() - self.bayes_opt_animation( - results["bo_history"], n_samples, n_iterations, bounds, res, dist - ) - self.scan = {} self.scan["bo_history"] = self.comm.gather(results["bo_history"], root=0) self.scan["params"] = self.comm.gather(results["params"], root=0) From cca229daea91b70742b29ba23a7f2699c963bacf Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 10 Jan 2025 08:37:56 -0800 Subject: [PATCH 370/375] [edit] Change q as usual --- lute/tasks/bayfai.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index bdf6583b..1feca53c 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -206,9 +206,9 @@ def min_intensity(self, powder): noise_pixels = powder[nice_pix][powder[nice_pix] <= threshold] noise = np.std(noise_pixels) SNRs.append(signal / noise) - # q = Imins[np.argmax(SNRs)] - Imin = np.percentile(powder[nice_pix], 95) - self.q = 95 + q = Imins[np.argmax(SNRs)] + Imin = np.percentile(powder[nice_pix], q) + self.q = q self.Imin = Imin self.powder = powder return Imin From e5dd2a6286c516d43adc14614c576041d1f549c4 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Mon, 13 Jan 2025 02:47:14 -0800 Subject: [PATCH 371/375] [edit] Fix writing out of thresholding --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 1feca53c..b5a75258 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -208,7 +208,7 @@ def min_intensity(self, powder): SNRs.append(signal / noise) q = Imins[np.argmax(SNRs)] Imin = np.percentile(powder[nice_pix], q) - self.q = q + self.q = round(q, 1) self.Imin = Imin self.powder = powder return Imin From a30a78bda81d3b632f4b1009d8b4233c0e7dc69b Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 17 Jan 2025 10:36:06 -0800 Subject: [PATCH 372/375] [edit] Change powder plot to match real lab coordinate system --- lute/tasks/bayfai.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index b5a75258..b8a4e42e 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -674,19 +674,19 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): # ai = sg.geometry_refinement label = sg.label detector = sg.detector - y, x, z = detector.calc_cartesian_positions() + x, y, z = detector.calc_cartesian_positions() if z is None: z = np.zeros_like(x) z += distance xmin, xmax = x.min(), x.max() ymin, ymax = y.min(), y.max() - ax.set_xlim(xmin * 1.1, xmax * 1.1) - ax.set_ylim(ymin * 1.1, ymax * 1.1) + ax.set_xlim(ymin * 1.1, ymax * 1.1) + ax.set_ylim(xmin * 1.1, xmax * 1.1) img = ax.scatter( - x.flatten(), y.flatten(), + -x.flatten(), c=powder.flatten(), s=1, edgecolors=None, @@ -704,8 +704,8 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ttha = np.arctan2(np.sqrt(x * x + y * y), z) for i in range(detector.n_modules): ax.contour( - x[i], y[i], + -x[i], ttha[i], levels=tth, cmap="autumn", @@ -715,8 +715,8 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): else: ttha = np.arctan2(np.sqrt(x * x + y * y), z) ax.contour( - x, y, + -x, ttha, levels=tth, cmap="autumn", @@ -769,12 +769,12 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): border_2_resol = 2 * np.pi / border_2_q circle_closest = plt.Circle( - (cx, cy), closest_pixel, color="green", linestyle="dashed", fill=False + (cy, -cx), closest_pixel, color="green", linestyle="dashed", fill=False ) ax.add_artist(circle_closest) ax.text( - cx + closest_pixel / np.sqrt(2), cy + closest_pixel / np.sqrt(2), + -cx + closest_pixel / np.sqrt(2), f"{closest_resol:.3f} \u00c5", color="red", fontsize=6, @@ -782,12 +782,12 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ) circle_furthest = plt.Circle( - (cx, cy), furthest_pixel, color="green", linestyle="dashed", fill=False + (cy, -cx), furthest_pixel, color="green", linestyle="dashed", fill=False ) ax.add_artist(circle_furthest) ax.text( - cx + furthest_pixel / np.sqrt(2), cy + furthest_pixel / np.sqrt(2), + -cx + furthest_pixel / np.sqrt(2), f"{furthest_resol:.3f} \u00c5", color="red", fontsize=6, @@ -795,12 +795,12 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ) circle_border = plt.Circle( - (cx, cy), border_pixel, color="green", linestyle="dashed", fill=False + (cy, -cx), border_pixel, color="green", linestyle="dashed", fill=False ) ax.add_artist(circle_border) ax.text( - cx + border_pixel / np.sqrt(2), cy + border_pixel / np.sqrt(2), + -cx + border_pixel / np.sqrt(2), f"{border_resol:.3f} \u00c5", color="red", fontsize=6, @@ -808,12 +808,12 @@ def powder_and_resolution(self, sg, distance, beam_center, ax=None): ) circle_border_2 = plt.Circle( - (cx, cy), border_pixel / 2, color="green", linestyle="dashed", fill=False + (cy, -cx), border_pixel / 2, color="green", linestyle="dashed", fill=False ) ax.add_artist(circle_border_2) ax.text( - cx + (border_pixel / 2) / np.sqrt(2), cy + (border_pixel / 2) / np.sqrt(2), + -cx + (border_pixel / 2) / np.sqrt(2), f"{border_2_resol:.3f} \u00c5", color="red", fontsize=6, From 6ff8c98f38c5658b1a62316df66ccaf53b29b93d Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 17 Jan 2025 13:48:52 -0800 Subject: [PATCH 373/375] [edit] Add animation in code --- lute/tasks/bayfai.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index b8a4e42e..2ad24ca9 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -535,6 +535,11 @@ def bayes_opt_geom( prior, seed, ) + + self.bayes_opt_animation( + results["bo_history"], n_samples, n_iterations, bounds, res, dist + ) + self.comm.Barrier() self.scan = {} From 8ff6e9cdef55ca923e234a56e6ea6b516da16d3c Mon Sep 17 00:00:00 2001 From: Autoformatter Date: Fri, 17 Jan 2025 21:49:19 +0000 Subject: [PATCH 374/375] Auto-commit black formatting --- lute/tasks/bayfai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 2ad24ca9..c2116b11 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -539,7 +539,7 @@ def bayes_opt_geom( self.bayes_opt_animation( results["bo_history"], n_samples, n_iterations, bounds, res, dist ) - + self.comm.Barrier() self.scan = {} From 4eb1ae34d98fd4e344f93f6a41b937d6453e62b2 Mon Sep 17 00:00:00 2001 From: LouConreux Date: Fri, 17 Jan 2025 14:02:44 -0800 Subject: [PATCH 375/375] [edit] Remove animation --- lute/tasks/bayfai.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lute/tasks/bayfai.py b/lute/tasks/bayfai.py index 2ad24ca9..f241c9dc 100644 --- a/lute/tasks/bayfai.py +++ b/lute/tasks/bayfai.py @@ -536,10 +536,6 @@ def bayes_opt_geom( seed, ) - self.bayes_opt_animation( - results["bo_history"], n_samples, n_iterations, bounds, res, dist - ) - self.comm.Barrier() self.scan = {}