Dispersion

LY99/spread1/3724210.sh

@@ -19,6 +19,8 @@ export NUMEXPR_MAX_THREADS=128
 export SLURM_CPU_BIND=cores # critical to force ranks onto different cores. verify with ps -o psr <pid>
 export OMP_PROC_BIND=spread
 export OMP_PLACES=threads
+export SIT_DATA=/global/common/software/lcls/psdm/data
+export SIT_ROOT=/reg/g/psdm
 export SIT_PSDM_DATA=/global/cfs/cdirs/lcls/psdm-sauter
 export CCTBX_GPUS_PER_NODE=1
 export XFEL_CUSTOM_WORKER_PATH=$MODULES/psii_spread/merging/application # User must export $MODULES path

LY99/spread1/3762323.sh

@@ -20,6 +20,8 @@ export SLURM_CPU_BIND=cores # critical to force ranks onto different cores. veri
 export OMP_PROC_BIND=spread
 export OMP_PLACES=threads
 export SIT_PSDM_DATA=/global/cfs/cdirs/lcls/psdm-sauter
+export SIT_DATA=/global/common/software/lcls/psdm/data
+export SIT_ROOT=/reg/g/psdm
 export CCTBX_GPUS_PER_NODE=1
 export XFEL_CUSTOM_WORKER_PATH=$MODULES/psii_spread/merging/application # User must export $MODULES path
 export WERK=/global/cfs/cdirs/lcls/sauter/LY99/

LY99/spread1/4709418.sh

@@ -5,7 +5,7 @@
 #SBATCH -A lcls_g           # allocation
 #SBATCH -C gpu
 #SBATCH -q regular          # regular queue
-#SBATCH -t 01:20:00         # wall clock time limit
+#SBATCH -t 02:00:00         # wall clock time limit
 #SBATCH --ntasks-per-gpu=1
 #SBATCH -o %j.out
 #SBATCH -e %j.err
@@ -19,6 +19,8 @@ export NUMEXPR_MAX_THREADS=128
 export SLURM_CPU_BIND=cores # critical to force ranks onto different cores. verify with ps -o psr <pid>
 export OMP_PROC_BIND=spread
 export OMP_PLACES=threads
+export SIT_DATA=/global/common/software/lcls/psdm/data
+export SIT_ROOT=/reg/g/psdm
 export SIT_PSDM_DATA=/global/cfs/cdirs/lcls/psdm-sauter
 export CCTBX_GPUS_PER_NODE=1
 export XFEL_CUSTOM_WORKER_PATH=$MODULES/psii_spread/merging/application # User must export $MODULES path

LY99/spread1/README.md

@@ -212,7 +212,7 @@ factors.  Do this as a function of lattice count and energy granularity to see w
    - Then sample both of these curves with Gaussian noise to simulate experimental measurement of the two curves.
    - Then develop a restraint model, and optimize the parameters.  Presumably use automatic differentiation for first-derivatives.
    - Compare the optimized model to the initial ground truth (and pass the test based on a tolerance). Show result in matplotlib.
-
+ - This work is now complete.  Here is a working [Slurm script, 5928113.sh](./5928113.sh) to apply Kramers-Kronig to Step 11.
 #### Generalization of the code for photosystem II
  - As currently written the program will run out of memory due to the size of the structure factor table.  Total structure factor
  count scales linearly with a) unit cell volume, b) volume of the reciprocal space annulus requested, and c) the number of energy 
@@ -230,5 +230,8 @@ factors.  Do this as a function of lattice count and energy granularity to see w
  information about the metals, and which is specialized for every use case.
    - The current code (labels=601, labels=602) needs to be generalized (labels=601,602).
    - The class would have to set it own preset_starting_model.
-   - All this is now done. Specific behavior is localized in sw1.py, and cases are selected with phil parameter exafel.metal= choice.
+   - All this is now done. Specific behavior is localized in sw1.py, and cases are selected with phil parameter exafel.metal= choice. Here is a [Slurm script](./5928113.sh) showing the use of exafel.metal=MMO2.
+     - MMO2: methane monooxygenase, treating the two Fe sites as chemically distinct (realistic)
+     - MMO2: methane monooxygenase, treating the Fe sites as chemically identical (for debug)
+     - PSII: not implemented
  - Currently the scattering factor refinement stops after 1 macrocycle.  Need to stabilize behavior and then extend to 3(?) macrocycles as in Sauter (2020) paper.

LY99/spread1/step11_kramers_kronig_restraint.sh

@@ -0,0 +1,121 @@
+#!/bin/bash -l
+#SBATCH -N 48               # Number of nodes on Perlmutter
+#SBATCH -J test_spread
+#SBATCH -L SCRATCH          # job requires SCRATCH files
+#SBATCH -A lcls_g           # allocation
+#SBATCH -C gpu
+#SBATCH -q regular          # regular queue
+#SBATCH -t 02:00:00         # wall clock time limit
+#SBATCH --ntasks-per-gpu=1
+#SBATCH -o %j.out
+#SBATCH -e %j.err
+
+mkdir -p $SLURM_JOB_ID; cd $SLURM_JOB_ID
+
+export CCTBX_NO_UUID=1
+export DIFFBRAGG_USE_CUDA=1
+export CUDA_LAUNCH_BLOCKING=1
+export NUMEXPR_MAX_THREADS=128
+export SLURM_CPU_BIND=cores # critical to force ranks onto different cores. verify with ps -o psr <pid>
+export OMP_PROC_BIND=spread
+export OMP_PLACES=threads
+export SIT_DATA=/global/common/software/lcls/psdm/data
+export SIT_ROOT=/reg/g/psdm
+export SIT_PSDM_DATA=/global/cfs/cdirs/lcls/psdm-sauter
+export CCTBX_GPUS_PER_NODE=1
+export XFEL_CUSTOM_WORKER_PATH=$MODULES/psii_spread/merging/application # User must export $MODULES path
+export WERK=/global/cfs/cdirs/lcls/sauter/LY99/
+
+echo "
+dispatch.step_list = input balance annulus
+input.path=$WERK/ready2/3724210/out
+input.experiments_suffix=.expt  # switch back for production
+input.reflections_suffix=.refl  # switch back for production
+input.parallel_file_load.balance=global2
+input.keep_imagesets=True
+input.read_image_headers=False
+input.persistent_refl_cols=shoebox
+input.persistent_refl_cols=bbox
+input.persistent_refl_cols=xyzcal.px
+input.persistent_refl_cols=xyzcal.mm
+input.persistent_refl_cols=xyzobs.px.value
+input.persistent_refl_cols=xyzobs.mm.value
+input.persistent_refl_cols=xyzobs.mm.variance
+input.persistent_refl_cols=delpsical.rad
+input.persistent_refl_cols=panel
+input.parallel_file_load.method=uniform
+
+scaling.model=$WERK/reference/6ydi.pdb
+scaling.unit_cell=107.00  107.00  304.01  90.00  90.00  90.00
+scaling.space_group=P41212
+scaling.resolution_scalar=0.993420862158964
+scaling.pdb.k_sol=0.435
+
+filter.unit_cell.cluster.covariance.file=$WERK/reference/covariance_run145_cells.pickle
+filter.unit_cell.cluster.covariance.component=0
+
+merging.d_max=None
+merging.d_min=2.5
+
+statistics.annulus.d_max=4.0
+statistics.annulus.d_min=2.5
+
+spread_roi.enable=True
+# spread_roi.strong=1.0 # only use for initial annulus definition, not subsequent
+
+output.log_level=0 # 0 = stdout stderr, 1 = terminal
+output.output_dir=out
+output.prefix=trial8_scenario3A
+output.save_experiments_and_reflections=True
+
+exafel.scenario=S1
+exafel.static_fcalcs.path=$WERK/reference/mmo_static_fcalcs.pickle
+exafel.static_fcalcs.whole_path=$WERK/reference/mmo_miller_array.pickle
+exafel.static_fcalcs.action=read
+exafel.trusted_mask=$WERK/reference/epix.mask
+exafel.shoebox_border=0
+exafel.context=kokkos_gpu
+exafel.model.plot=False
+exafel.model.mosaic_spread.value=0.0512
+exafel.model.Nabc.value=48,48,24
+exafel.debug.lastfiles=False # write out *.h5, *.mask for each image
+exafel.debug.verbose=False
+exafel.debug.finite_diff=-1
+exafel.debug.eps=1.e-8
+exafel.debug.format_offset=0
+exafel.debug.energy_offset_eV=0
+exafel.debug.energy_stride_eV=4.00
+exafel.skin=False # whether to use diffBragg
+exafel{
+  refpar{
+    label = *background *G
+    background {
+      algorithm=rossmann_2d_linear
+      scope=spot
+      slice_init=border
+      slice=all
+    }
+    G {
+      scope=lattice
+      reparameterize=bound
+    }
+  }
+}
+exafel.metal=MMO2
+sauter20.LLG_evaluator.enable_plot=True
+sauter20.LLG_evaluator.title=tell
+sauter20.LLG_evaluator.restraints.fp.mean=0.0
+sauter20.LLG_evaluator.restraints.fp.sigma=0.04
+sauter20.LLG_evaluator.restraints.fdp.mean=0.03
+sauter20.LLG_evaluator.restraints.fdp.sigma=0.08
+sauter20.LLG_evaluator.restraints.kramers_kronig.use=True
+sauter20.LLG_evaluator.restraints.kramers_kronig.pad=100
+sauter20.LLG_evaluator.restraints.kramers_kronig.trim=0
+sauter20.LLG_evaluator.restraints.kramers_kronig.weighting_factor=1000000.0
+sauter20.LLG_evaluator.max_calls=30
+trumpet.plot_all.enable=False
+trumpet.plot_all.savepng=True
+" > trial8.phil
+echo "jobstart $(date)";pwd
+srun -n 192 -c 4 cctbx.xfel.merge trial8.phil
+echo "jobend $(date)";pwd

ML_push/phil.py

@@ -78,6 +78,16 @@
       sigma = 0.2
         .type = float
     }
+    kramers_kronig{
+      use = True
+        .type = bool
+      pad = 10
+        .type = int
+      trim = 10
+        .type = int
+      weighting_factor = 1.0
+        .type = float
+    }
   }
   restraints_II_enable = False
     .type = bool