Merge pull request #3851 from lsst-sqre/tickets/DM-47387

DM-47387: Prompt Processing can't handle blank filters in nextVisit message

applications/prompt-proto-service-hsc-gpu/README.md

@@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream |
 | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. |
 | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. |
+| prompt-proto-service.cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. |
 | prompt-proto-service.cache.patchesPerImage | int | `4` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. |
 | prompt-proto-service.cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. |
 | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container |

applications/prompt-proto-service-hsc-gpu/values.yaml

@@ -60,6 +60,9 @@ prompt-proto-service:
     refcatsPerImage: 4
     # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets.
     patchesPerImage: 4
+    # -- The maximum number of datasets of a given type the service might load if the filter is unknown.
+    # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves.
+    maxFilters: 20
 
   s3:
     # -- Bucket containing the incoming raw images

applications/prompt-proto-service-hsc/README.md

@@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream |
 | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. |
 | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. |
+| prompt-proto-service.cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. |
 | prompt-proto-service.cache.patchesPerImage | int | `4` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. |
 | prompt-proto-service.cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. |
 | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container |

applications/prompt-proto-service-hsc/values.yaml

@@ -60,6 +60,9 @@ prompt-proto-service:
     refcatsPerImage: 4
     # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets.
     patchesPerImage: 4
+    # -- The maximum number of datasets of a given type the service might load if the filter is unknown.
+    # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves.
+    maxFilters: 20
 
   s3:
     # -- Bucket containing the incoming raw images

applications/prompt-proto-service-latiss/README.md

@@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream |
 | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. |
 | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. |
+| prompt-proto-service.cache.maxFilters | int | `10` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. |
 | prompt-proto-service.cache.patchesPerImage | int | `6` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. |
 | prompt-proto-service.cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. |
 | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container |

applications/prompt-proto-service-latiss/values.yaml

@@ -61,6 +61,9 @@ prompt-proto-service:
     refcatsPerImage: 4
     # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets.
     patchesPerImage: 6
+    # -- The maximum number of datasets of a given type the service might load if the filter is unknown.
+    # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves.
+    maxFilters: 10
 
   s3:
     # -- Bucket containing the incoming raw images

applications/prompt-proto-service-lsstcam/README.md

@@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream |
 | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. |
 | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. |
+| prompt-proto-service.cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. |
 | prompt-proto-service.cache.patchesPerImage | int | `4` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. |
 | prompt-proto-service.cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. |
 | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container |

applications/prompt-proto-service-lsstcam/values.yaml

@@ -60,6 +60,9 @@ prompt-proto-service:
     refcatsPerImage: 4
     # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets.
     patchesPerImage: 4
+    # -- The maximum number of datasets of a given type the service might load if the filter is unknown.
+    # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves.
+    maxFilters: 20
 
   s3:
     # -- Bucket containing the incoming raw images

applications/prompt-proto-service-lsstcomcam/README.md

@@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream |
 | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. |
 | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. |
+| prompt-proto-service.cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. |
 | prompt-proto-service.cache.patchesPerImage | int | `16` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. |
 | prompt-proto-service.cache.refcatsPerImage | int | `6` | A factor by which to multiply `baseSize` for refcat datasets. |
 | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container |

applications/prompt-proto-service-lsstcomcam/values.yaml

@@ -63,6 +63,9 @@ prompt-proto-service:
     refcatsPerImage: 6
     # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets.
     patchesPerImage: 16
+    # -- The maximum number of datasets of a given type the service might load if the filter is unknown.
+    # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves.
+    maxFilters: 20
 
   s3:
     # -- Bucket containing the incoming raw images

applications/prompt-proto-service-lsstcomcamsim/README.md

@@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th
 | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream |
 | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. |
 | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. |
+| prompt-proto-service.cache.maxFilters | int | `3` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. |
 | prompt-proto-service.cache.patchesPerImage | int | `16` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. |
 | prompt-proto-service.cache.refcatsPerImage | int | `6` | A factor by which to multiply `baseSize` for refcat datasets. |
 | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container |

applications/prompt-proto-service-lsstcomcamsim/values.yaml

@@ -63,6 +63,9 @@ prompt-proto-service:
     refcatsPerImage: 6
     # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets.
     patchesPerImage: 16
+    # -- The maximum number of datasets of a given type the service might load if the filter is unknown.
+    # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves.
+    maxFilters: 3
 
   s3:
     # -- Bucket containing the incoming raw images

charts/prompt-proto-service/README.md

@@ -19,6 +19,7 @@ Event-driven processing of camera images
 | alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream |
 | apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. |
 | cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. |
+| cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. |
 | cache.patchesPerImage | int | `4` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. |
 | cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. |
 | cacheCalibs | bool | `true` | Whether or not calibs should be cached between runs of a pod. This is a temporary flag that should only be unset in specific circumstances, and only in the development environment. |

charts/prompt-proto-service/templates/prompt-proto-service.yaml

@@ -115,6 +115,8 @@ spec:
           value: {{ .Values.cache.refcatsPerImage | toString | quote }}
         - name: PATCHES_PER_IMAGE
           value: {{ .Values.cache.patchesPerImage | toString | quote }}
+        - name: FILTERS_WITH_CALIBS
+          value: {{ .Values.cache.maxFilters | toString | quote }}
         - name: DEBUG_CACHE_CALIBS
           value: {{ if .Values.cacheCalibs }}'1'{{ else }}'0'{{ end }}
         volumeMounts:

charts/prompt-proto-service/values.yaml

@@ -62,6 +62,9 @@ cache:
   refcatsPerImage: 4
   # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets.
   patchesPerImage: 4
+  # -- The maximum number of datasets of a given type the service might load if the filter is unknown.
+  # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves.
+  maxFilters: 20
 
 s3:
   # -- Bucket containing the incoming raw images