From cbac53ed121bae3798a0019911546a3d0b9bbba4 Mon Sep 17 00:00:00 2001 From: Krzysztof Findeisen Date: Tue, 5 Nov 2024 09:13:56 -0800 Subject: [PATCH] Configure per-filter cache size for Prompt Processing. This config serves as a floor to the cache size that guarantees at least one dataset per filter can be cached at once. --- applications/prompt-proto-service-hsc-gpu/README.md | 1 + applications/prompt-proto-service-hsc-gpu/values.yaml | 3 +++ applications/prompt-proto-service-hsc/README.md | 1 + applications/prompt-proto-service-hsc/values.yaml | 3 +++ applications/prompt-proto-service-latiss/README.md | 1 + applications/prompt-proto-service-latiss/values.yaml | 3 +++ applications/prompt-proto-service-lsstcam/README.md | 1 + applications/prompt-proto-service-lsstcam/values.yaml | 3 +++ applications/prompt-proto-service-lsstcomcam/README.md | 1 + applications/prompt-proto-service-lsstcomcam/values.yaml | 3 +++ applications/prompt-proto-service-lsstcomcamsim/README.md | 1 + applications/prompt-proto-service-lsstcomcamsim/values.yaml | 3 +++ charts/prompt-proto-service/README.md | 1 + .../prompt-proto-service/templates/prompt-proto-service.yaml | 2 ++ charts/prompt-proto-service/values.yaml | 3 +++ 15 files changed, 30 insertions(+) diff --git a/applications/prompt-proto-service-hsc-gpu/README.md b/applications/prompt-proto-service-hsc-gpu/README.md index 2415159676..c15afbac89 100644 --- a/applications/prompt-proto-service-hsc-gpu/README.md +++ b/applications/prompt-proto-service-hsc-gpu/README.md @@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream | | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. | | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. | +| prompt-proto-service.cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. | | prompt-proto-service.cache.patchesPerImage | int | `4` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. | | prompt-proto-service.cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. | | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container | diff --git a/applications/prompt-proto-service-hsc-gpu/values.yaml b/applications/prompt-proto-service-hsc-gpu/values.yaml index 7efc93a3bb..b769efe5e4 100644 --- a/applications/prompt-proto-service-hsc-gpu/values.yaml +++ b/applications/prompt-proto-service-hsc-gpu/values.yaml @@ -60,6 +60,9 @@ prompt-proto-service: refcatsPerImage: 4 # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets. patchesPerImage: 4 + # -- The maximum number of datasets of a given type the service might load if the filter is unknown. + # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. + maxFilters: 20 s3: # -- Bucket containing the incoming raw images diff --git a/applications/prompt-proto-service-hsc/README.md b/applications/prompt-proto-service-hsc/README.md index fbb60fceae..e42e98570b 100644 --- a/applications/prompt-proto-service-hsc/README.md +++ b/applications/prompt-proto-service-hsc/README.md @@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream | | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. | | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. | +| prompt-proto-service.cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. | | prompt-proto-service.cache.patchesPerImage | int | `4` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. | | prompt-proto-service.cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. | | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container | diff --git a/applications/prompt-proto-service-hsc/values.yaml b/applications/prompt-proto-service-hsc/values.yaml index c3921fcb42..4f2bb6153d 100644 --- a/applications/prompt-proto-service-hsc/values.yaml +++ b/applications/prompt-proto-service-hsc/values.yaml @@ -60,6 +60,9 @@ prompt-proto-service: refcatsPerImage: 4 # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets. patchesPerImage: 4 + # -- The maximum number of datasets of a given type the service might load if the filter is unknown. + # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. + maxFilters: 20 s3: # -- Bucket containing the incoming raw images diff --git a/applications/prompt-proto-service-latiss/README.md b/applications/prompt-proto-service-latiss/README.md index 941c350a20..ce765f2da7 100644 --- a/applications/prompt-proto-service-latiss/README.md +++ b/applications/prompt-proto-service-latiss/README.md @@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream | | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. | | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. | +| prompt-proto-service.cache.maxFilters | int | `10` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. | | prompt-proto-service.cache.patchesPerImage | int | `6` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. | | prompt-proto-service.cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. | | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container | diff --git a/applications/prompt-proto-service-latiss/values.yaml b/applications/prompt-proto-service-latiss/values.yaml index 38fddacd35..fed814b736 100644 --- a/applications/prompt-proto-service-latiss/values.yaml +++ b/applications/prompt-proto-service-latiss/values.yaml @@ -61,6 +61,9 @@ prompt-proto-service: refcatsPerImage: 4 # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets. patchesPerImage: 6 + # -- The maximum number of datasets of a given type the service might load if the filter is unknown. + # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. + maxFilters: 10 s3: # -- Bucket containing the incoming raw images diff --git a/applications/prompt-proto-service-lsstcam/README.md b/applications/prompt-proto-service-lsstcam/README.md index b2d000f026..b49f608383 100644 --- a/applications/prompt-proto-service-lsstcam/README.md +++ b/applications/prompt-proto-service-lsstcam/README.md @@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream | | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. | | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. | +| prompt-proto-service.cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. | | prompt-proto-service.cache.patchesPerImage | int | `4` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. | | prompt-proto-service.cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. | | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container | diff --git a/applications/prompt-proto-service-lsstcam/values.yaml b/applications/prompt-proto-service-lsstcam/values.yaml index a590661413..c72da7e373 100644 --- a/applications/prompt-proto-service-lsstcam/values.yaml +++ b/applications/prompt-proto-service-lsstcam/values.yaml @@ -60,6 +60,9 @@ prompt-proto-service: refcatsPerImage: 4 # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets. patchesPerImage: 4 + # -- The maximum number of datasets of a given type the service might load if the filter is unknown. + # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. + maxFilters: 20 s3: # -- Bucket containing the incoming raw images diff --git a/applications/prompt-proto-service-lsstcomcam/README.md b/applications/prompt-proto-service-lsstcomcam/README.md index 59907947a7..b7991bf994 100644 --- a/applications/prompt-proto-service-lsstcomcam/README.md +++ b/applications/prompt-proto-service-lsstcomcam/README.md @@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream | | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. | | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. | +| prompt-proto-service.cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. | | prompt-proto-service.cache.patchesPerImage | int | `16` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. | | prompt-proto-service.cache.refcatsPerImage | int | `6` | A factor by which to multiply `baseSize` for refcat datasets. | | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container | diff --git a/applications/prompt-proto-service-lsstcomcam/values.yaml b/applications/prompt-proto-service-lsstcomcam/values.yaml index fd880bdaca..52e133a11a 100644 --- a/applications/prompt-proto-service-lsstcomcam/values.yaml +++ b/applications/prompt-proto-service-lsstcomcam/values.yaml @@ -63,6 +63,9 @@ prompt-proto-service: refcatsPerImage: 6 # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets. patchesPerImage: 16 + # -- The maximum number of datasets of a given type the service might load if the filter is unknown. + # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. + maxFilters: 20 s3: # -- Bucket containing the incoming raw images diff --git a/applications/prompt-proto-service-lsstcomcamsim/README.md b/applications/prompt-proto-service-lsstcomcamsim/README.md index 6854bea8e2..7fd4eb76a1 100644 --- a/applications/prompt-proto-service-lsstcomcamsim/README.md +++ b/applications/prompt-proto-service-lsstcomcamsim/README.md @@ -19,6 +19,7 @@ Prompt Proto Service is an event driven service for processing camera images. Th | prompt-proto-service.alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream | | prompt-proto-service.apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. | | prompt-proto-service.cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. | +| prompt-proto-service.cache.maxFilters | int | `3` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. | | prompt-proto-service.cache.patchesPerImage | int | `16` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. | | prompt-proto-service.cache.refcatsPerImage | int | `6` | A factor by which to multiply `baseSize` for refcat datasets. | | prompt-proto-service.containerConcurrency | int | `1` | The number of Knative requests that can be handled simultaneously by one container | diff --git a/applications/prompt-proto-service-lsstcomcamsim/values.yaml b/applications/prompt-proto-service-lsstcomcamsim/values.yaml index 4b5797b52d..ac825014c7 100644 --- a/applications/prompt-proto-service-lsstcomcamsim/values.yaml +++ b/applications/prompt-proto-service-lsstcomcamsim/values.yaml @@ -63,6 +63,9 @@ prompt-proto-service: refcatsPerImage: 6 # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets. patchesPerImage: 16 + # -- The maximum number of datasets of a given type the service might load if the filter is unknown. + # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. + maxFilters: 3 s3: # -- Bucket containing the incoming raw images diff --git a/charts/prompt-proto-service/README.md b/charts/prompt-proto-service/README.md index 03390726d6..94f154cdcb 100644 --- a/charts/prompt-proto-service/README.md +++ b/charts/prompt-proto-service/README.md @@ -19,6 +19,7 @@ Event-driven processing of camera images | alerts.username | string | `"kafka-admin"` | Username for sending alerts to the alert stream | | apdb.config | string | None, must be set | URL to a serialized APDB configuration, or the "label:" prefix followed by the indexed name of such a config. | | cache.baseSize | int | `3` | The default number of datasets of each type to keep. The pipeline only needs one of most dataset types (one bias, one flat, etc.), so this is roughly the number of visits that fit in the cache. | +| cache.maxFilters | int | `20` | The maximum number of datasets of a given type the service might load if the filter is unknown. Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. | | cache.patchesPerImage | int | `4` | A factor by which to multiply `baseSize` for templates and other patch-based datasets. | | cache.refcatsPerImage | int | `4` | A factor by which to multiply `baseSize` for refcat datasets. | | cacheCalibs | bool | `true` | Whether or not calibs should be cached between runs of a pod. This is a temporary flag that should only be unset in specific circumstances, and only in the development environment. | diff --git a/charts/prompt-proto-service/templates/prompt-proto-service.yaml b/charts/prompt-proto-service/templates/prompt-proto-service.yaml index f08eb4b17e..b25784a038 100644 --- a/charts/prompt-proto-service/templates/prompt-proto-service.yaml +++ b/charts/prompt-proto-service/templates/prompt-proto-service.yaml @@ -115,6 +115,8 @@ spec: value: {{ .Values.cache.refcatsPerImage | toString | quote }} - name: PATCHES_PER_IMAGE value: {{ .Values.cache.patchesPerImage | toString | quote }} + - name: FILTERS_WITH_CALIBS + value: {{ .Values.cache.maxFilters | toString | quote }} - name: DEBUG_CACHE_CALIBS value: {{ if .Values.cacheCalibs }}'1'{{ else }}'0'{{ end }} volumeMounts: diff --git a/charts/prompt-proto-service/values.yaml b/charts/prompt-proto-service/values.yaml index 954c50e7d1..7169250899 100644 --- a/charts/prompt-proto-service/values.yaml +++ b/charts/prompt-proto-service/values.yaml @@ -62,6 +62,9 @@ cache: refcatsPerImage: 4 # -- A factor by which to multiply `baseSize` for templates and other patch-based datasets. patchesPerImage: 4 + # -- The maximum number of datasets of a given type the service might load if the filter is unknown. + # Should be greater than or equal to the number of filters that have e.g. flats or transmission curves. + maxFilters: 20 s3: # -- Bucket containing the incoming raw images