Dynamic block device volume size and entry point changes

ecs_model_deployer/src/lib/ecsCluster.ts

@@ -102,7 +102,7 @@ export class ECSCluster extends Construct {
             blockDevices: [
                 {
                     deviceName: '/dev/xvda',
-                    volume: BlockDeviceVolume.ebs(30, {
+                    volume: BlockDeviceVolume.ebs(ecsConfig.autoScalingConfig.blockDeviceVolumeSize, {
                         encrypted: true,
                     }),
                 },

ecs_model_deployer/src/lib/schema.ts

@@ -417,6 +417,7 @@ const MetricConfigSchema = z.object({
 * @property {MetricConfig} metricConfig - Metric configuration for auto scaling.
 */
 const AutoScalingConfigSchema = z.object({
+    blockDeviceVolumeSize: z.number().min(1).default(30),
     minCapacity: z.number().min(1).default(1),
     maxCapacity: z.number().min(1).default(2),
     defaultInstanceWarmup: z.number().default(180),

lambda/models/domain_objects.py

@@ -91,6 +91,7 @@ class LoadBalancerConfig(BaseModel):
 class AutoScalingConfig(BaseModel):
     """Autoscaling configuration upon model creation."""
 
+    blockDeviceVolumeSize: Optional[NonNegativeInt] = 30
     minCapacity: NonNegativeInt
     maxCapacity: NonNegativeInt
     cooldown: PositiveInt

lib/api-base/ecsCluster.ts

@@ -102,7 +102,7 @@ export class ECSCluster extends Construct {
             blockDevices: [
                 {
                     deviceName: '/dev/xvda',
-                    volume: BlockDeviceVolume.ebs(30, {
+                    volume: BlockDeviceVolume.ebs(ecsConfig.autoScalingConfig.blockDeviceVolumeSize, {
                         encrypted: true,
                     }),
                 },

lib/schema.ts

@@ -440,6 +440,7 @@ const MetricConfigSchema = z.object({
 * @property {MetricConfig} metricConfig - Metric configuration for auto scaling.
 */
 const AutoScalingConfigSchema = z.object({
+    blockDeviceVolumeSize: z.number().min(1).default(30),
     minCapacity: z.number().min(1).default(1),
     maxCapacity: z.number().min(1).default(2),
     defaultInstanceWarmup: z.number().default(180),

lib/serve/ecs-model/textgen/tgi/src/entrypoint.sh

@@ -30,8 +30,22 @@ export MAX_TOTAL_TOKENS="${MAX_TOTAL_TOKENS}"
 if [[ -n "${QUANTIZE}" ]]; then
   export QUANTIZE="${QUANTIZE}"
 fi
+# Check if CUDA_VISIBLE_DEVICES is set, otherwise set it to use GPU 0
+if [[ -z "${CUDA_VISIBLE_DEVICES}" ]]; then
+  export CUDA_VISIBLE_DEVICES="0"
+fi
+# Check if number of shards is set, otherwise set it to use 1
+if [[ -z "${NUM_SHARD}" ]]; then
+  export NUM_SHARD="${NUM_SHARD:-1}"
+fi
 echo "$(env)"
 
 # Start the webserver
 echo "Starting TGI"
-text-generation-launcher --model-id $LOCAL_MODEL_PATH --port 8080 --json-output
+CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} \
+text-generation-launcher \
+  --model-id $LOCAL_MODEL_PATH \
+  --quantize ${QUANTIZE} \
+  --port 8080 \
+  --num-shard ${NUM_SHARD} \
+  --json-output

lib/user-interface/react/src/components/model-management/create-model/AutoScalingConfig.tsx

@@ -35,6 +35,14 @@ export function AutoScalingConfig (props: AutoScalingConfigProps) : ReactElement
                     <Header variant='h3'>Auto Scaling Capacity</Header>
                 }
             >
+                <FormField label='Block Device Volume Size' errorText={props.formErrors?.autoScalingConfig?.blockDeviceVolumeSize}>
+                    <Grid gridDefinition={[{colspan: 10}, {colspan: 2}]} disableGutters={true}>
+                        <Input value={props.item.blockDeviceVolumeSize.toString()} type='number' inputMode='numeric' onBlur={() => props.touchFields(['autoScalingConfig.blockDeviceVolumeSize'])} disabled={props.isEdit} onChange={({ detail }) => {
+                            props.setFields({ 'autoScalingConfig.blockDeviceVolumeSize': Number(detail.value) });
+                        }}/>
+                        <span style={{lineHeight: '2.5em', paddingLeft: '0.5em'}}>GBs</span>
+                    </Grid>
+                </FormField>
                 <FormField label='Min Capacity' errorText={props.formErrors?.autoScalingConfig?.minCapacity}>
                     <Grid gridDefinition={[{colspan: 10}, {colspan: 2}]} disableGutters={true}>
                         <Input value={props.item.minCapacity.toString()} type='number' inputMode='numeric' onBlur={() => props.touchFields(['autoScalingConfig.minCapacity'])} onChange={({ detail }) => {

lib/user-interface/react/src/shared/model/model-management.model.ts

@@ -72,6 +72,7 @@ export type ILoadBalancerConfig = {
 };
 
 export type IAutoScalingConfig = {
+    blockDeviceVolumeSize: number;
     minCapacity: number;
     maxCapacity: number;
     desiredCapacity?: number;
@@ -161,6 +162,7 @@ export const loadBalancerConfigSchema = z.object({
 });
 
 export const autoScalingConfigSchema = z.object({
+    blockDeviceVolumeSize: z.number().min(1).default(30),
     minCapacity: z.number().min(1).default(1),
     maxCapacity: z.number().min(1).default(1),
     desiredCapacity: z.number().optional(),