AWS Lambda deployment - service is deployed but fails to launch

[adamwawrzynski]

Describe the bug
After model deployment service is not responsive due to error.

To Reproduce

1. Follow steps in aws-lambda-deplyoment repository: https://github.com/bentoml/aws-lambda-deploy.
2. Test in AWS Lambda console.

Expected behavior

Screenshots/Logs

AWS Lambda logs after testing:

{
  "errorMessage": "operation failed, [Errno 30] Read-only file system: b'/home/bentoml/bento/models/iris_clf/latest'",
  "errorType": "OperationFailed",
  "requestId": "",
  "stackTrace": [
    "  File \"/usr/local/lib/python3.10/importlib/__init__.py\", line 126, in import_module\n    return _bootstrap._gcd_import(name[level:], package, level)\n",
    "  File \"<frozen importlib._bootstrap>\", line 1050, in _gcd_import\n",
    "  File \"<frozen importlib._bootstrap>\", line 1027, in _find_and_load\n",
    "  File \"<frozen importlib._bootstrap>\", line 1006, in _find_and_load_unlocked\n",
    "  File \"<frozen importlib._bootstrap>\", line 688, in _load_unlocked\n",
    "  File \"<frozen importlib._bootstrap_external>\", line 883, in exec_module\n",
    "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
    "  File \"/home/bentoml/bento/app.py\", line 8, in <module>\n    bento_service = load(\"./\")\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/service/loader.py\", line 336, in load\n    svc = load_bento_dir(bento_path, standalone_load=standalone_load)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/service/loader.py\", line 236, in load_bento_dir\n    return _load_bento(bento, standalone_load)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/service/loader.py\", line 254, in _load_bento\n    svc = import_service(\n",
    "  File \"/usr/local/lib/python3.10/site-packages/simple_di/__init__.py\", line 139, in _\n    return func(*_inject_args(bind.args), **_inject_kwargs(bind.kwargs))\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/service/loader.py\", line 137, in import_service\n    module = importlib.import_module(module_name, package=working_dir)\n",
    "  File \"/usr/local/lib/python3.10/importlib/__init__.py\", line 126, in import_module\n    return _bootstrap._gcd_import(name[level:], package, level)\n",
    "  File \"<frozen importlib._bootstrap>\", line 1050, in _gcd_import\n",
    "  File \"<frozen importlib._bootstrap>\", line 1027, in _find_and_load\n",
    "  File \"<frozen importlib._bootstrap>\", line 1006, in _find_and_load_unlocked\n",
    "  File \"<frozen importlib._bootstrap>\", line 688, in _load_unlocked\n",
    "  File \"<frozen importlib._bootstrap_external>\", line 883, in exec_module\n",
    "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
    "  File \"/home/bentoml/bento/src/service.py\", line 5, in <module>\n    iris_clf_runner = bentoml.sklearn.get(\"iris_clf:latest\").to_runner()\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/frameworks/sklearn.py\", line 50, in get\n    model = bentoml.models.get(tag_like)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/simple_di/__init__.py\", line 139, in _\n    return func(*_inject_args(bind.args), **_inject_kwargs(bind.kwargs))\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/models.py\", line 45, in get\n    return _model_store.get(tag)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/store.py\", line 149, in get\n    self._recreate_latest(_tag)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/bentoml/_internal/store.py\", line 132, in _recreate_latest\n    with self._fs.open(tag.latest_path(), \"w\") as latest_file:\n",
    "  File \"/usr/local/lib/python3.10/site-packages/fs/wrapfs.py\", line 472, in open\n    open_file = _fs.open(\n",
    "  File \"/usr/local/lib/python3.10/site-packages/fs/osfs.py\", line 643, in open\n    with convert_os_errors(\"open\", path):\n",
    "  File \"/usr/local/lib/python3.10/site-packages/fs/error_tools.py\", line 89, in __exit__\n    reraise(fserror, fserror(self._path, exc=exc_value), traceback)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/six.py\", line 718, in reraise\n    raise value.with_traceback(tb)\n",
    "  File \"/usr/local/lib/python3.10/site-packages/fs/osfs.py\", line 647, in open\n    return io.open(\n"
  ]
}

Environment:

• OS: [e.g. MacOS 10.14.3]: Ubuntu 22.04.01
• Python Version [e.g. Python 3.7.1]: 3.10.12
• BentoML Version [e.g. BentoML-0.8.6]: 1.1.11
• Bentoctl Version [e.g. bentoctl-0.1]: 0.4.0

Additional context

[amustafi1986]

I am also facing the same issue while trying to test model deploymnet in lamda. I also used the sample model in the bentoml project to check, and the same error is coming up:

{"errorMessage": "operation failed, [Errno 30] Read-only file system: b'/home/bentoml/bento/models/iris_clf/latest'", "errorType": "OperationFailed", "requestId": "", "stackTrace": [" File "/usr/local/lib/python3.11/importlib/init.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)",
" File "", line 1204, in _gcd_import",
" File "", line 1176, in _find_and_load",
" File "", line 1147, in _find_and_load_unlocked",
" File "", line 690, in _load_unlocked",
" File "", line 940, in exec_module",
" File "", line 241, in _call_with_frames_removed",
" File "/home/bentoml/bento/app.py", line 8, in
bento_service = load("./")",
" File "/usr/local/lib/python3.11/site-packages/bentoml/_internal/service/loader.py", line 336, in load
svc = load_bento_dir(bento_path, standalone_load=standalone_load)",
" File "/usr/local/lib/python3.11/site-packages/bentoml/_internal/service/loader.py", line 236, in load_bento_dir
return _load_bento(bento, standalone_load)",
" File "/usr/local/lib/python3.11/site-packages/bentoml/_internal/service/loader.py", line 254, in _load_bento
svc = import_service(\n", " File "/usr/local/lib/python3.11/site-packages/simple_di/init.py", line 139, in _
return func(_inject_args(bind.args), **_inject_kwargs(bind.kwargs))",
" File "/usr/local/lib/python3.11/site-packages/bentoml/_internal/service/loader.py", line 137, in import_service
module = importlib.import_module(module_name, package=working_dir)",
" File "/usr/local/lib/python3.11/importlib/init.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)",
" File "", line 1204, in _gcd_import",
" File "", line 1176, in _find_and_load",
" File "", line 1147, in _find_and_load_unlocked",
" File "", line 690, in _load_unlocked",
" File "", line 940, in exec_module",
" File "", line 241, in _call_with_frames_removed",
" File "/home/bentoml/bento/src/iris_classifier.py", line 7, in
iris_clf_runner = bentoml.sklearn.get("iris_clf:latest").to_runner()",
" File "/usr/local/lib/python3.11/site-packages/bentoml/_internal/frameworks/sklearn.py", line 50, in get
model = bentoml.models.get(tag_like)",
" File "/usr/local/lib/python3.11/site-packages/simple_di/init.py", line 139, in _
return func(_inject_args(bind.args), **_inject_kwargs(bind.kwargs))",
" File "/usr/local/lib/python3.11/site-packages/bentoml/models.py", line 45, in get
return _model_store.get(tag)",
" File "/usr/local/lib/python3.11/site-packages/bentoml/_internal/store.py", line 149, in get
self._recreate_latest(_tag)\n", " File "/usr/local/lib/python3.11/site-packages/bentoml/_internal/store.py", line 132, in _recreate_latest
with self._fs.open(tag.latest_path(), "w") as latest_file:
", " File "/usr/local/lib/python3.11/site-packages/fs/wrapfs.py", line 472, in open
open_file = _fs.open(\n", " File "/usr/local/lib/python3.11/site-packages/fs/osfs.py", line 643, in open
with convert_os_errors("open", path):
", " File "/usr/local/lib/python3.11/site-packages/fs/error_tools.py", line 89, in exit
reraise(fserror, fserror(self._path, exc=exc_value), traceback)\n", " File "/usr/local/lib/python3.11/site-packages/six.py", line 718, in reraise
raise value.with_traceback(tb)\n", " File "/usr/local/lib/python3.11/site-packages/fs/osfs.py", line 647, in open
return io.open(\n"]}