From 250229f25144327702702966fd2cba7a019d5b96 Mon Sep 17 00:00:00 2001 From: Wallen-Ribeiro Date: Wed, 11 Sep 2024 23:42:13 +0100 Subject: [PATCH 1/5] added protobufs instructions to Dockerfile --- .gitmodules | 3 +++ django/Dockerfile | 20 +++++++++++--------- tts-protobufs | 1 + 3 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 .gitmodules create mode 160000 tts-protobufs diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..de73ac4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tts-protobufs"] + path = tts-protobufs + url = https://github.com/NIAEFEUP/tts-protobufs.git diff --git a/django/Dockerfile b/django/Dockerfile index cafc8b3..500d218 100644 --- a/django/Dockerfile +++ b/django/Dockerfile @@ -1,31 +1,33 @@ # deps FROM python:3.8-slim-buster AS deps -WORKDIR /usr/src/django/ +WORKDIR /usr/src/django/ -# Get's the output from the django in realtime. ENV PYTHONUNBUFFERED=1 -# Copy requirements -COPY ./requirements.txt ./requirements.txt +COPY ./requirements.txt ./requirements.txt -# Dependencies for building the requirements RUN apt-get update RUN apt-get -y install build-essential +RUN apt-get -y install protobuf-compiler + +COPY ../tts-protobufs/protos/ ./tts-protobufs/protos/ + +RUN mkdir -p ./generated + +RUN protoc --python_out=./generated ./tts-protobufs/protos/ -# Install postgres dependencies (pgsql client and development files) COPY ./etc/pgdg.sh /tmp/pgdg.sh RUN /tmp/pgdg.sh RUN apt -y install libpq-dev postgresql-client-16 RUN apt -y clean && rm -rf /var/lib/apt/lists/* -# Install the requirements RUN pip install -r requirements.txt EXPOSE 8000 -COPY ./entrypoint.sh ./entrypoint.sh +COPY ./entrypoint.sh ./entrypoint.sh ENTRYPOINT ["/usr/src/django/entrypoint.sh"] # prod @@ -36,4 +38,4 @@ COPY university/ ./university COPY manage.py tasks.py ./ COPY ./entrypoint_prod.sh ./entrypoint_prod.sh -ENTRYPOINT ["/usr/src/django/entrypoint_prod.sh"] +ENTRYPOINT ["/usr/src/django/entrypoint_prod.sh"] \ No newline at end of file diff --git a/tts-protobufs b/tts-protobufs new file mode 160000 index 0000000..12aefb5 --- /dev/null +++ b/tts-protobufs @@ -0,0 +1 @@ +Subproject commit 12aefb5c815feaecfa311a286934a8c72c362a99 From 038a200df563fb960371e8145855fff35071d083 Mon Sep 17 00:00:00 2001 From: Wallen-Ribeiro Date: Mon, 16 Sep 2024 21:21:10 +0100 Subject: [PATCH 2/5] added protos volume to yaml --- django/Dockerfile | 12 ++++++++---- docker-compose.yaml | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/django/Dockerfile b/django/Dockerfile index 500d218..1ae14ed 100644 --- a/django/Dockerfile +++ b/django/Dockerfile @@ -3,20 +3,23 @@ FROM python:3.8-slim-buster AS deps WORKDIR /usr/src/django/ +# Get's the output from the django in realtime. ENV PYTHONUNBUFFERED=1 +# Copy requirements COPY ./requirements.txt ./requirements.txt +# Dependencies for building the requirements RUN apt-get update RUN apt-get -y install build-essential RUN apt-get -y install protobuf-compiler -COPY ../tts-protobufs/protos/ ./tts-protobufs/protos/ - +# Generating the python files from the protobufs +COPY ./tts-protobufs/protos/ ./protos RUN mkdir -p ./generated +RUN protoc --python_out=./generated ./protos -RUN protoc --python_out=./generated ./tts-protobufs/protos/ - +# Install postgres dependencies (pgsql client and development files) COPY ./etc/pgdg.sh /tmp/pgdg.sh RUN /tmp/pgdg.sh @@ -27,6 +30,7 @@ RUN pip install -r requirements.txt EXPOSE 8000 +# Install the requirements COPY ./entrypoint.sh ./entrypoint.sh ENTRYPOINT ["/usr/src/django/entrypoint.sh"] diff --git a/docker-compose.yaml b/docker-compose.yaml index 6fa1dc4..f93b394 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -35,6 +35,7 @@ services: command: ["python", "manage.py", "runserver", "0.0.0.0:8000"] volumes: - ./django/:/usr/src/django/ + - ./tts-protobufs/protos:/usr/src/django/protos ports: - 8100:8000 From 3faad904ccc4fc3414db72d5db22d72b974430c0 Mon Sep 17 00:00:00 2001 From: Wallen-Ribeiro Date: Wed, 25 Sep 2024 12:28:16 +0100 Subject: [PATCH 3/5] added protoc command to django entrypoint scripts --- django/Dockerfile | 5 ----- django/entrypoint.sh | 7 ++++++- django/entrypoint_prod.sh | 5 +++++ docker-compose.yaml | 3 ++- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/django/Dockerfile b/django/Dockerfile index 1ae14ed..ff281a5 100644 --- a/django/Dockerfile +++ b/django/Dockerfile @@ -14,11 +14,6 @@ RUN apt-get update RUN apt-get -y install build-essential RUN apt-get -y install protobuf-compiler -# Generating the python files from the protobufs -COPY ./tts-protobufs/protos/ ./protos -RUN mkdir -p ./generated -RUN protoc --python_out=./generated ./protos - # Install postgres dependencies (pgsql client and development files) COPY ./etc/pgdg.sh /tmp/pgdg.sh RUN /tmp/pgdg.sh diff --git a/django/entrypoint.sh b/django/entrypoint.sh index b7e8af4..b672397 100755 --- a/django/entrypoint.sh +++ b/django/entrypoint.sh @@ -16,7 +16,12 @@ until PGPASSWORD="${POSTGRES_PASSWORD}" psql -h "${POSTGRES_HOST}" -U "${POSTGRE >&2 echo "PostgreSQL is unavailable - sleeping" sleep 4 done ->&2 echo "PostgreSQL is up - executing command" +>&2 echo "PostgreSQL is up - executing commands" + +# Compile protobuf files +echo "Compiling protobuf files..." +protoc --python_out=generated ./protos/*.proto +echo "Protobuf files compiled successfully." # Migrate the Django. python manage.py inspectdb > university/models.py diff --git a/django/entrypoint_prod.sh b/django/entrypoint_prod.sh index b77e049..1914cc1 100755 --- a/django/entrypoint_prod.sh +++ b/django/entrypoint_prod.sh @@ -7,6 +7,11 @@ until PGPASSWORD="${POSTGRES_PASSWORD}" psql -h "${POSTGRES_HOST}" -U "${POSTGRE done >&2 echo "PostgreSQL is up - executing command" +# Compile protobuf files +echo "Compiling protobuf files..." +protoc --python_out=generated -I=./protos/ ./protos/*.proto +echo "Protobuf files compiled successfully." + # Migrate the Django. python manage.py inspectdb >university/models.py python manage.py makemigrations diff --git a/docker-compose.yaml b/docker-compose.yaml index f93b394..dea5736 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -35,7 +35,8 @@ services: command: ["python", "manage.py", "runserver", "0.0.0.0:8000"] volumes: - ./django/:/usr/src/django/ - - ./tts-protobufs/protos:/usr/src/django/protos + - ./tts-protobufs/protos/:/usr/src/django/protos/ + - ./tts-protobufs/generated/:/usr/src/django/generated/ ports: - 8100:8000 From 3d74f86a40d7731d17a05883812fac1b088c9f14 Mon Sep 17 00:00:00 2001 From: Wallen-Ribeiro Date: Wed, 25 Sep 2024 14:22:14 +0100 Subject: [PATCH 4/5] changed generated dir to ./protos --- docker-compose.yaml | 2 +- protos/class_pb2.py | 93 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 protos/class_pb2.py diff --git a/docker-compose.yaml b/docker-compose.yaml index dea5736..09b8e07 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -36,7 +36,7 @@ services: volumes: - ./django/:/usr/src/django/ - ./tts-protobufs/protos/:/usr/src/django/protos/ - - ./tts-protobufs/generated/:/usr/src/django/generated/ + - ./protos/:/usr/src/django/generated/ ports: - 8100:8000 diff --git a/protos/class_pb2.py b/protos/class_pb2.py new file mode 100644 index 0000000..0b8d21d --- /dev/null +++ b/protos/class_pb2.py @@ -0,0 +1,93 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: class.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='class.proto', + package='example', + syntax='proto3', + serialized_options=None, + serialized_pb=_b('\n\x0b\x63lass.proto\x12\x07\x65xample\x1a\x1fgoogle/protobuf/timestamp.proto\"k\n\x05\x43lass\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\x0e\x63ourse_unit_id\x18\x03 \x01(\x05\x12\x30\n\x0clast_updated\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestampb\x06proto3') + , + dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,]) + + + + +_CLASS = _descriptor.Descriptor( + name='Class', + full_name='example.Class', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='example.Class.id', index=0, + number=1, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='name', full_name='example.Class.name', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='course_unit_id', full_name='example.Class.course_unit_id', index=2, + number=3, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='last_updated', full_name='example.Class.last_updated', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=57, + serialized_end=164, +) + +_CLASS.fields_by_name['last_updated'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +DESCRIPTOR.message_types_by_name['Class'] = _CLASS +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Class = _reflection.GeneratedProtocolMessageType('Class', (_message.Message,), dict( + DESCRIPTOR = _CLASS, + __module__ = 'class_pb2' + # @@protoc_insertion_point(class_scope:example.Class) + )) +_sym_db.RegisterMessage(Class) + + +# @@protoc_insertion_point(module_scope) From 6e66e3f362e4a844684f16aeec16c9e0322eb189 Mon Sep 17 00:00:00 2001 From: Wallen-Ribeiro Date: Tue, 1 Oct 2024 18:01:08 +0100 Subject: [PATCH 5/5] changed protos path for better folder structure --- django/entrypoint.sh | 2 +- django/entrypoint_prod.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/django/entrypoint.sh b/django/entrypoint.sh index b672397..14dafff 100755 --- a/django/entrypoint.sh +++ b/django/entrypoint.sh @@ -20,7 +20,7 @@ done # Compile protobuf files echo "Compiling protobuf files..." -protoc --python_out=generated ./protos/*.proto +protoc --python_out=generated -I=./protos/ ./protos/**/*.proto echo "Protobuf files compiled successfully." # Migrate the Django. diff --git a/django/entrypoint_prod.sh b/django/entrypoint_prod.sh index 1914cc1..0802773 100755 --- a/django/entrypoint_prod.sh +++ b/django/entrypoint_prod.sh @@ -9,7 +9,7 @@ done # Compile protobuf files echo "Compiling protobuf files..." -protoc --python_out=generated -I=./protos/ ./protos/*.proto +protoc --python_out=generated -I=./protos/ ./protos/**/*.proto echo "Protobuf files compiled successfully." # Migrate the Django.