Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

UPBGE: Allow asynchronous mesh libload. #897

Open
wants to merge 1 commit into
base: ge_libload_status_cb
Choose a base branch
from
Open

UPBGE: Allow asynchronous mesh libload. #897

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build_files/cmake/macros.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -621,11 +621,11 @@ function(SETUP_BLENDER_SORTED_LIBS)
ge_device
ge_rasterizer
ge_oglrasterizer
ge_common
ge_logic_expressions
ge_scenegraph
ge_logic_network
ge_videotex
ge_common

bf_render
bf_python
Expand Down
2 changes: 1 addition & 1 deletion source/blenderplayer/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,10 @@ endif()
ge_rasterizer
ge_oglrasterizer
ge_logic_expressions
ge_common
ge_scenegraph
ge_logic_network
ge_videotex
ge_common

bf_editor_datafiles

Expand Down
158 changes: 85 additions & 73 deletions source/gameengine/Converter/BL_Converter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ extern "C" {
# include "BKE_report.h"
}

#include "BLI_task.h"
#include "CM_Message.h"

#include <cstring>
Expand Down Expand Up @@ -126,9 +125,6 @@ BL_Converter::BL_Converter(Main *maggie, KX_KetsjiEngine *engine, bool alwaysUse
m_alwaysUseExpandFraming(alwaysUseExpandFraming),
m_camZoom(camZoom)
{
BKE_main_id_tag_all(maggie, LIB_TAG_DOIT, false); // avoid re-tagging later on
m_threadinfo.m_pool = BLI_task_pool_create(engine->GetTaskScheduler(), nullptr);

m_maggies.push_back(m_maggie);
}

Expand All @@ -138,11 +134,6 @@ BL_Converter::~BL_Converter()
while (!m_dynamicMaggies.empty()) {
FreeBlendFile(m_dynamicMaggies.front());
}

/* Thread infos like mutex must be freed after FreeBlendFile function.
Because it needs to lock the mutex, even if there's no active task when it's
in the scene converter destructor. */
BLI_task_pool_free(m_threadinfo.m_pool);
}

Scene *BL_Converter::GetBlenderSceneForName(const std::string &name)
Expand Down Expand Up @@ -262,56 +253,82 @@ std::vector<std::string> BL_Converter::GetLibraryNames() const
return names;
}

void BL_Converter::ProcessScheduledLibraries()
void BL_Converter::ProcessScheduledMerge()
{
m_threadinfo.m_mutex.Lock();
m_mergeMutex.lock();
const std::vector<KX_LibLoadStatus *> mergeQueue = m_mergequeue;
m_mergequeue.clear();
m_threadinfo.m_mutex.Unlock();
m_mergeMutex.unlock();

for (KX_LibLoadStatus *libload : mergeQueue) {
KX_Scene *mergeScene = libload->GetMergeScene();
std::vector<BL_SceneConverter>& converters = libload->GetSceneConverters();
for (const BL_SceneConverter& converter : converters) {
MergeScene(mergeScene, converter);
for (const BL_SceneConverter& converter : libload->GetSceneConverters()) {
KX_Scene *scene = converter.GetScene();

// Finalize synchronous conversion.
PostConvertScene(converter);
// Merge the scene data.
MergeSceneData(mergeScene, converter);

// The scene must be merged ?
if (scene != mergeScene) {
// Merge the scene objects and managers.
mergeScene->MergeScene(scene);
}

// Reload shader with all lights in the merged scene.
ReloadShaders(mergeScene);

if (scene != mergeScene) {
// Free the merged scene.
delete scene;
}
}

libload->Finish();
}
}

void BL_Converter::ProcessScheduledFree()
{
for (Main *maggie : m_freeQueue) {
FreeBlendFileData(maggie);
}
m_freeQueue.clear();
}


void BL_Converter::ProcessScheduledLibraries()
{
ProcessScheduledMerge();
ProcessScheduledFree();
}

void BL_Converter::FinalizeAsyncLoads()
{
// Finish all loading libraries.
BLI_task_pool_work_and_wait(m_threadinfo.m_pool);
m_convertThread.wait();
// Merge all libraries data in the current scene, to avoid memory leak of unmerged scenes.
ProcessScheduledLibraries();
}

void BL_Converter::AddScenesToMergeQueue(KX_LibLoadStatus *status)
void BL_Converter::ConvertLibraryTask(KX_LibLoadStatus *status)
{
m_threadinfo.m_mutex.Lock();
m_mergequeue.push_back(status);
m_threadinfo.m_mutex.Unlock();
}

void BL_Converter::AsyncConvertTask(TaskPool *pool, void *ptr, int UNUSED(threadid))
{
KX_LibLoadStatus *status = static_cast<KX_LibLoadStatus *>(ptr);
BL_Converter *converter = status->GetConverter();
// Get the function used to convert data.
const KX_LibLoadStatus::ConvertFunction& func = status->GetConvertFunction();

std::vector<BL_SceneConverter>& converters = status->GetSceneConverters();
for (BL_SceneConverter& sceneConverter : converters) {
converter->ConvertScene(sceneConverter, true, false);
status->AddProgress((1.0f / converters.size()) * 0.9f); // We'll call conversion 90% and merging 10% for now
// Call conversion.
func(*this, sceneConverter);
// We'll call conversion 90% and merging 10% for now.
status->AddProgress((1.0f / converters.size()) * 0.9f);
}

status->GetConverter()->AddScenesToMergeQueue(status);
m_mergeMutex.lock();
// Register library for merging.
m_mergequeue.push_back(status);
m_mergeMutex.unlock();
}

Main *BL_Converter::GetLibraryPath(const std::string& path)
Expand Down Expand Up @@ -433,32 +450,40 @@ KX_LibLoadStatus *BL_Converter::LinkBlendFile(BlendHandle *blendlib, const char

// Linking done.

KX_LibLoadStatus *status = new KX_LibLoadStatus(this, m_ketsjiEngine, scene_merge, path);

const BL_Resource::Library libraryId(main_newlib);
// The function used to convert data.
KX_LibLoadStatus::ConvertFunction convertFunction;
// Scenes used during the conversion.
std::vector<KX_Scene *> scenes;

// Select the conversion function and register scenes holding data to convert.
switch (idcode) {
case ID_ME:
{
BL_SceneConverter sceneConverter(scene_merge, libraryId);
// Convert all new meshes into BGE meshes
for (Mesh *mesh = (Mesh *)main_newlib->mesh.first; mesh; mesh = (Mesh *)mesh->id.next) {
BL_ConvertMesh((Mesh *)mesh, nullptr, scene_merge, sceneConverter);
}
convertFunction =
[main_newlib](BL_Converter& converter, BL_SceneConverter& sceneConverter)
{
// Convert all new meshes into BGE meshes
for (Mesh *mesh = (Mesh *)main_newlib->mesh.first; mesh; mesh = (Mesh *)mesh->id.next) {
BL_ConvertMesh((Mesh *)mesh, nullptr, sceneConverter.GetScene(), sceneConverter);
}
};

scenes.push_back(scene_merge);

// Merge the meshes and materials in the targeted scene.
MergeSceneData(scene_merge, sceneConverter);
// Load shaders for new created materials.
ReloadShaders(scene_merge);
break;
}
case ID_AC:
{
BL_SceneConverter sceneConverter(scene_merge, libraryId);
// Convert all actions and register.
BL_ConvertActions(scene_merge, main_newlib, sceneConverter);
// Merge the actions in the targeted scene.
MergeSceneData(scene_merge, sceneConverter);
convertFunction =
[main_newlib](BL_Converter& converter, BL_SceneConverter& sceneConverter)
{
// Convert all actions and register.
BL_ConvertActions(sceneConverter.GetScene(), main_newlib, sceneConverter);
};

scenes.push_back(scene_merge);

break;
}
case ID_SCE:
Expand All @@ -484,29 +509,30 @@ KX_LibLoadStatus *BL_Converter::LinkBlendFile(BlendHandle *blendlib, const char
MergeSceneData(scene_merge, sceneConverter);
}

convertFunction =
[](BL_Converter& converter, BL_SceneConverter& sceneConverter)
{
// Convert all scene data.
converter.ConvertScene(sceneConverter, true, false);
};

for (Scene *bscene = (Scene *)main_newlib->scene.first; bscene; bscene = (Scene *)bscene->id.next) {
KX_Scene *scene = m_ketsjiEngine->CreateScene(bscene);

// Schedule conversion and merge.
if (options & LIB_LOAD_ASYNC) {
status->AddSceneConverter(scene, libraryId);
}
// Or proceed direct conversion and merge.
else {
BL_SceneConverter sceneConverter(scene, libraryId);
ConvertScene(sceneConverter, true, false);
MergeScene(scene_merge, sceneConverter);
}
scenes.push_back(scene);
}
break;
}
}

KX_LibLoadStatus *status = new KX_LibLoadStatus(scenes, scene_merge, convertFunction, libraryId, path);

if (options & LIB_LOAD_ASYNC) {
BLI_task_pool_push(m_threadinfo.m_pool, AsyncConvertTask, (void *)status, false, TASK_PRIORITY_LOW);
m_convertThread.run([status, this](){ this->ConvertLibraryTask(status); });
}
else {
status->Finish();
ConvertLibraryTask(status);
ProcessScheduledMerge();
}

// Register new library.
Expand All @@ -526,9 +552,9 @@ bool BL_Converter::FreeBlendFileData(Main *maggie)

KX_LibLoadStatus *status = m_libloadStatus[maggie].get();
// If the given library is currently in loading, we do nothing.
m_threadinfo.m_mutex.Lock();
m_mergeMutex.lock();
const bool finished = status->IsFinished();
m_threadinfo.m_mutex.Unlock();
m_mergeMutex.unlock();

if (!finished) {
CM_Error("Library (" << maggie->name << ") is currently being loaded asynchronously, and cannot be freed until this process is done");
Expand Down Expand Up @@ -669,20 +695,6 @@ void BL_Converter::MergeSceneData(KX_Scene *to, const BL_SceneConverter& convert
m_sceneSlots[to].Merge(converter);
}

void BL_Converter::MergeScene(KX_Scene *to, const BL_SceneConverter& converter)
{
PostConvertScene(converter);

MergeSceneData(to, converter);

KX_Scene *from = converter.GetScene();
to->MergeScene(from);

ReloadShaders(to);

delete from;
}

void BL_Converter::ReloadShaders(KX_Scene *scene)
{
for (std::unique_ptr<KX_BlenderMaterial>& mat : m_sceneSlots[scene].m_materials) {
Expand Down
31 changes: 10 additions & 21 deletions source/gameengine/Converter/BL_Converter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@

#include "BL_SceneConverter.h"

#include "CM_Thread.h"
#include "tbb/task_group.h"

#include <mutex>

class EXP_StringValue;
class BL_SceneConverter;
Expand All @@ -64,7 +66,6 @@ struct Material;
struct bAction;
struct bActuator;
struct bController;
struct TaskPool;

template<class Value>
using UniquePtrList = std::vector<std::unique_ptr<Value> >;
Expand All @@ -90,10 +91,8 @@ class BL_Converter

std::map<KX_Scene *, SceneSlot> m_sceneSlots;

struct ThreadInfo {
TaskPool *m_pool;
CM_ThreadMutex m_mutex;
} m_threadinfo;
tbb::task_group m_convertThread;
std::mutex m_mergeMutex;

/// List of loaded libraries to merge.
std::vector<KX_LibLoadStatus *> m_mergequeue;
Expand Down Expand Up @@ -128,28 +127,15 @@ class BL_Converter
*/
void MergeSceneData(KX_Scene *to, const BL_SceneConverter& converter);

/** Complete process of scene merging:
* - post convert
* - merge data
* - merge scene (KX_Scene::MergeScene)
* - finalize data
*/
void MergeScene(KX_Scene *to, const BL_SceneConverter& converter);

/** Regenerate material shader after a converting or merging a scene
* depending on all the lights into the destination scene.
*/
void ReloadShaders(KX_Scene *scene);
/// Regenerate shaders of material in given scene converter, used when creating mesh.
void ReloadShaders(const BL_SceneConverter& converter);

/// Delay library merging to ProcessScheduledLibraries.
void AddScenesToMergeQueue(KX_LibLoadStatus *status);

/** Asynchronously convert scenes from a library.
* \param ptr Pointer to the library status.
*/
static void AsyncConvertTask(TaskPool *pool, void *ptr, int UNUSED(threadid));
/// Asynchronously convert scenes from a library.
void ConvertLibraryTask(KX_LibLoadStatus *status);

Main *GetLibraryPath(const std::string& path);

Expand All @@ -160,6 +146,9 @@ class BL_Converter
/// Free blend file and remove library from internal lists.
void FreeBlendFile(Main *maggie);

void ProcessScheduledMerge();
void ProcessScheduledFree();

public:
BL_Converter(Main *maggie, KX_KetsjiEngine *engine, bool alwaysUseExpandFraming, float camZoom);
virtual ~BL_Converter();
Expand Down
Loading