Skip to content

Commit

Permalink
Use wide strings to deal with utf8 files/command line args on Windows
Browse files Browse the repository at this point in the history
  • Loading branch information
@mstimberg
mstimberg committed Apr 25, 2024
1 parent 38fdbc8 commit 15ae443
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 46 deletions.
13 changes: 7 additions & 6 deletions brian2/devices/cpp_standalone/templates/main.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include <stdlib.h>
#include <filesystem>
#include "objects.h"
#include <ctime>
#include <time.h>
Expand All @@ -24,24 +25,24 @@

{{report_func|autoindent}}

void set_from_command_line(const std::vector<std::string> args)
void set_from_command_line(const std::vector<std::wstring> args)
{
for (const auto& arg : args) {
// Split into two parts
size_t equal_sign = arg.find("=");
size_t equal_sign = arg.find(L"=");
auto name = arg.substr(0, equal_sign);
auto value = arg.substr(equal_sign + 1, arg.length());
brian::set_variable_by_name(name, value);
}
}
int main(int argc, char **argv)
int wmain(int argc, wchar_t* argv[])
{
std::vector<std::string> args(argv + 1, argv + argc);
if (args.size() >=2 && args[0] == "--results_dir")
std::vector<std::wstring> args(argv + 1, argv + argc);
if (args.size() >=2 && args[0] == L"--results_dir")
{
brian::results_dir = args[1];
#ifdef DEBUG
std::cout << "Setting results dir to '" << brian::results_dir << "'" << std::endl;
std::cout << "Setting results dir to '" << std::filesystem::path(brian::results_dir) << "'" << std::endl;
#endif
args.erase(args.begin(), args.begin()+2);
}
Expand Down
80 changes: 41 additions & 39 deletions brian2/devices/cpp_standalone/templates/objects.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@

{% macro set_from_value(var_dtype, array_name) %}
{% if c_data_type(var_dtype) == 'double' %}
set_variable_from_value<double>(name, {{array_name}}, var_size, (double)atof(s_value.c_str()));
set_variable_from_value<double>(name, {{array_name}}, var_size, (double)std::stof(s_value));
{% elif c_data_type(var_dtype) == 'float' %}
set_variable_from_value<float>(name, {{array_name}}, var_size, (float)atof(s_value.c_str()));
set_variable_from_value<float>(name, {{array_name}}, var_size, (float)std::stof(s_value));
{% elif c_data_type(var_dtype) == 'int32_t' %}
set_variable_from_value<int32_t>(name, {{array_name}}, var_size, (int32_t)atoi(s_value.c_str()));
set_variable_from_value<int32_t>(name, {{array_name}}, var_size, (int32_t)std::stoi(s_value));
{% elif c_data_type(var_dtype) == 'int64_t' %}
set_variable_from_value<int64_t>(name, {{array_name}}, var_size, (int64_t)atol(s_value.c_str()));
set_variable_from_value<int64_t>(name, {{array_name}}, var_size, (int64_t)std::stoi(s_value));
{% elif c_data_type(var_dtype) == 'char' %}
set_variable_from_value(name, {{array_name}}, var_size, (char)atoi(s_value.c_str()));
set_variable_from_value(name, {{array_name}}, var_size, (char)std::stoi(s_value));
{% endif %}
{%- endmacro %}

Expand All @@ -24,75 +24,76 @@ set_variable_from_value(name, {{array_name}}, var_size, (char)atoi(s_value.c_str
#include<vector>
#include<iostream>
#include<fstream>
#include<filesystem>
#include<map>
#include<tuple>
#include<cstdlib>
#include<string>

namespace brian {

std::string results_dir = "results/"; // can be overwritten by --results_dir command line arg
std::filesystem::path results_dir = L"results/"; // can be overwritten by --results_dir command line arg
std::vector< rk_state* > _mersenne_twister_states;

//////////////// networks /////////////////
{% for net in networks | sort(attribute='name') %}
Network {{net.name}};
{% endfor %}

void set_variable_from_value(std::string varname, char* var_pointer, size_t size, char value) {
void set_variable_from_value(std::wstring varname, char* var_pointer, size_t size, wchar_t value) {
#ifdef DEBUG
std::cout << "Setting '" << varname << "' to " << (value == 1 ? "True" : "False") << std::endl;
std::cout << "Setting '" << varname.c_str() << "' to " << (value == 1 ? "True" : "False") << std::endl;
#endif
std::fill(var_pointer, var_pointer+size, value);
}

template<class T> void set_variable_from_value(std::string varname, T* var_pointer, size_t size, T value) {
template<class T> void set_variable_from_value(std::wstring varname, T* var_pointer, size_t size, T value) {
#ifdef DEBUG
std::cout << "Setting '" << varname << "' to " << value << std::endl;
std::cout << "Setting '" << varname.c_str() << "' to " << value << std::endl;
#endif
std::fill(var_pointer, var_pointer+size, value);
}

template<class T> void set_variable_from_file(std::string varname, T* var_pointer, size_t data_size, std::string filename) {
template<class T> void set_variable_from_file(std::wstring varname, T* var_pointer, size_t data_size, std::wstring filename) {
ifstream f;
streampos size;
#ifdef DEBUG
std::cout << "Setting '" << varname << "' from file '" << filename << "'" << std::endl;
std::cout << "Setting '" << varname.c_str() << "' from file '" << filename.c_str() << "'" << std::endl;
#endif
f.open(filename, ios::in | ios::binary | ios::ate);
size = f.tellg();
if (size != data_size) {
std::cerr << "Error reading '" << filename << "': file size " << size << " does not match expected size " << data_size << std::endl;
std::cerr << "Error reading '" << filename.c_str() << "': file size " << size << " does not match expected size " << data_size << std::endl;
return;
}
f.seekg(0, ios::beg);
if (f.is_open())
f.read(reinterpret_cast<char *>(var_pointer), data_size);
else
std::cerr << "Could not read '" << filename << "'" << std::endl;
std::cerr << "Could not read '" << filename.c_str() << "'" << std::endl;
if (f.fail())
std::cerr << "Error reading '" << filename << "'" << std::endl;
std::cerr << "Error reading '" << filename.c_str() << "'" << std::endl;
}

//////////////// set arrays by name ///////
void set_variable_by_name(std::string name, std::string s_value) {
void set_variable_by_name(std::wstring name, std::wstring s_value) {
size_t var_size;
size_t data_size;
std::for_each(s_value.begin(), s_value.end(), [](char& c) // modify in-place
std::for_each(s_value.begin(), s_value.end(), [](wchar_t& c) // modify in-place
{
c = std::tolower(static_cast<unsigned char>(c));
c = std::tolower(static_cast<unsigned wchar_t>(c));
});
if (s_value == "true")
s_value = "1";
else if (s_value == "false")
s_value = "0";
if (s_value == L"true")
s_value = L"1";
else if (s_value == L"false")
s_value = L"0";
// non-dynamic arrays
{% for var, varname in array_specs | dictsort(by='value') %}
{% if not var in dynamic_array_specs and not var.read_only %}
if (name == "{{var.owner.name}}.{{var.name}}") {
if (name == L"{{var.owner.name}}.{{var.name}}") {
var_size = {{var.size}};
data_size = {{var.size}}*sizeof({{c_data_type(var.dtype)}});
if (s_value[0] == '-' || (s_value[0] >= '0' && s_value[0] <= '9')) {
if (s_value[0] == L'-' || (s_value[0] >= L'0' && s_value[0] <= L'9')) {
// set from single value
{{ set_from_value(var.dtype, get_array_name(var)) }}
} else {
Expand All @@ -106,10 +107,10 @@ void set_variable_by_name(std::string name, std::string s_value) {
// dynamic arrays (1d)
{% for var, varname in dynamic_array_specs | dictsort(by='value') %}
{% if not var.read_only %}
if (name == "{{var.owner.name}}.{{var.name}}") {
if (name == L"{{var.owner.name}}.{{var.name}}") {
var_size = {{get_array_name(var, access_data=False)}}.size();
data_size = var_size*sizeof({{c_data_type(var.dtype)}});
if (s_value[0] == '-' || (s_value[0] >= '0' && s_value[0] <= '9')) {
if (s_value[0] == L'-' || (s_value[0] >= L'0' && s_value[0] <= L'9')) {
// set from single value
{{ set_from_value(var.dtype, "&" + get_array_name(var, False) + "[0]") }}
} else {
Expand All @@ -121,10 +122,10 @@ void set_variable_by_name(std::string name, std::string s_value) {
{% endif %}
{% endfor %}
{% for var, varname in timed_arrays | dictsort(by='value') %}
if (name == "{{varname}}.values") {
if (name == L"{{varname}}.values") {
var_size = {{var.values.size}};
data_size = var_size*sizeof({{c_data_type(var.values.dtype)}});
if (s_value[0] == '-' || (s_value[0] >= '0' && s_value[0] <= '9')) {
if (s_value[0] == L'-' || (s_value[0] >= L'0' && s_value[0] <= L'9')) {
// set from single value
{{ set_from_value(var.values.dtype, varname + "_values") }}

Expand All @@ -135,7 +136,7 @@ void set_variable_by_name(std::string name, std::string s_value) {
return;
}
{% endfor %}
std::cerr << "Cannot set unknown variable '" << name << "'." << std::endl;
std::cerr << "Cannot set unknown variable '" << name.c_str() << "'." << std::endl;
exit(1);
}
//////////////// arrays ///////////////////
Expand Down Expand Up @@ -233,10 +234,10 @@ void _init_arrays()
void _load_arrays()
{
using namespace brian;

namespace fs = std::filesystem;
{% for (name, dtype_spec, N, filename) in static_array_specs | sort %}
ifstream f{{name}};
f{{name}}.open("static_arrays/{{name}}", ios::in | ios::binary);
f{{name}}.open(fs::path(L"static_arrays/{{name}}"), ios::in | ios::binary);
if(f{{name}}.is_open())
{
{% if name in dynamic_array_specs.values() %}
Expand All @@ -254,11 +255,11 @@ void _load_arrays()
void _write_arrays()
{
using namespace brian;

namespace fs = std::filesystem;
{% for var, varname in array_specs | dictsort(by='value') %}
{% if not (var in dynamic_array_specs or var in dynamic_array_2d_specs) %}
ofstream outfile_{{varname}};
outfile_{{varname}}.open(results_dir + "{{get_array_filename(var)}}", ios::binary | ios::out);
outfile_{{varname}}.open(results_dir / fs::path(L"{{get_array_filename(var)}}"), ios::binary | ios::out);
if(outfile_{{varname}}.is_open())
{
outfile_{{varname}}.write(reinterpret_cast<char*>({{varname}}), {{var.size}}*sizeof({{get_array_name(var)}}[0]));
Expand All @@ -272,7 +273,7 @@ void _write_arrays()

{% for var, varname in dynamic_array_specs | dictsort(by='value') %}
ofstream outfile_{{varname}};
outfile_{{varname}}.open(results_dir + "{{get_array_filename(var)}}", ios::binary | ios::out);
outfile_{{varname}}.open(results_dir / fs::path(L"{{get_array_filename(var)}}"), ios::binary | ios::out);
if(outfile_{{varname}}.is_open())
{
if (! {{varname}}.empty() )
Expand All @@ -288,7 +289,7 @@ void _write_arrays()

{% for var, varname in dynamic_array_2d_specs | dictsort(by='value') %}
ofstream outfile_{{varname}};
outfile_{{varname}}.open(results_dir + "{{get_array_filename(var)}}", ios::binary | ios::out);
outfile_{{varname}}.open(results_dir / fs::path(L"{{get_array_filename(var)}}"), ios::binary | ios::out);
if(outfile_{{varname}}.is_open())
{
for (int n=0; n<{{varname}}.n; n++)
Expand All @@ -307,7 +308,7 @@ void _write_arrays()
{% if profiled_codeobjects is defined and profiled_codeobjects %}
// Write profiling info to disk
ofstream outfile_profiling_info;
outfile_profiling_info.open(results_dir + "profiling_info.txt", ios::out);
outfile_profiling_info.open(results_dir / fs::path("profiling_info.txt"), ios::out);
if(outfile_profiling_info.is_open())
{
{% for codeobj in profiled_codeobjects | sort %}
Expand All @@ -321,7 +322,7 @@ void _write_arrays()
{% endif %}
// Write last run info to disk
ofstream outfile_last_run_info;
outfile_last_run_info.open(results_dir + "last_run_info.txt", ios::out);
outfile_last_run_info.open(results_dir / fs::path("last_run_info.txt"), ios::out);
if(outfile_last_run_info.is_open())
{
outfile_last_run_info << (Network::_last_run_time) << " " << (Network::_last_run_completed_fraction) << std::endl;
Expand Down Expand Up @@ -373,12 +374,13 @@ void _dealloc_arrays()
#include "brianlib/stdint_compat.h"
#include "network.h"
#include "randomkit.h"
#include<filesystem>
#include<vector>
{{ openmp_pragma('include') }}

namespace brian {

extern std::string results_dir;
extern std::filesystem::path results_dir;
// In OpenMP we need one state per thread
extern std::vector< rk_state* > _mersenne_twister_states;

Expand All @@ -394,7 +396,7 @@ extern Network {{net.name}};



void set_variable_by_name(std::string, std::string);
void set_variable_by_name(std::wstring, std::wstring);

//////////////// dynamic arrays ///////////
{% for var, varname in dynamic_array_specs | dictsort(by='value') %}
Expand Down
2 changes: 1 addition & 1 deletion brian2/devices/cpp_standalone/templates/win_makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ clean:

{% for fname, base in zip(source_files, source_bases) | sort(attribute='0')%}
{{base}}.obj: win_makefile
cl /c /EHsc /I. {{compiler_flags}} {{openmp_flag}} {{fname}} /Fo{{base}}.obj {{compiler_debug_flags}}
cl /c /EHsc /I. /source-charset:utf-8 /std:c++17 {{compiler_flags}} {{openmp_flag}} {{fname}} /Fo{{base}}.obj {{compiler_debug_flags}}

{% endfor %}

Expand Down

0 comments on commit 15ae443

Please sign in to comment.