Add tools/hyg-database parser.cpp

All of the parsing code is implemented except Bayer-Flamsteed names
Still need to arrange writing JSON / CBOR data and applying compression

Implement everything but LZ4 compression

Friendly reminder that CBOR is network byte order encoded, AKA big-endian
Validation, parsing, JSON and CBOR all working
Only thing remaining is to handle LZ4 de/compression

Split cbor writing to its own header

CMakeLists.txt

@@ -218,7 +218,6 @@ endif (MSVC)
 find_package(Freetype REQUIRED)
 find_package(OpenGL REQUIRED)
 
-add_subdirectory(contrib/csv-parser)
 add_subdirectory(contrib/lz4)
 add_subdirectory(contrib/fmt)
 
@@ -334,6 +333,8 @@ target_link_libraries(${PROJECT_NAME} LINK_PRIVATE ${pioneerLibs} ${winLibs})
 target_link_libraries(modelcompiler LINK_PRIVATE ${pioneerLibs} ${winLibs})
 target_link_libraries(savegamedump LINK_PRIVATE pioneer-core ${SDL2_IMAGE_LIBRARIES} ${winLibs})
 
+add_subdirectory(tools)
+
 set_cxx11_properties(${PROJECT_NAME} modelcompiler savegamedump)
 
 if(MSVC)

tools/CMakeLists.txt

@@ -0,0 +1,20 @@
+
+list(APPEND HYG_DATABASE_PARSER_SOURCES
+	hyg-database-to-json.cpp
+	${CMAKE_SOURCE_DIR}/src/LZ4Format.cpp
+)
+
+find_package(Threads REQUIRED)
+add_executable(hyg-database-parser ${HYG_DATABASE_PARSER_SOURCES})
+target_link_libraries(hyg-database-parser PRIVATE
+	Threads::Threads
+	lz4)
+target_include_directories(hyg-database-parser PRIVATE
+	${CMAKE_SOURCE_DIR}/contrib/string-view-lite
+	${CMAKE_SOURCE_DIR}/contrib/csv-parser)
+
+set_target_properties(${PROJECT_NAME} modelcompiler savegamedump pioneerLib PROPERTIES
+	CXX_STANDARD 11
+	CXX_STANDARD_REQUIRED ON
+	CXX_EXTENSIONS ON
+)

tools/basic_cbor.h

@@ -0,0 +1,105 @@
+// Copyright © 2008-2020 Pioneer Developers. See AUTHORS.txt for details
+// Licensed under the terms of the GPL v3. See licenses/GPL-3.txt
+
+/*
+	The basic guts of a CBOR encoder - we're not concerned with full standards
+	compliance, merely with efficiently outputting valid data that
+	nlohmann::json can then read in as JSON data.
+
+	REMEMBER THAT CBOR IS BIG-ENDIAN ENCODED!
+*/
+
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <vector>
+
+enum class CBORTag {
+	Integer = 0,
+	NegInteger = 1,
+	ByteString = 2,
+	String = 3,
+	Array = 4,
+	Object = 5,
+	Simple = 7
+};
+
+namespace cbor {
+
+	// Thanks to https://stackoverflow.com/questions/2782725/converting-float-values-from-big-endian-to-little-endian
+	void swap4(uint8_t *data)
+	{
+		auto *ptr = reinterpret_cast<uint32_t *>(data);
+		uint32_t n = *ptr;
+		n = ((n >> 8) & 0x00ff00ff) | ((n << 8) & 0xff00ff00); // 1234 -> 2143
+		n = ((n >> 16) & 0x0000ffff) | ((n << 16) & 0xffff0000); // 2143 -> 4321
+		*ptr = n;
+	}
+
+	template <typename T>
+	void push4(std::vector<uint8_t> &out, const T val)
+	{
+		size_t cur_size = out.size();
+		out.resize(out.size() + 4);
+		*reinterpret_cast<T *>(out.data() + cur_size) = val;
+		swap4(out.data() + cur_size);
+	}
+
+	// Push a tag and an additional information value up to 32 bits.
+	void push_tag(std::vector<uint8_t> &out, const CBORTag tag, const uint32_t val)
+	{
+		uint8_t tagVal = static_cast<uint8_t>(tag) << 5;
+		if (val < 24)
+			out.push_back(tagVal | (val & 0xFF));
+
+		else if (val < 1 << 8) {
+			out.push_back(tagVal | 24);
+			out.push_back(val & 0xFF);
+		}
+
+		else if (val < 1 << 16) {
+			out.push_back(tagVal | 25);
+			out.push_back(val >> 8 & 0xFF);
+			out.push_back(val & 0xFF);
+		}
+
+		else {
+			out.push_back(tagVal | 26);
+			push4(out, val);
+		}
+	}
+
+	void push_simple(std::vector<uint8_t> &out, const uint8_t simple)
+	{
+		out.push_back(static_cast<uint8_t>(CBORTag::Simple) << 5 | (simple & 0x1F));
+		if (simple > 31)
+			out.push_back(simple);
+	}
+
+	void push_float(std::vector<uint8_t> &out, const float val)
+	{
+		push_simple(out, 26);
+		push4(out, val);
+	}
+
+	void push_string(std::vector<uint8_t> &out, const std::string &string)
+	{
+		push_tag(out, CBORTag::String, string.size());
+		if (string.size() > 0) {
+			size_t cur_size = out.size();
+			out.resize(out.size() + string.size());
+
+			std::memcpy(out.data() + cur_size, string.data(), string.size());
+		}
+	}
+
+	void push_int(std::vector<uint8_t> &out, const int val)
+	{
+		if (val < 0)
+			push_tag(out, CBORTag::NegInteger, -1 - val);
+		else
+			push_tag(out, CBORTag::Integer, val);
+	}
+} // namespace cbor