ProtobufJson.cc

#include <getopt.h>
#include <strings.h>
#include <wordexp.h>
#include <iostream>
#include <sstream>
#include <filesystem>
#include <fstream>
#include <queue>
#include <google/protobuf/compiler/importer.h>
#include <google/protobuf/descriptor.h>
#include <google/protobuf/util/json_util.h>
#include <google/protobuf/dynamic_message.h>
#include <google/protobuf/stubs/strutil.h>

using namespace google::protobuf::compiler;
using namespace google::protobuf::util;
using google::protobuf::FileDescriptor;
using google::protobuf::FileDescriptorSet;
using google::protobuf::Descriptor;
using google::protobuf::DynamicMessageFactory;
using google::protobuf::Message;
using google::protobuf::Base64Unescape;
using google::protobuf::DescriptorPool;

/**
 * Command line options for this program.
 */
struct Options {
  /**
   * If verbose > 0, print debugging information to stderr.
   */
  int verbose;

  /**
   * True means that we are converting protobuf to JSON. False means we are
   * converting JSON to Protobuf.
   */
  bool toJson;

  /**
   * A list of directories to look for proto files in when resolving imports.
   * Imports must be rooted at one of these directories.
   * If omitted, the current working directory is assumed.
   */
  std::vector<const char*> protoPaths;

  /**
   * A list of root proto files to search over, as relative paths from the
   * protoPaths. These files and their dependencies will be searched over. If
   * not provided, all proto files in the protoPaths will be searched.
   */
  std::vector<const char*> protoFiles;

  /**
   * A list of proto descriptor sets generated by invoking `protoc` with
   * `--descriptor_set_out`.
   */
  std::vector<const char*> descriptorSets;

  /**
   * The name of the protobuf message we wish to convert to or from json.
   */
  const char* messageName;

  /**
   * If given, this is either literal data provided on the command line or a
   * filename.
   * If not given, our converter reads from stdin.
   */
  const char* data;
};

/**
 * Print usage to stderr and exit.
 */
static void usage(const char* progName, bool isHelp = 0) {
  fprintf(stderr,
       "Usage: %s [--proto_path=PATH...] [--verbose] <message_name> [data] \n"
       "\n"
       "  There are two names for this tool:\n"
       "    JsonToProto will assume the input is JSON and write binary protobuf to stdout.\n"
       "    ProtoToJson will assume the input is Proto and write JSON to stdout.\n"
       "\n"
       "  Arguments:\n"
       "    -IPATH, --proto_path=PATH   Specify the directory in which to search for\n"
       "                                imports.  May be specified multiple times;\n"
       "                                directories will be searched in order.  If not\n"
       "                                given, the current working directory is used.\n"
       "    -P file, --proto_file=PATH  Specify the path of the file defining the protobuf \n"
       "                                message we wish to convert to or from json, relative to\n"
       "                                any of the proto_path options. This is an optimization.\n"
       "                                This option is only consumed when `--proto_path` is given.\n"
       "    --descriptor_set <DS>       Specify one or more descriptor sets produced by running\n"
       "                                `protoc` with `--descriptor_set_out`. This option is\n"
       "                                ignored if `--proto_path` is given.\n"
       "    --verbose                   When given, debug output will be printed to stderr.\n"
       "    message_name                The name of the protobuf message we wish to convert to or from json.\n"
       "    data                        If not provided as an argument, input is read from stdin.\n"
       "                                If provided, it is interpreted as a filename if it is prefixed\n"
       "                                with the `@` symbol, and otherwise as literal data.\n"
       "                                For json to proto, literal data is interpreted as JSON.\n"
       "                                For proto to json, literal data is interpreted as base64\n"
       "                                encoded protobuf.\n"
       "    --help                      Show this message.\n",
      progName);
  if (isHelp) {
    exit(0);
  }
  exit(1);
}

/**
 * Attempt to parse command line arguments. Print out an informative message to
 * stderr if an error is encountered.
 */
Options parseArguments(int argc, char** argv) {
  // Ensure that we have enough arguments for all required arguments.
  if (argc < 2 ) {
    fprintf(stderr, "Not enough arguments.\n");
    usage(argv[0]);
  }

  // Retain the orignal argc and argv for verbose printing.
  int oldArgc = argc;
  char** oldArgv = argv;

  // Initialize default options
  Options options;
  options.verbose = 0;
  options.toJson = false;
  options.messageName = NULL;
  options.data = NULL;

  // Get the name without the path
  const char* exeName = strrchr(argv[0], '/');
  if (exeName) {
    exeName++;
  } else {
    exeName = argv[0];
  }
  if (strcasecmp("JsonToProto", exeName) == 0) {
    options.toJson = false;
  } else if (strcasecmp("ProtoToJson", exeName) == 0) {
    options.toJson = true;
  } else {
    fprintf(stderr, "Please invoke this binary as JsonToProto or ProtoToJson.\n");
    usage(oldArgv[0]);
  }

  static struct option long_options[] =
    {
      /* These options set a flag. */
      {"verbose", no_argument, &options.verbose, 1},
      {"proto_path", required_argument, 0, 'I'},
      {"proto_file", required_argument, 0, 'P'},
      {"descriptor_set", required_argument, 0, 'D'},
      {"help", no_argument, NULL, 'h'},
      {0, 0, 0, 0}
    };
  while (1) {
    int option_index = 0;
    int c;
    c = getopt_long(argc, argv, "hI:P:D:", long_options, &option_index);
    if (c == -1) {
      break;
    }
    switch (c) {
      // Do nothing if just set verbose flag
      case 0:
        break;
      case 'I':
        options.protoPaths.push_back(optarg);
        break;
      case 'P':
        options.protoFiles.push_back(optarg);
        break;
      case 'D':
        options.descriptorSets.push_back(optarg);
        break;
      case '?':
        /* getopt_long already printed an error message. */
        break;
      case 'h':
        usage(oldArgv[0], 1);
      default:
        usage(oldArgv[0]);
    }
  }
  argc -= optind;
  argv += optind;

  // Make the protoPaths the current directory if there are neither descriptor
  // sets nor protoPaths specified on the command line.
  if (options.protoPaths.empty() && options.descriptorSets.empty()) {
    options.protoPaths.push_back(".");
  }

  // Parse mandatory arguments and optional positional argument.
  if (argc < 1) {
    fprintf(stderr, "Missing mandatory positional arguments.\n");
    usage(oldArgv[0]);
  }

  options.messageName = argv[0];

  if (argc > 1) {
    options.data = argv[1];
  }

  // Print options output to debug option parsing.
  if (options.verbose) {
    fprintf(stderr, "argc: %d\nargv:\n", oldArgc);
    for (int i = 0; i < oldArgc; i++) {
      fprintf(stderr, "\t%d: %s\n", i, oldArgv[i]);
    }

    fprintf(stderr, "Options:\n");
    fprintf(stderr, "\tToJson: %d\n", options.toJson);
    fprintf(stderr, "\tverbose: %d\n", options.verbose);
    fprintf(stderr, "\tproto_path:\n");
    for (int i = 0; i < options.protoPaths.size(); i++) {
      fprintf(stderr, "\t\t%s\n", options.protoPaths[i]);
    }
    fprintf(stderr, "\tproto_file:\n");
    for (int i = 0; i < options.protoFiles.size(); i++) {
      fprintf(stderr, "\t\t%s\n", options.protoFiles[i]);
    }
    fprintf(stderr, "\tdescriptor_set:\n");
    for (int i = 0; i < options.descriptorSets.size(); i++) {
      fprintf(stderr, "\t\t%s\n", options.descriptorSets[i]);
    }

    fprintf(stderr, "\tmessage_name: %s\n", options.messageName);
    fprintf(stderr, "\tdata: %s\n", options.data);
  }
  return options;
}

/**
 * Read all the bytes of a string.
 */
std::string readFile(std::string filename) {
  constexpr auto read_size = std::size_t(4096);
  auto stream = std::ifstream(filename);
  stream.exceptions(std::ios_base::badbit);

  if (not stream) {
    throw std::ios_base::failure("file does not exist");
  }

  auto out = std::string();
  auto buf = std::string(read_size, '\0');
  while (stream.read(& buf[0], read_size)) {
    out.append(buf, 0, stream.gcount());
  }
  out.append(buf, 0, stream.gcount());
  return out;
}
/**
 * Add the message types in a file descriptor to a map of fully qualified
 * message names to descriptors.
 */
void importFileDescriptor(const FileDescriptor* fd,
    std::map<std::string, const Descriptor*>& messageDescriptors) {
  std::string packageName = fd->package();
  if (!packageName.empty()) {
    packageName += ".";
  }
  for (int i = 0; i < fd->message_type_count(); i++) {
    const Descriptor* messageDescriptor = fd->message_type(i);
    std::string messageDescriptorFullName = packageName + messageDescriptor->name();
    messageDescriptors[messageDescriptorFullName] = messageDescriptor;

    // Add nested messages with a BFS.
    std::queue<std::string> q;
    q.push(messageDescriptorFullName);
    while (!q.empty()) {
      std::string messageName = q.front();
      q.pop();
      const Descriptor* descriptor = messageDescriptors[messageName];
      for (int j = 0; j < descriptor->nested_type_count(); j++) {
        std::string childMessageName = messageName;
        childMessageName += ".";
        childMessageName += descriptor->nested_type(j)->name();
        messageDescriptors[childMessageName] = descriptor->nested_type(j);
        q.push(childMessageName);
      }
    }
  }
}

/**
 * Convert between JSON and protobuf.
 *
 * The behavior of this process will vary depending on the name that the
 * executable is invoked under.
 *   JsonToProto will assume the input is JSON and write binary protobuf to
 *   stdout.
 *   ProtoToJson will assume the input is Proto and write JSON to stdout.
 *
 * If [data] is not provided as an argument, input is read from stdin.
 *
 * If [data] is provided, it is interpreted as a filename if it is prefixed
 * with the `@` symbol, and otherwise as literal data.
 *   For ToProto, literal data is interpreted as JSON.
 *   For ToJson, literal data is interpreted as base64 encoded protobuf.
 */
int main(int argc, char** argv){
  // Check if options need to be prefixed from an env var.
  char* prefixedArguments = getenv("PROTO_JSON_OPTIONS");
  while (prefixedArguments) {
    wordexp_t word_expansion;

    int error = wordexp(prefixedArguments, &word_expansion, 0);
    if (error) {
      fprintf(stderr, "Failed to expand PROTO_JSON_OPTIONS '%s'. Ignoring it.", prefixedArguments);
      break;
    }
    int oldArgc = argc;
    char** oldArgv = argv;

    argc += word_expansion.we_wordc;
    argv = (char**) malloc(argc * sizeof(char*));
    argv[0] = oldArgv[0];
    // Copy the new arguments in front.
    for (int i = 0; i < word_expansion.we_wordc; i++) {
      argv[i+1] = strdup(word_expansion.we_wordv[i]);
    }

    // Copy the original arguments afterwards.
    for (int i = 1; i < oldArgc; i++) {
      argv[i + word_expansion.we_wordc] = oldArgv[i];
    }

    wordfree(&word_expansion);
    break;
  }

  Options options = parseArguments(argc, argv);

  // Construct the list of all proto file names we will use.
  std::vector<std::string> allFilenames;
  if (options.protoFiles.empty()) {
    // If the user did not specify proto files, then parse all file names from
    // the filesystem, since SourceTreeDescriptorDatabase does not appear to
    // implement FindAllFileNames.
    for (const char* importPath: options.protoPaths) {
      std::filesystem::path protoDirPath(importPath);
      std::filesystem::recursive_directory_iterator it(protoDirPath,
          std::filesystem::directory_options::follow_directory_symlink);
      for (const std::filesystem::directory_entry& dir_entry : it) {
        if (dir_entry.is_regular_file()) {
          auto path = dir_entry.path();
          if (path.extension().string() == ".proto") {
            // We need to strip out the first directory because the
            // recursive_directory_iterator includes the name of the import path
            // and the proto tools assume paths relative to the import path.
            allFilenames.push_back(path.lexically_relative(protoDirPath));
          }
        }
      }
    }
  } else {
    // The user specified proto files so just use those. Note that we assume
    // the user has given filenames relative to at least one of the import
    // paths, just like protoc.
    for (const char* filename : options.protoFiles) {
      allFilenames.push_back(filename);
    }
  }

  // Construct source tree and importer for source files.
  DiskSourceTree diskSourceTree;
  for (const char* importPath: options.protoPaths) {
    diskSourceTree.MapPath("", importPath);
  }

  class ErrorReporter: public MultiFileErrorCollector {
    public:
      virtual void AddError(const std::string & filename, int line, int column, const std::string & message) {
        std::cerr << "Error occured for " << filename << ":" << line << ":" <<
          column  << " " << message << std::endl;
      }
  } errorReporter;

  Importer importer(&diskSourceTree, &errorReporter);

  // Map of fully-qualified message names to packages.
  std::map<std::string, const Descriptor*> messageDescriptors;

  // Add message descriptors from proto files.
  for (std::string& filename: allFilenames) {
    const FileDescriptor* fd = importer.Import(filename);
    if (fd == NULL) {
      std::cerr << "Encoutered errors when importing file " << filename << std::endl;
      exit(1);
    }
    importFileDescriptor(fd, messageDescriptors);
  }

  // Add message descriptors from descriptor sets iff the are no protoPaths
  // specified. The behavior of ignoring descriptorSets when protoPaths is
  // specified is consistent with `protoc` behavior, and also removes any
  // potential risks associated with using descriptors from multiple pools at
  // the same time.
  DescriptorPool pool;
  if (options.protoPaths.empty()) {
    for (int i = 0; i < options.descriptorSets.size(); i++) {
      std::string descriptorSetBytes = readFile(options.descriptorSets[i]);
      FileDescriptorSet proto_files;
      proto_files.ParseFromString(descriptorSetBytes);
      for (int j = 0; j < proto_files.file_size(); j++) {
        const FileDescriptor* fd = pool.BuildFile(proto_files.file(j));
        importFileDescriptor(fd, messageDescriptors);
      }
    }
  }

  // Search for a descriptor based on the message name.
  std::vector<std::string> matches;
  for (auto message: messageDescriptors) {
    // Exact match has higher priority than partial matches.
    // This is necessary for correct behavior when one proto's full name is an
    // exact subset of another.
    if (message.first == options.messageName) {
      matches.clear();
      matches.push_back(message.first);
      break;
    }

    if (message.first.find(options.messageName) != std::string::npos) {
      matches.push_back(message.first);
    }
  }

  if (matches.size() == 0) {
    std::cerr << "Failed to find message " << options.messageName << std::endl;
    return 1;
  }
  if (matches.size() > 1) {
    std::cerr << "Found multiple messages matching " << options.messageName << std::endl;
    for (std::string& match: matches) {
      std::cerr << '\t' << match << std::endl;
    }
    return 1;
  }

  // Construct a message from the given descriptor. This name must exclude the
  // package.
  const Descriptor* messageDescriptor = messageDescriptors[matches[0]];
  if (options.verbose) {
    std::cerr << messageDescriptor->DebugString() << std::endl;
  }
  DynamicMessageFactory dynamicMessageFactory;
  Message* message = dynamicMessageFactory.GetPrototype(messageDescriptor)->New();

  Status conversionStatus;
  if (options.toJson) {
    bool parseSuccessful;
    // Converting protobuf to JSON
    if (options.data) {
      if (options.data[0] == '@') {
        // Read binary from file
        FILE* protoFile = fopen(options.data + 1, "rb");
        parseSuccessful = message->ParseFromFileDescriptor(fileno(protoFile));
        fclose(protoFile);
      } else {
        // Interpret as base64 encoded binary protobuf.
        std::string binaryProto;
        if (!Base64Unescape(options.data, &binaryProto)) {
          std::cout << "Failed to decode base64." << std::endl;
          exit(1);
        }
        parseSuccessful = message->ParseFromString(binaryProto);
      }
    } else {
      // Read binary from stdin.
      parseSuccessful = message->ParseFromIstream(&std::cin);
    }
    if (!parseSuccessful) {
      std::cout << "Failed to parse protobuf message." << std::endl;
      exit(1);
    }

    std::string jsonOutput;
    JsonPrintOptions printOptions;
    printOptions.preserve_proto_field_names = true;
    printOptions.always_print_primitive_fields = false;
    printOptions.add_whitespace = true;
    printOptions.always_print_enums_as_ints = false;
    conversionStatus = MessageToJsonString(*message, &jsonOutput, printOptions);
    if (!conversionStatus.ok()) {
      std::cout << conversionStatus << std::endl;
      exit(1);
    }
    std::cout << jsonOutput << std::endl;
  } else {
    //  Converting JSON to protobuf
    if (options.data) {
      if (options.data[0] == '@') {
        // Read JSON from file
        std::ifstream jsonFile(options.data + 1);
        std::stringstream buffer;
        buffer << jsonFile.rdbuf();
        conversionStatus = JsonStringToMessage(buffer.str(), message);
      } else {
        // Interpret as JSON Text
        conversionStatus = JsonStringToMessage(options.data, message);
      }
    } else {
      // Read JSON from stdin.
      std::stringstream buffer;
      buffer << std::cin.rdbuf();
      conversionStatus = JsonStringToMessage(buffer.str(), message);
    }
    if (!conversionStatus.ok()) {
      std::cout << conversionStatus << std::endl;
      exit(1);
    }
    message->SerializeToOstream(&std::cout);
  }
  // Give a hint about a different command that would run faster next time.
  if (options.protoFiles.empty() && !options.protoPaths.empty()) {
    fprintf(stderr, "For faster conversion of this message, add \e[1m-P %s\e[m to the command invocation.\n",
       messageDescriptor->file()->name().c_str());
  }
}