// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2008 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// BOINC is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License
// along with BOINC.  If not, see <http://www.gnu.org/licenses/>.

// vmwrapper.C
// VMWare wrapper program - lets you use BOINC to drive a VMWare Server
// guest OS

#include <stdio.h>
#include <vector>
#include <string>
#ifdef _WIN32
#include "boinc_win.h"
#include "win_util.h"
#else
#include <sys/wait.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "procinfo.h"
#endif

#include "boinc_api.h"
#include "diagnostics.h"
#include "filesys.h"
#include "parse.h"
#include "str_util.h"
#include "util.h"
#include "error_numbers.h"

#include "vmware-vix/vix.h"

#define JOB_FILENAME "job.xml"
#define CHECKPOINT_FILENAME "checkpoint.txt"

#define POLL_PERIOD 1.0

using std::vector;
using std::string;

struct TASK
{
	string application;
	string stdin_filename;
	string stdout_filename;
	string stderr_filename;

	string vm;
	string innerjob;
	string datadir;
	string partial_credit;
	string snapshots;
	string user;
	string password;

	vector < string > data;
	vector < string > output;

	string checkpoint_filename;
	// name of task's checkpoint file, if any
	double checkpoint_cpu_time;
	// CPU time at last checkpoint
	string command_line;
	double weight;
	// contribution of this task to overall fraction done
	double final_cpu_time;
	double starting_cpu;
	// how much CPU time was used by tasks before this in the job file
	bool	suspended;
	double wall_cpu_time;
	// for estimating CPU time on Win98/ME and Mac


	#ifdef _WIN32

	HANDLE pid_handle;
	DWORD pid;
	HANDLE thread_handle;
	struct _stat last_stat;				 // mod time of checkpoint file

	#else

	int pid;
	struct stat last_stat;

	#endif


	bool stat_first;
	int parse (XML_PARSER &);
	bool poll (int &status);
	int run (int argc, char **argv);
	void kill ();
	void stop ();
	void resume ();
	double cpu_time ();

	inline bool has_checkpointed ()
	{
		bool changed = false;

		if (checkpoint_filename.size () == 0)
			return false;

		struct stat new_stat;

		int retval = stat (checkpoint_filename.c_str (), &new_stat);

		if (retval)
			return false;

		if (!stat_first && new_stat.st_mtime != last_stat.st_mtime)
		{
			changed = true;
		}

		stat_first = false;

		last_stat.st_mtime = new_stat.st_mtime;
		return changed;
	}
};

vector < TASK > tasks;
APP_INIT_DATA aid;
bool graphics = false;

int TASK::parse (XML_PARSER & xp)
{

	string this_data, this_output;
	char tag[1024], buf[8192], buf2[8192];

	bool is_tag;

	weight = 1;
	final_cpu_time = 0;
	stat_first = true;
	while (!xp.get (tag, sizeof (tag), is_tag))
	{
		if (!is_tag)
		{
			fprintf (stderr, "SCHED_CONFIG::parse(): unexpected text %s\n",
				tag);
			continue;
		}
		if (!strcmp (tag, "/task"))
		{
			return 0;
		}
		else if (xp.parse_string (tag, "application", application))
			continue;

		else if (xp.parse_string (tag, "innerjob", innerjob))
			continue;
		else if (xp.parse_string (tag, "vm", vm))
			continue;
		else if (xp.parse_string (tag, "datadir", datadir))
			continue;
		else if (xp.parse_string (tag, "partial_credit", partial_credit))
			continue;
		else if (xp.parse_string (tag, "snapshots", snapshots))
			continue;


		else if (xp.parse_string (tag, "data", this_data)) {
			data.push_back(this_data);
			continue;
		}

		else if (xp.parse_string (tag, "output", this_output)) {
			output.push_back(this_output);
			continue;
		}

		else if (xp.parse_string (tag, "user", user))
			continue;
		else if (xp.parse_string (tag, "password", password))
			continue;

		else if (xp.parse_string (tag, "stdin_filename", stdin_filename))
			continue;
		else if (xp.parse_string (tag, "stdout_filename", stdout_filename))
			continue;
		else if (xp.parse_string (tag, "stderr_filename", stderr_filename))
			continue;
		else if (xp.parse_str (tag, "command_line", buf, sizeof (buf)))
		{
			while (1)
			{
				char *p = strstr (buf, "$PROJECT_DIR");

				if (!p)
					break;
				strcpy (buf2, p + strlen ("$PROJECT_DIR"));
				strcpy (p, aid.project_dir);
				strcat (p, buf2);
			}
			command_line = buf;
			continue;
		}
		else if (xp.
			parse_string (tag, "checkpoint_filename",
			checkpoint_filename))
			continue;
		else if (xp.parse_double (tag, "weight", weight))
			continue;
	}
	return ERR_XML_PARSE;
}


int parse_job_file ()
{
	MIOFILE mf;
	char tag[1024], buf[256];

	bool is_tag;

	boinc_resolve_filename (JOB_FILENAME, buf, 1024);
	FILE *f = boinc_fopen (buf, "r");

	if (!f)
	{
		fprintf (stderr, "can't open job file %s\n", buf);
		return ERR_FOPEN;
	}
	mf.init_file (f);
	XML_PARSER xp (&mf);

	if (!xp.parse_start ("job_desc"))
		return ERR_XML_PARSE;
	while (!xp.get (tag, sizeof (tag), is_tag))
	{
		if (!is_tag)
		{
			fprintf (stderr, "SCHED_CONFIG::parse(): unexpected text %s\n",
				tag);
			continue;
		}
		if (!strcmp (tag, "/job_desc"))
		{
			fclose (f);
			return 0;
		}
		if (!strcmp (tag, "task"))
		{
			TASK task;
			int retval = task.parse (xp);

			if (!retval)
			{
				tasks.push_back (task);
			}
		}
	}
	fclose (f);
	return ERR_XML_PARSE;
}


#ifdef _WIN32
// CreateProcess() takes HANDLEs for the stdin/stdout.
// We need to use CreateFile() to get them.  Ugh.
//
HANDLE
win_fopen (const char *path, const char *mode)
{
	SECURITY_ATTRIBUTES sa;
	memset (&sa, 0, sizeof (sa));
	sa.nLength = sizeof (sa);
	sa.bInheritHandle = TRUE;

	if (!strcmp (mode, "r"))
	{
		return CreateFile (path,
			GENERIC_READ,
			FILE_SHARE_READ, &sa, OPEN_EXISTING, 0, 0);
	}
	else if (!strcmp (mode, "w"))
	{
		return CreateFile (path,
			GENERIC_WRITE,
			FILE_SHARE_WRITE, &sa, OPEN_ALWAYS, 0, 0);
	}
	else if (!strcmp (mode, "a"))
	{
		HANDLE
			hAppend = CreateFile (path,
			GENERIC_WRITE,
			FILE_SHARE_WRITE, &sa, OPEN_ALWAYS, 0, 0);
		SetFilePointer (hAppend, 0, NULL, FILE_END);
		return hAppend;
	}
	else
	{
		return 0;
	}
}
#endif

void slash_to_backslash (char *p)
{
	while (1)
	{
		char *q = strchr (p, '/');

		if (!q)
			break;
		*q = '\\';
	}
}


int TASK::run (int argct, char **argvt)
{
	string stdout_path, stdin_path, stderr_path;
	char app_path[1024], buf[256];

	strcpy (buf, application.c_str ());
	char *p = strstr (buf, "$PROJECT_DIR");

	if (p)
	{
		p += strlen ("$PROJECT_DIR");
		sprintf (app_path, "%s%s", aid.project_dir, p);
	}
	else
	{
		boinc_resolve_filename (buf, app_path, sizeof (app_path));
	}

	// Append wrapper's command-line arguments to those in the job file.
	//
	for (int i = 1; i < argct; i++)
	{
		command_line += argvt[i];
		if ((i + 1) < argct)
		{
			command_line += string (" ");
		}
	}

	fprintf (stderr, "wrapper: running %s (%s)\n",
		app_path, command_line.c_str ());

	#ifdef _WIN32
	PROCESS_INFORMATION process_info;
	STARTUPINFO startup_info;
	string command;

	slash_to_backslash (app_path);
	memset (&process_info, 0, sizeof (process_info));
	memset (&startup_info, 0, sizeof (startup_info));
	command = string ("\"") + app_path + string ("\" ") + command_line;

	// pass std handles to app
	//
	startup_info.dwFlags = STARTF_USESTDHANDLES;
	if (stdout_filename != "")
	{
		boinc_resolve_filename_s (stdout_filename.c_str (), stdout_path);
		startup_info.hStdOutput = win_fopen (stdout_path.c_str (), "a");
	}
	if (stdin_filename != "")
	{
		boinc_resolve_filename_s (stdin_filename.c_str (), stdin_path);
		startup_info.hStdInput = win_fopen (stdin_path.c_str (), "r");
	}
	if (stderr_filename != "")
	{
		boinc_resolve_filename_s (stderr_filename.c_str (), stderr_path);
		startup_info.hStdError = win_fopen (stderr_path.c_str (), "a");
	}
	else
	{
		startup_info.hStdError = win_fopen (STDERR_FILE, "a");
	}

								 // bInheritHandles
	if (!CreateProcess (app_path, (LPSTR) command.c_str (), NULL, NULL, TRUE,
		CREATE_NO_WINDOW | IDLE_PRIORITY_CLASS,
		NULL, NULL, &startup_info, &process_info))
	{
		return ERR_EXEC;
	}
	pid_handle = process_info.hProcess;
	pid = process_info.dwProcessId;
	thread_handle = process_info.hThread;
	SetThreadPriority (thread_handle, THREAD_PRIORITY_IDLE);
	#else
	int retval, argc;
	char progname[256];
	char *argv[256];
	char arglist[4096];
	FILE *stdout_file;
	FILE *stdin_file;
	FILE *stderr_file;

	pid = fork ();
	if (pid == -1)
	{
		boinc_finish (ERR_FORK);
	}
	if (pid == 0)
	{
		// we're in the child process here
		//
		// open stdout, stdin if file names are given
		// NOTE: if the application is restartable,
		// we should deal with atomicity somehow
		//
		if (stdout_filename != "")
		{
			boinc_resolve_filename_s (stdout_filename.c_str (), stdout_path);
			stdout_file = freopen (stdout_path.c_str (), "a", stdout);
			if (!stdout_file)
				return ERR_FOPEN;
		}
		if (stdin_filename != "")
		{
			boinc_resolve_filename_s (stdin_filename.c_str (), stdin_path);
			stdin_file = freopen (stdin_path.c_str (), "r", stdin);
			if (!stdin_file)
				return ERR_FOPEN;
		}
		if (stderr_filename != "")
		{
			boinc_resolve_filename_s (stderr_filename.c_str (), stderr_path);
			stderr_file = freopen (stderr_path.c_str (), "a", stderr);
			if (!stderr_file)
				return ERR_FOPEN;
		}
		// construct argv
		// TODO: use malloc instead of stack var
		//
		argv[0] = app_path;
		strlcpy (arglist, command_line.c_str (), sizeof (arglist));
		argc = parse_command_line (arglist, argv + 1);
		setpriority (PRIO_PROCESS, 0, PROCESS_IDLE_PRIORITY);
		retval = execv (app_path, argv);
		exit (ERR_EXEC);
	}
	#endif
	wall_cpu_time = 0;
	suspended = false;
	return 0;
}


bool TASK::poll (int &status)
{
	if (!suspended)
		wall_cpu_time += POLL_PERIOD;
	#ifdef _WIN32
	unsigned long
		exit_code;

	if (GetExitCodeProcess (pid_handle, &exit_code))
	{
		if (exit_code != STILL_ACTIVE)
		{
			status = exit_code;
			final_cpu_time = cpu_time ();
			return true;
		}
	}
	#else
	int
		wpid,
		stat;
	struct rusage
		ru;

	wpid = wait4 (pid, &status, WNOHANG, &ru);
	if (wpid)
	{
		final_cpu_time =
			(float) ru.ru_utime.tv_sec + ((float) ru.ru_utime.tv_usec) / 1e+6;
		return true;
	}
	#endif
	return false;
}


void TASK::kill ()
{
	#ifdef _WIN32
	TerminateProcess (pid_handle, -1);
	#else
	::kill (pid, SIGKILL);
	#endif
}


void TASK::stop ()
{
	suspended = true;
}


void TASK::resume ()
{
	suspended = false;
}




double TASK::cpu_time ()
{
	#ifdef _WIN32
	FILETIME creation_time, exit_time, kernel_time, user_time;
	ULARGE_INTEGER tKernel, tUser;
	LONGLONG totTime;

	int retval = GetProcessTimes (pid_handle, &creation_time, &exit_time,
		&kernel_time,
		&user_time);

	if (retval == 0)
	{
		return wall_cpu_time;
	}

	tKernel.LowPart = kernel_time.dwLowDateTime;
	tKernel.HighPart = kernel_time.dwHighDateTime;
	tUser.LowPart = user_time.dwLowDateTime;
	tUser.HighPart = user_time.dwHighDateTime;
	totTime = tKernel.QuadPart + tUser.QuadPart;

	return totTime / 1.e7;
	#elif defined(__APPLE__)
	// There's no easy way to get another process's CPU time in Mac OS X
	//
	return wall_cpu_time;
	#else
	return linux_cpu_time (pid);
	#endif
}




// Support for multiple tasks.
// We keep a checkpoint file that says how many tasks we've completed
// and how much CPU time has been used so far
//
void write_checkpoint (int ntasks, double cpu)
{
	FILE *f = fopen (CHECKPOINT_FILENAME, "w");

	if (!f)
		return;
	fprintf (f, "%d %f\n", ntasks, cpu);
	fclose (f);
}


void read_checkpoint (int &ntasks, double &cpu)
{
	int nt;
	double c;

	ntasks = 0;
	cpu = 0;
	FILE *f = fopen (CHECKPOINT_FILENAME, "r");

	if (!f)
		return;
	int n = fscanf (f, "%d %lf", &nt, &c);

	fclose (f);
	if (n != 2)
		return;
	ntasks = nt;
	cpu = c;
}



void check_vm_result(VixError err, VixHandle hostHandle, string errorString)
{
	if (VIX_OK != err)
	{
		fprintf (stderr, "\n\n Error: %s\n",errorString.c_str());
		fprintf (stderr, "Error message: \"%s\"\n", Vix_GetErrorText (err, NULL));
		VixHost_Disconnect (hostHandle);
		boinc_finish(100);
	}
}

int main (int argc, char **argv)
{

	BOINC_OPTIONS options;
	int retval, ntasks;
	unsigned int i;
	double cpu, total_weight = 0, w = 0;

	for (i = 1; i < (unsigned int) argc; i++)
	{
		if (!strcmp (argv[i], "--graphics"))
		{
			graphics = true;
		}
	}

	memset (&options, 0, sizeof (options));
	options.main_program = true;
	options.check_heartbeat = true;
	options.handle_process_control = true;
	if (graphics)
	{
		options.backwards_compatible_graphics = true;
	}

	// boinc_init_options(&options);

	fprintf (stderr, "vmwrapper: starting\n");

	boinc_get_init_data (aid);

	retval = parse_job_file ();
	if (retval)
	{
		fprintf (stderr, "can't parse job file: %d\n", retval);
		boinc_finish (retval);
	}

	read_checkpoint (ntasks, cpu);
	if (ntasks > (int) tasks.size ())
	{
		fprintf (stderr, "Checkpoint file: ntasks %d too large\n", ntasks);
		boinc_finish (1);
	}

	for (i = 0; i < tasks.size (); i++)
	{
		total_weight += tasks[i].weight;
	}




	VixHandle hostHandle = VIX_INVALID_HANDLE;
	VixHandle jobHandle = VIX_INVALID_HANDLE;
	VixError err;
	string vmxFilename;

	jobHandle = VixHost_Connect (VIX_API_VERSION, VIX_SERVICEPROVIDER_VMWARE_SERVER, NULL, 0, NULL,	NULL, 0, VIX_INVALID_HANDLE, NULL, NULL);

	err = VixJob_Wait (jobHandle, VIX_PROPERTY_JOB_RESULT_HANDLE, &hostHandle, VIX_PROPERTY_NONE);

	if (VIX_OK != err)
	{
		fprintf (stderr, "Unable to connect to server!\n");
		fprintf (stderr, "Error message: \"%s\"\n", Vix_GetErrorText (err, NULL));
		VixHost_Disconnect (hostHandle);
		boinc_finish(1);
	}

	fprintf (stderr, "Connected to server.\n");


	Vix_ReleaseHandle (jobHandle);
	jobHandle = VIX_INVALID_HANDLE;


	for (i = 0; i < tasks.size (); i++)
	{

		fprintf(stderr,"Processing task %d.\n", i);

		TASK & task = tasks[i];

		// w += task.weight;

		// this line is to do with cpu time storage
		// if ((int) i < ntasks)
		//	continue;

		double frac_done = w / total_weight;


		fprintf(stderr, "Task %d.\n========\n\n", i+1);


		fprintf(stderr, "VM file root is %s\n", task.vm.c_str ());

		string vm_file_in_1, vm_file_in_2;
		char vm_file_1[1024], vm_file_2[1024];

		string pwd = get_current_dir_name ();


		// VMWare Server 1 requires full path
		vm_file_in_1 = pwd + "/" + task.vm + ".vmx";
		vm_file_in_2 = pwd + "/" + task.vm + ".vmdk";

//		fprintf (stderr, "Input filenames are %s, %s\n",
//			vm_file_in_1.c_str (), vm_file_in_2.c_str ());

		boinc_resolve_filename (vm_file_in_1.c_str (), vm_file_1, 1024);
		boinc_resolve_filename (vm_file_in_2.c_str (), vm_file_2, 1024);

//		fprintf (stderr, "Resolved to %s, %s\n", vm_file_1, vm_file_2);

		if (access (vm_file_1, R_OK))
		{
			fprintf (stderr, "Unable to access VM file %s!\n",vm_file_1);
			boinc_finish (1);
		}

		if (access (vm_file_2, R_OK))
		{
			fprintf (stderr, "Unable to access VM file %s!\n",vm_file_2);
			boinc_finish (1);
		}


		// Register VM
		jobHandle = VixHost_RegisterVM (hostHandle, vm_file_1, NULL, NULL);
		err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
		check_vm_result(err, hostHandle, "Unable to register VM.");
		fprintf (stderr, "Registered virtual machine with the server.\n");

		VixHandle vmHandle = VIX_INVALID_HANDLE;

		// Open VM
		jobHandle = VixVM_Open (hostHandle, vm_file_1, NULL, NULL);
		err = VixJob_Wait (jobHandle, VIX_PROPERTY_JOB_RESULT_HANDLE, &vmHandle, VIX_PROPERTY_NONE);
		check_vm_result(err, hostHandle, "Unable to open VM.");
		fprintf (stderr, "Got handle.\n");

		// Power on VM
		jobHandle = VixVM_PowerOn (vmHandle, 0, VIX_INVALID_HANDLE, NULL, NULL);
		err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
		check_vm_result(err, hostHandle, "Unable to power on VM.");
		fprintf (stderr, "VM is now on (if it wasn't already).\n");

		// Wait for guest OS to start
		jobHandle = VixVM_WaitForToolsInGuest (vmHandle, 0, NULL, NULL);
		err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
		check_vm_result(err, hostHandle, "Did not get confirmation of VMWare Tools.");

		// Log in
		jobHandle = VixVM_LoginInGuest (vmHandle, task.user.c_str(), task.password.c_str(), 0, NULL, NULL);
		err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
		check_vm_result(err, hostHandle, "Unable to log in.");
		fprintf (stderr, "Logged in. Preparing job...\n");


		string guestfile = task.datadir +  "/" + task.innerjob;
		string st_hostfile = pwd + "/" + task.innerjob;
		char hostfile[1024];

		boinc_resolve_filename (st_hostfile.c_str (), hostfile, 1024);

		jobHandle = VixVM_CopyFileFromHostToGuest (vmHandle, hostfile, guestfile.c_str(), 0, VIX_INVALID_HANDLE, NULL, NULL);
		err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
		check_vm_result(err, hostHandle, "Unable to copy executable.");

		Vix_ReleaseHandle (jobHandle);

		// Changing permissions, do we really need to do this?
		jobHandle = VixVM_RunProgramInGuest (vmHandle, "/bin/chmod", ("+x " + guestfile).c_str (), 0, VIX_INVALID_HANDLE, NULL, NULL);	
		err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
		check_vm_result(err, hostHandle, "Unable to change permissions.");

		Vix_ReleaseHandle (jobHandle);


		fprintf (stderr, "Copying in input...\n");

		for(int files=0;files<task.data.size();files++)
		{
			fprintf(stderr,"Processing: %s\n",task.data[files].c_str());

			// should use boinc_resolve_filename here
			jobHandle = VixVM_CopyFileFromHostToGuest (vmHandle, (pwd + "/" + task.data[files]).c_str(), (task.datadir + "/" + task.data[files]).c_str(),
				0, VIX_INVALID_HANDLE,  NULL, NULL);
			err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
			check_vm_result(err, hostHandle, "Error copying in file.");

			Vix_ReleaseHandle (jobHandle);

		}






		// Run the target program.
		fprintf (stderr, "Running main program: ");
		jobHandle = VixVM_RunProgramInGuest (vmHandle, guestfile.c_str (), "", 0, VIX_INVALID_HANDLE, NULL, NULL);

		Bool complete=false;

		task.starting_cpu = cpu;
		task.checkpoint_cpu_time = cpu;


		BOINC_STATUS status;
		VixHandle innerJobHandle;
		VixHandle snapshotHandle;

		while(!complete)
		{
			if(boinc_time_to_checkpoint())
				fprintf(stderr,"Time to checkpoint run here...\n");



			boinc_get_status (&status);

			if (status.no_heartbeat || status.quit_request || status.abort_request)
			{
				VixHost_Disconnect (hostHandle);
				boinc_finish(0);
			}


			// contents of this control structure needs fixing, since suspend doesn't work
			if (status.suspended)
			{
				if (!task.suspended)
				{
					// innerJobHandle = VixVM_Suspend(vmHandle,0,VIX_INVALID_HANDLE,NULL,NULL);
					// err = VixJob_Wait(innerJobHandle, VIX_PROPERTY_JOB_RESULT_HANDLE, &snapshotHandle, VIX_PROPERTY_NONE);
					// check_vm_result(err, hostHandle, "Error pausing VM.");
					task.stop();
				}
			}
			else
			{
				if (task.suspended)
				{
					// innerJobHandle = VixVM_Unpause(vmHandle,0,VIX_INVALID_HANDLE,NULL,NULL);
					// err = VixJob_Wait(innerJobHandle, VIX_PROPERTY_JOB_RESULT_HANDLE, &snapshotHandle, VIX_PROPERTY_NONE);
					// check_vm_result(err, hostHandle, "Error unpausing VM.");
					task.resume ();
				}
			}


			// send_status_message (task, frac_done);
			// fprintf(stderr,"Partial credit check here...\n");

			// double current_cpu_time = task.starting_cpu + task.cpu_time ();

			// if (task.has_checkpointed ())
			// {
			// 	task.checkpoint_cpu_time = current_cpu_time;
			// }

			// not sure how to do this yet
			// boinc_report_app_status (current_cpu_time, task.checkpoint_cpu_time, frac_done);


			boinc_sleep(POLL_PERIOD);


			if(!task.suspended) {
				err = VixJob_CheckCompletion(jobHandle, &complete);
				check_vm_result(err, hostHandle, "Error polling for job completion.");
			}
		}


		cpu += task.final_cpu_time;
		// write_checkpoint here is necessary to keep track of CPU time
		// we either use the time measured, or the time from within
		write_checkpoint (i + 1, cpu);

		err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
		check_vm_result(err, hostHandle, "Error running target program.");
		fprintf (stderr, "done.\n");




		Vix_ReleaseHandle (jobHandle);

	
		fprintf(stderr, "Copying out output...\n");

		for(int files=0;files<task.output.size();files++)
		{
			fprintf(stderr,"Processing: %s\n",task.output[files].c_str());


			// FIXME: need to tell BOINC about these files
			jobHandle = VixVM_CopyFileFromGuestToHost (vmHandle, (task.datadir + "/" + task.output[0]).c_str(), 
				(pwd + "/" + task.output[files]).c_str (), 0, VIX_INVALID_HANDLE, NULL, NULL);
			err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
			check_vm_result(err, hostHandle, "Error copying out file.");

			Vix_ReleaseHandle (jobHandle);

			jobHandle = VixVM_RunProgramInGuest (vmHandle, "/bin/rm",  (task.datadir + "/" + task.output[files]).c_str(), 0, VIX_INVALID_HANDLE, NULL, NULL);	
			err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
			check_vm_result(err, hostHandle, "Error deleting file.");

			Vix_ReleaseHandle (jobHandle);
		}


		// now delete the executable, the output and the data
		fprintf(stderr, "Sanitising virtual machine...\n");

		for(int files=0;files<task.data.size();files++)
		{
			fprintf(stderr,"Removing: %s\n",task.data[files].c_str());

			jobHandle = VixVM_RunProgramInGuest (vmHandle, "/bin/rm",  (task.datadir + "/" + task.data[files]).c_str(), 0, VIX_INVALID_HANDLE, NULL, NULL);	
			err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
			check_vm_result(err, hostHandle, "Error deleting file.");

			Vix_ReleaseHandle (jobHandle);
		}


		fprintf(stderr,"Removing executable.\n");
		jobHandle = VixVM_RunProgramInGuest (vmHandle, "/bin/rm",  guestfile.c_str(), 0, VIX_INVALID_HANDLE, NULL, NULL);	
		err = VixJob_Wait (jobHandle, VIX_PROPERTY_NONE);
		check_vm_result(err, hostHandle, "Error deleting file.");

		Vix_ReleaseHandle (jobHandle);

	

	}

	boinc_finish (0);
}


#ifdef _WIN32

int WINAPI
WinMain (HINSTANCE hInst, HINSTANCE hPrevInst, LPSTR Args, int WinMode)
{
	LPSTR command_line;
	char *argv[100];
	int argc;

	command_line = GetCommandLine ();
	argc = parse_command_line (command_line, argv);
	return main (argc, argv);
}
#endif