-
Notifications
You must be signed in to change notification settings - Fork 1
/
pput.js
executable file
·122 lines (111 loc) · 2.99 KB
/
pput.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
#!/usr/bin/env node --harmony
var fs = require('fs'),
knox = require('knox'),
path = require('path'),
parseArgs = require('minimist');
function *walks(files, prefix) {
for (var i in files) {
yield* walk(files[i], path.dirname(files[i]), prefix);
}
}
function *walk(file, dirname, prefix) {
var stat = fs.statSync(file);
if (stat.isDirectory()) {
var files = fs.readdirSync(file);
for (var i in files) {
yield* walk(path.join(file, files[i]), dirname, prefix);
}
} else {
var key = file;
if (key.indexOf(dirname) === 0) {
key = key.substring(dirname.length + 1);
}
if (prefix) {
key = path.join(prefix, key);
}
if (folder) {
key = path.join(folder, key);
}
yield {
file: file,
key: key,
stat: stat
};
}
}
var argv = parseArgs(process.argv.slice(2));
var files = walks(argv._, argv.prefix || '');
var s3 = knox.createClient({
key: argv.key || process.env.AWS_KEY,
secret: argv.secret || process.env.AWS_SECRET,
bucket: argv.bucket,
region: argv.region || 'eu-west-1',
secure: false,
agent: false
});
var workers = argv.workers || 100,
folder = argv.folder,
done = 0,
bytesSent = 0,
filesSent = 0,
start = new Date();
function putNext(task) {
if (!task) {
try {
var next = files.next();
if (next.done) {
return done++;
}
task = next.value;
} catch (x) {
return done++;
}
}
fs.readFile(task.file, function (err, data) {
if (err) {
throw err;
}
var req = s3.put(task.key, {
'Content-Length': data.length,
'x-amz-acl': 'public-read'
});
req.on('error', function (err) {
console.error(err.message);
putNext(task);
});
req.on('response', function(res) {
res.resume();
if (res.statusCode !== 200) {
console.error("%d - Retrying %s", req.statusCode, task.file);
putNext(task);
} else {
filesSent++;
bytesSent += task.stat.size;
putNext();
}
});
req.end(data);
});
}
function getSpeed() {
var seconds = (new Date() - start) / 1000,
bytesPerSec = bytesSent / seconds;
if (bytesPerSec > 1e6) {
return (bytesPerSec / 1e6).toFixed(1) + ' MB/s';
} else if (bytesPerSec > 1e3) {
return (bytesPerSec / 1e3).toFixed(1) + ' KB/s';
}
return bytesPerSec.toFixed(1) + ' B/s';
}
for (var i = 0; i < workers; ++i) {
putNext();
}
(function printProgress() {
process.stdout.clearLine();
process.stdout.write('Files sent: ' + filesSent + '. Average speed: ' + getSpeed() + '\r');
if (done < workers) {
setTimeout(printProgress, 1000);
} else {
console.log('\nDone!');
}
}());