client.py fetching urls: stream to disk using shutil.copyfileobj

dstndstn · Sep 26, 2023 · c4b1114 · c4b1114
1 parent 2536110
commit c4b1114
Showing 1 changed file with 4 additions and 5 deletions.
diff --git a/net/client/client.py b/net/client/client.py
@@ -10,6 +10,7 @@
 import sys
 import time
 import base64
+import shutil
 
 try:
     # py3
@@ -439,11 +440,9 @@ def jobs_by_tag(self, tag, exact):
 
         for url,fn in retrieveurls:
             print('Retrieving file from', url, 'to', fn)
-            f = urlopen(url)
-            txt = f.read()
-            w = open(fn, 'wb')
-            w.write(txt)
-            w.close()
+            with urlopen(url) as r:
+                with open(fn, 'wb') as w:
+                    shutil.copyfileobj(r, w)
             print('Wrote to', fn)
 
         if opt.annotate: