EQStream abstraction layer

libs/format/support/download.py

@@ -0,0 +1,46 @@
+# A file downloader.
+
+import contextlib, os, tempfile, timer, urllib2, urlparse
+
+class Downloader:
+  def __init__(self, dir=None):
+    self.dir = dir
+
+  # Downloads a file and removes it when exiting a block.
+  # Usage:
+  #  d = Downloader()
+  #  with d.download(url) as f:
+  #    use_file(f)
+  def download(self, url, cookie=None):
+    suffix = os.path.splitext(urlparse.urlsplit(url)[2])[1]
+    fd, filename = tempfile.mkstemp(suffix=suffix, dir=self.dir)
+    os.close(fd)
+    with timer.print_time('Downloading', url, 'to', filename):
+      opener = urllib2.build_opener()
+      if cookie:
+        opener.addheaders.append(('Cookie', cookie))
+      num_tries = 2
+      for i in range(num_tries):
+        try:
+          f = opener.open(url)
+        except urllib2.URLError, e:
+          print('Failed to open url', url)
+          continue
+        length = f.headers.get('content-length')
+        if not length:
+          print('Failed to get content-length')
+          continue
+        length = int(length)
+        with open(filename, 'wb') as out:
+          count = 0
+          while count < length:
+            data = f.read(1024 * 1024)
+            count += len(data)
+            out.write(data)
+    @contextlib.contextmanager
+    def remove(filename):
+      try:
+        yield filename
+      finally:
+        os.remove(filename)
+    return remove(filename)