ubuntu-server-iso-testing-dev team mailing list archive
-
ubuntu-server-iso-testing-dev team
-
Mailing list archive
-
Message #00074
lp:~jibel/ubuntu-server-iso-testing/download-latest-test-iso.fixes into lp:ubuntu-server-iso-testing
Jean-Baptiste Lallement has proposed merging lp:~jibel/ubuntu-server-iso-testing/download-latest-test-iso.fixes into lp:ubuntu-server-iso-testing.
Requested reviews:
Ubuntu Server Iso Testing Developers (ubuntu-server-iso-testing-dev)
Related bugs:
Bug #740853 in ubuntu-server-iso-testing: "download-latest-test-iso.py fails with KeyError when the image is not listed in SHA256SUMS "
https://bugs.launchpad.net/ubuntu-server-iso-testing/+bug/740853
Bug #751292 in ubuntu-server-iso-testing: "testing continues even if ISO is not available"
https://bugs.launchpad.net/ubuntu-server-iso-testing/+bug/751292
Bug #752321 in ubuntu-server-iso-testing: "download-latest-test-iso.py fails when launchpad is not available"
https://bugs.launchpad.net/ubuntu-server-iso-testing/+bug/752321
For more details, see:
https://code.launchpad.net/~jibel/ubuntu-server-iso-testing/download-latest-test-iso.fixes/+merge/69967
Major diff due to code refactoring and some python style cleaning.
The logic is exactly the same the main differences being:
- Check if both ISO and Checksum files are available on the server before starting sync
- Validation of checksum _after_ download to be sure that what's in cache is really what is expected.
- More exception handling.
--
https://code.launchpad.net/~jibel/ubuntu-server-iso-testing/download-latest-test-iso.fixes/+merge/69967
Your team Ubuntu Server Iso Testing Developers is requested to review the proposed merge of lp:~jibel/ubuntu-server-iso-testing/download-latest-test-iso.fixes into lp:ubuntu-server-iso-testing.
=== modified file 'download-latest-test-iso.py'
--- download-latest-test-iso.py 2011-07-19 22:34:19 +0000
+++ download-latest-test-iso.py 2011-08-01 08:22:29 +0000
@@ -1,209 +1,238 @@
#!/usr/bin/python
-#
-# Copyright (C) 2010, Canonical Ltd (http://www.canonical.com/)
+#
+# Copyright (C) 2010-2011, Canonical Ltd (http://www.canonical.com/)
#
# This file is part of ubuntu-server-iso-testing.
-#
-# ubuntu-server-iso-testing is free software: you can redistribute it
-# and/or modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation, either version 3 of
+#
+# ubuntu-server-iso-testing is free software: you can redistribute it
+# and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation, either version 3 of
# the License, or (at your option) any later version.
-#
-# ubuntu-server-iso-testing is distributed in the hope that it will
-# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+#
+# ubuntu-server-iso-testing is distributed in the hope that it will
+# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
-#
+#
# You should have received a copy of the GNU General Public License
-# along with ubuntu-server-iso-testing. If not, see
+# along with ubuntu-server-iso-testing. If not, see
# <http://www.gnu.org/licenses/>.
-#
-
-from HTMLParser import HTMLParser
+#
from lockfile import FileLock
import os
import optparse
import sys
import logging
-import urllib
-import urlparse
+import urllib2
import subprocess
-import datetime
import hashlib
+from time import sleep
# Default options
DEFAULT_VARIANT = 'server'
-DEFAULT_RELEASE = 'maverick'
+DEFAULT_RELEASE = 'oneiric'
DEFAULT_ARCH = 'all'
-DEFAULT_FLAVOR= 'ubuntu-server'
+DEFAULT_FLAVOR = 'ubuntu-server'
DEFAULT_ISOROOT = os.path.expanduser('~/isos')
+TIMEOUT = 1 # Timeout between retries when the image is not on the server
+RETRY_MAX = 4 # Number of retries
+
# URLS
-BASE_URL="http://cdimage.ubuntu.com";
+BASE_URL = "http://cdimage.ubuntu.com";
# List of 'all' architectures
-ALL_ARCHS =[ 'i386', 'amd64']
+ALL_ARCHS = [ 'i386', 'amd64']
+
+class HeadRequest(urllib2.Request):
+ """ Used to check to retrieve headers of a remote file """
+ def get_method(self):
+ return "HEAD"
# Calculates the sha256 hash for a given file
-def sha256_for_file(f, block_size=2**20):
+def validate_sha256(fname, sha256_hash, block_size=2**20):
+ """ Validate checksum of file fname against sha256 hash """
+ logging.debug("Calculating hash for file '%s'", fname)
sha256 = hashlib.sha256()
- while True:
- data = f.read(block_size)
- if not data:
- break
- sha256.update(data)
- return sha256.hexdigest()
-
-
-# HTML Parser for parsing ISO directory listing
-class ISOHTMLParser(HTMLParser):
- in_a = False
- builds = []
- year = str(datetime.datetime.utcnow().year)
-
- def handle_starttag(self, tag, attrs):
- if tag == 'a':
- self.in_a = True
- else:
- self.in_a = False
-
- def handle_endtag(self, tag):
- if tag == 'a':
- self.in_a = False
- else:
- self.in_a = True
-
- def handle_data(self, data):
- if self.in_a:
- l_stripped = data.strip().rstrip('/')
- if self.year in l_stripped:
- self.builds.append(l_stripped)
-
- def getbuilds(self):
- return self.builds
-
- def getlatestbuild(self):
- self.builds.sort()
- return self.builds.pop()
-
-usage="usage: %prog [options]"
-parser = optparse.OptionParser(usage=usage)
-parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False,
- help="enable debugging")
-parser.add_option("-r", "--release", dest="release", default=DEFAULT_RELEASE,
- help="release of Ubuntu to download (default=%s)" % DEFAULT_RELEASE)
-parser.add_option("-v", "--variant", dest="variant", default=DEFAULT_VARIANT,
- help="variant of Ubuntu to download (default=%s)" % DEFAULT_VARIANT)
-parser.add_option("-a", "--arch", dest="arch", default=DEFAULT_ARCH,
- help="arch of Ubuntu to download (default=%s)" % DEFAULT_ARCH)
-parser.add_option("-i", "--isoroot", dest="isoroot", default=DEFAULT_ISOROOT,
- help="location to store iso images (default=%s)" % DEFAULT_ISOROOT)
-parser.add_option("-f", "--flavor", dest="flavor", default=DEFAULT_FLAVOR,
- help="flavor of Ubuntu to download (default=%s)" % DEFAULT_FLAVOR)
-parser.add_option('-n', '--no-act', default=False, action='store_true',
- dest='no_act', help='compute everything but don\'t actually download')
-parser.add_option('-u', '--url', dest="base_url", default=BASE_URL,
- help="Base URL for cdimage retrieval website (default=%s)" % BASE_URL)
-parser.add_option('-c', '--custom-url', dest="custom_url", default=False,
- help="Full URL for cdimage retrieval website")
-
-(options, args) = parser.parse_args()
-
-if options.debug:
- logging.basicConfig(level=logging.DEBUG)
-else:
- logging.basicConfig(level=logging.INFO)
-
-# Setup URLS
-ALTERNATE_URL= '%s/daily/' % options.base_url
-DESKTOP_URL='%s/daily-live/' % options.base_url
-SERVER_URL='%s/ubuntu-server/daily/' % options.base_url
-
-# Setup Parser to parse CDIMAGE pages.
-parser = ISOHTMLParser()
-l_url = SERVER_URL
-if options.variant == 'server':
- l_url= SERVER_URL
-elif options.variant == 'desktop':
- l_url = DESKTOP_URL
-elif options.variant == 'alternate':
- l_url = ALTERNATE_URL
-
-# override url setting if custom url is provided
-if options.custom_url:
- l_url = options.custom_url
-
-# Check options for architectures
-l_archs = []
-if options.arch == 'all':
- l_archs = ALL_ARCHS
-else:
- l_archs.append(options.arch)
-
-# Retrieve page and parse list
-fh = urllib.urlopen(l_url)
-page = fh.read()
-fh.close()
-
-# Parse the page for information
-parser.feed(page)
-logging.debug(parser.getbuilds())
-l_build = parser.getlatestbuild()
-
-logging.info("Checking/Downloading ISO build %s for variant %s from %s" % (l_build, options.variant, l_url))
-
-# Download and store sha256 digest for required ISO images
-l_current_iso_digest = {}
-fh = urllib.urlopen(l_url + l_build + '/SHA256SUMS')
-# Process the file (will contain a number of entries
-for digest in fh:
- (id, sep, iso) = digest.partition(' ')
- l_current_iso_digest[iso.rstrip('\n').lstrip('*')] = id
-fh.close()
-
-logging.debug(l_current_iso_digest)
-
-# Check to see if version on disk is already the latest
-for arch in l_archs:
- l_download = False
- l_iso_name = options.release + '-' + options.variant + '-' + arch + '.iso'
- lock = FileLock('/tmp/' + l_iso_name)
- with lock:
- l_iso_dir = options.flavor
- l_iso_location = os.path.join(options.isoroot, l_iso_dir, l_iso_name)
- # If iso does not exists then mark for retrieval
- if os.path.exists(l_iso_location):
- # Check to see if latest version
- f = open(l_iso_location)
- l_sha256 = sha256_for_file(f)
- f.close()
- # If digests don't match then mark for retrieval
- if l_current_iso_digest[l_iso_name] != l_sha256:
- logging.debug("ISO is not the latest")
- l_download = True
- else:
- logging.info("Required ISO version (%s,%s) already in cache" % (l_iso_name, l_build))
- else:
- # ISO does not exist need to download
- logging.info("Downloading ISO (%s,%s) to local cache" % (l_iso_name, l_build))
- l_download = True
-
- if l_download:
- # Call dl-ubunut-test-iso with correct arguments
- logging.debug("Downloading ISO as local cache is stale/empty")
- cmd = ['dl-ubuntu-test-iso', '--variant=%s' % options.variant,
- '--arch=%s' % arch, '--release=%s' % options.release,
- '--build=%s' % l_build,'--isoroot=%s' % options.isoroot, '-P']
- if options.flavor:
- cmd.append('--flavor=%s' % options.flavor)
-
- if options.no_act:
- cmd.append('-n')
-
- logging.debug("Cmd: %s" % (cmd))
- subprocess.check_call(cmd)
- # Write out build version to file for later use - needs to be refactored
- f = open(l_iso_location + '.build-version', 'w')
- f.write(l_build)
- f.close()
-
+ with open(fname) as f:
+ while True:
+ data = f.read(block_size)
+ if not data:
+ break
+ sha256.update(data)
+
+ l_sha256 = sha256.hexdigest()
+ logging.debug("Expected Checksum: '%s'", sha256_hash)
+ logging.debug("Local File Checksum: '%s'", l_sha256)
+ return sha256_hash == l_sha256
+
+def download_iso(isoroot, release, variant, arch, flavor=None, no_act=False):
+ """ Download an iso """
+ logging.info("Downloading ISO...")
+ cmd = ['dl-ubuntu-test-iso', '--variant=%s' % variant,
+ '--arch=%s' % arch, '--release=%s' % release,
+ '--build=current' ,'--isoroot=%s' % isoroot, '-P']
+ if flavor:
+ cmd.append('--flavor=%s' % flavor)
+
+ if no_act:
+ cmd.append('-n')
+
+ logging.debug("Cmd: %s", cmd)
+ try:
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError, e:
+ logging.error(e)
+ return e.returncode
+ return 0
+
+def remote_file_exists(file_url):
+ """ Check if the remote file exists
+
+ Return file info on success false otherwise """
+ logging.debug("Checking if remote file exists: '%s'", file_url)
+ try:
+ res = urllib2.urlopen(HeadRequest(file_url))
+ logging.debug('URL Found:\n%s', res.info())
+ return res.info()
+ except (urllib2.HTTPError, urllib2.URLError), e:
+ logging.debug("Check failed: %s", e)
+ return False
+
+def main():
+ """ Main routine """
+ usage = "Usage: %prog [options]"
+ parser = optparse.OptionParser(usage=usage)
+ parser.add_option("-d", "--debug", dest="debug", action="store_true",
+ default=False, help="enable debugging")
+ parser.add_option("-r", "--release", dest="release",
+ default=DEFAULT_RELEASE,
+ help="release of Ubuntu to download (default=%s)"
+ % DEFAULT_RELEASE)
+ parser.add_option("-v", "--variant", dest="variant",
+ default=DEFAULT_VARIANT,
+ help="variant of Ubuntu to download (default=%s)"
+ % DEFAULT_VARIANT)
+ parser.add_option("-a", "--arch", dest="arch", default=DEFAULT_ARCH,
+ help="arch of Ubuntu to download (default=%s)"
+ % DEFAULT_ARCH)
+ parser.add_option("-i", "--isoroot", dest="isoroot",
+ default=DEFAULT_ISOROOT,
+ help="location to store iso images (default=%s)"
+ % DEFAULT_ISOROOT)
+ parser.add_option("-f", "--flavor", dest="flavor", default=DEFAULT_FLAVOR,
+ help="flavor of Ubuntu to download (default=%s)"
+ % DEFAULT_FLAVOR)
+ parser.add_option('-n', '--no-act', default=False, action='store_true',
+ dest='no_act',
+ help='compute everything but don\'t actually download')
+ parser.add_option('-u', '--url', dest="base_url", default=BASE_URL,
+ help="Base URL for cdimage retrieval website (default=%s)"
+ % BASE_URL)
+
+ (options, args) = parser.parse_args()
+
+ if options.debug:
+ logging.basicConfig(level=logging.DEBUG)
+ else:
+ logging.basicConfig(level=logging.INFO)
+
+ # Setup URLS
+ release_part = options.release
+ if options.release == DEFAULT_RELEASE:
+ release_part = ''
+ if options.variant == 'desktop':
+ l_url = os.path.join(options.base_url, release_part, 'daily-live')
+ elif options.variant == 'alternate':
+ l_url = os.path.join(options.base_url, release_part, 'daily')
+ else:
+ # Default to server
+ l_url = os.path.join(options.base_url, 'ubuntu-server', release_part,
+ 'daily')
+
+ # Check options for architectures
+ l_archs = []
+ if options.arch == 'all':
+ l_archs = ALL_ARCHS
+ else:
+ l_archs.append(options.arch)
+
+ logging.info("Checking/Downloading current build for variant %s from %s",
+ options.variant, l_url)
+
+ # Download and store sha256 digest for required ISO images
+ l_current_iso_digest = {}
+ sha256sum_url = os.path.join(l_url, 'current', 'SHA256SUMS')
+ try:
+ fh = urllib2.urlopen(sha256sum_url)
+ except urllib2.HTTPError, e:
+ logging.error("Failed to fetch URL '%s': %s . Aborting!",
+ sha256sum_url, e)
+ sys.exit(1)
+
+ # Process the file (will contain a number of entries
+ for digest in fh:
+ (chksum, sep, iso) = digest.partition(' *')
+ l_current_iso_digest[iso.rstrip('\n')] = chksum
+ fh.close()
+
+ # Check to see if version on disk is already the latest
+ for arch in l_archs:
+ l_iso_name = '-'.join((options.release, options.variant, arch)) + '.iso'
+ l_iso_url = os.path.join(l_url, 'current', l_iso_name)
+
+ # Check ISO availability on remote server
+ for retry in range(1, RETRY_MAX+1):
+ logging.debug('Try %d', retry)
+ if remote_file_exists(l_iso_url):
+ break
+
+ if retry == RETRY_MAX:
+ logging.info('Max number of retries reached. Aborting!')
+ sys.exit(1)
+ logging.info("Remote file not available. Retrying in %d minutes",
+ l_iso_url, TIMEOUT)
+ sleep(TIMEOUT * 60)
+
+ try:
+ iso_sha256 = l_current_iso_digest[l_iso_name]
+ except KeyError:
+ logging.error("No digest found for image '%s'", l_iso_name)
+
+ lock = FileLock('/tmp/' + l_iso_name)
+ with lock:
+ rc = 0
+ l_iso_dir = options.flavor
+ l_iso_location = os.path.join(options.isoroot, l_iso_dir,
+ l_iso_name)
+ logging.debug("Checking state of local cache '%s'", l_iso_location)
+ # If iso does not exists then mark for retrieval
+ if os.path.exists(l_iso_location):
+ # Check to see if latest version
+ # If digests don't match then mark for retrieval
+ if not validate_sha256(l_iso_location, iso_sha256):
+ logging.debug("ISO '%s' is not the latest", l_iso_name)
+ rc = download_iso(options.isoroot, options.release,
+ options.variant, arch, options.flavor,
+ options.no_act)
+ else:
+ logging.info("Required ISO version '%s' already in cache",
+ l_iso_name)
+ else:
+ # ISO does not exist need to download
+ logging.info("Downloading ISO '%s' to local cache", l_iso_name)
+ rc = download_iso(options.isoroot, options.release,
+ options.variant, arch, options.flavor,
+ options.no_act)
+
+ if rc != 0:
+ logging.error("Failed to download ISO. Aborting!")
+ sys.exit(1)
+ else:
+ # We downloaded something, let see if that is what we expected
+ if not validate_sha256(l_iso_location, iso_sha256):
+ logging.info("Local file doesn't match checksum. Aborting!")
+ sys.exit(1)
+
+if __name__ == '__main__':
+ main()
Follow ups
