@ -5,11 +5,42 @@
import re
import glob
from HTMLParser import HTMLParser
import os
import sys
import tempfile
import urllib2
import bsettings
import command
# Simple class to collect links from a page
class MyHTMLParser ( HTMLParser ) :
def __init__ ( self , arch ) :
""" Create a new parser
After the parser runs , self . links will be set to a list of the links
to . xz archives found in the page , and self . arch_link will be set to
the one for the given architecture ( or None if not found ) .
Args :
arch : Architecture to search for
"""
HTMLParser . __init__ ( self )
self . arch_link = None
self . links = [ ]
self . _match = ' _ %s - ' % arch
def handle_starttag ( self , tag , attrs ) :
if tag == ' a ' :
for tag , value in attrs :
if tag == ' href ' :
if value and value . endswith ( ' .xz ' ) :
self . links . append ( value )
if self . _match in value :
self . arch_link = value
class Toolchain :
""" A single toolchain
@ -20,7 +51,6 @@ class Toolchain:
arch : Architecture of toolchain as determined from the first
component of the filename . E . g . arm - linux - gcc becomes arm
"""
def __init__ ( self , fname , test , verbose = False ) :
""" Create a new toolchain object.
@ -116,18 +146,29 @@ class Toolchains:
self . paths = [ ]
self . _make_flags = dict ( bsettings . GetItems ( ' make-flags ' ) )
def GetSettings ( self ) :
def GetPathList ( self ) :
""" Get a list of available toolchain paths
Returns :
List of strings , each a path to a toolchain mentioned in the
[ toolchain ] section of the settings file .
"""
toolchains = bsettings . GetItems ( ' toolchain ' )
if not toolchains :
print ( " Warning: No tool chains - please add a [toolchain] section "
" to your buildman config file %s . See README for details " %
bsettings . config_fname )
paths = [ ]
for name , value in toolchains :
if ' * ' in value :
self . paths + = glob . glob ( value )
paths + = glob . glob ( value )
else :
self . paths . append ( value )
paths . append ( value )
return paths
def GetSettings ( self ) :
self . paths + = self . GetPathList ( )
def Add ( self , fname , test = True , verbose = False ) :
""" Add a toolchain to our list
@ -147,6 +188,24 @@ class Toolchains:
if add_it :
self . toolchains [ toolchain . arch ] = toolchain
def ScanPath ( self , path , verbose ) :
""" Scan a path for a valid toolchain
Args :
path : Path to scan
verbose : True to print out progress information
Returns :
Filename of C compiler if found , else None
"""
for subdir in [ ' . ' , ' bin ' , ' usr/bin ' ] :
dirname = os . path . join ( path , subdir )
if verbose : print " - looking in ' %s ' " % dirname
for fname in glob . glob ( dirname + ' /*gcc ' ) :
if verbose : print " - found ' %s ' " % fname
return fname
return None
def Scan ( self , verbose ) :
""" Scan for available toolchains and select the best for each arch.
@ -160,12 +219,9 @@ class Toolchains:
if verbose : print ' Scanning for tool chains '
for path in self . paths :
if verbose : print " - scanning path ' %s ' " % path
for subdir in [ ' . ' , ' bin ' , ' usr/bin ' ] :
dirname = os . path . join ( path , subdir )
if verbose : print " - looking in ' %s ' " % dirname
for fname in glob . glob ( dirname + ' /*gcc ' ) :
if verbose : print " - found ' %s ' " % fname
self . Add ( fname , True , verbose )
fname = self . ScanPath ( path , verbose )
if fname :
self . Add ( fname , True , verbose )
def List ( self ) :
""" List out the selected toolchains for each architecture """
@ -264,3 +320,160 @@ class Toolchains:
else :
i + = 1
return args
def LocateArchUrl ( self , fetch_arch ) :
""" Find a toolchain available online
Look in standard places for available toolchains . At present the
only standard place is at kernel . org .
Args :
arch : Architecture to look for , or ' list ' for all
Returns :
If fetch_arch is ' list ' , a tuple :
Machine architecture ( e . g . x86_64 )
List of toolchains
else
URL containing this toolchain , if avaialble , else None
"""
arch = command . OutputOneLine ( ' uname ' , ' -m ' )
base = ' https://www.kernel.org/pub/tools/crosstool/files/bin '
versions = [ ' 4.6.3 ' , ' 4.6.2 ' , ' 4.5.1 ' , ' 4.2.4 ' ]
links = [ ]
for version in versions :
url = ' %s / %s / %s / ' % ( base , arch , version )
print ' Checking: %s ' % url
response = urllib2 . urlopen ( url )
html = response . read ( )
parser = MyHTMLParser ( fetch_arch )
parser . feed ( html )
if fetch_arch == ' list ' :
links + = parser . links
elif parser . arch_link :
return url + parser . arch_link
if fetch_arch == ' list ' :
return arch , links
return None
def Download ( self , url ) :
""" Download a file to a temporary directory
Args :
url : URL to download
Returns :
Tuple :
Temporary directory name
Full path to the downloaded archive file in that directory ,
or None if there was an error while downloading
"""
print " Downloading: %s " % url
leaf = url . split ( ' / ' ) [ - 1 ]
tmpdir = tempfile . mkdtemp ( ' .buildman ' )
response = urllib2 . urlopen ( url )
fname = os . path . join ( tmpdir , leaf )
fd = open ( fname , ' wb ' )
meta = response . info ( )
size = int ( meta . getheaders ( " Content-Length " ) [ 0 ] )
done = 0
block_size = 1 << 16
status = ' '
# Read the file in chunks and show progress as we go
while True :
buffer = response . read ( block_size )
if not buffer :
print chr ( 8 ) * ( len ( status ) + 1 ) , ' \r ' ,
break
done + = len ( buffer )
fd . write ( buffer )
status = r " %10d MiB [ %3d %% ] " % ( done / 1024 / 1024 ,
done * 100 / size )
status = status + chr ( 8 ) * ( len ( status ) + 1 )
print status ,
sys . stdout . flush ( )
fd . close ( )
if done != size :
print ' Error, failed to download '
os . remove ( fname )
fname = None
return tmpdir , fname
def Unpack ( self , fname , dest ) :
""" Unpack a tar file
Args :
fname : Filename to unpack
dest : Destination directory
Returns :
Directory name of the first entry in the archive , without the
trailing /
"""
stdout = command . Output ( ' tar ' , ' xvfJ ' , fname , ' -C ' , dest )
return stdout . splitlines ( ) [ 0 ] [ : - 1 ]
def TestSettingsHasPath ( self , path ) :
""" Check if builmand will find this toolchain
Returns :
True if the path is in settings , False if not
"""
paths = self . GetPathList ( )
return path in paths
def ListArchs ( self ) :
""" List architectures with available toolchains to download """
host_arch , archives = self . LocateArchUrl ( ' list ' )
re_arch = re . compile ( ' [-a-z0-9.]*_([^-]*)-.* ' )
arch_set = set ( )
for archive in archives :
# Remove the host architecture from the start
arch = re_arch . match ( archive [ len ( host_arch ) : ] )
if arch :
arch_set . add ( arch . group ( 1 ) )
return sorted ( arch_set )
def FetchAndInstall ( self , arch ) :
""" Fetch and install a new toolchain
arch :
Architecture to fetch , or ' list ' to list
"""
# Fist get the URL for this architecture
url = self . LocateArchUrl ( arch )
if not url :
print ( " Cannot find toolchain for arch ' %s ' - use ' list ' to list " %
arch )
return 2
home = os . environ [ ' HOME ' ]
dest = os . path . join ( home , ' .buildman-toolchains ' )
if not os . path . exists ( dest ) :
os . mkdir ( dest )
# Download the tar file for this toolchain and unpack it
tmpdir , tarfile = self . Download ( url )
if not tarfile :
return 1
print ' Unpacking to: %s ' % dest ,
sys . stdout . flush ( )
path = self . Unpack ( tarfile , dest )
os . remove ( tarfile )
os . rmdir ( tmpdir )
print
# Check that the toolchain works
print ' Testing '
dirpath = os . path . join ( dest , path )
compiler_fname = self . ScanPath ( dirpath , True )
if not compiler_fname :
print ' Could not locate C compiler - fetch failed. '
return 1
toolchain = Toolchain ( compiler_fname , True , True )
# Make sure that it will be found by buildman
if not self . TestSettingsHasPath ( dirpath ) :
print ( " Adding ' download ' to config file ' %s ' " %
bsettings . config_fname )
tools_dir = os . path . dirname ( dirpath )
bsettings . SetItem ( ' toolchain ' , ' download ' , ' %s /* ' % tools_dir )
return 0
Simon Glass
10 years ago