1## @file
2# This file is used to define each component of tools_def.txt file
3#
4# Copyright (c) 2007 - 2016, Intel Corporation. All rights reserved.<BR>
5# This program and the accompanying materials
6# are licensed and made available under the terms and conditions of the BSD License
7# which accompanies this distribution.  The full text of the license may be found at
8# http://opensource.org/licenses/bsd-license.php
9#
10# THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
11# WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
12#
13
14##
15# Import Modules
16#
17import Common.LongFilePathOs as os
18import re
19import EdkLogger
20
21from Dictionary import *
22from BuildToolError import *
23from TargetTxtClassObject import *
24from Common.LongFilePathSupport import OpenLongFilePath as open
25from Common.Misc import PathClass
26from Common.String import NormPath
27import Common.GlobalData as GlobalData
28from Common import GlobalData
29from Common.MultipleWorkspace import MultipleWorkspace as mws
30
31##
32# Static variables used for pattern
33#
34gMacroRefPattern = re.compile('(DEF\([^\(\)]+\))')
35gEnvRefPattern = re.compile('(ENV\([^\(\)]+\))')
36gMacroDefPattern = re.compile("DEFINE\s+([^\s]+)")
37gDefaultToolsDefFile = "tools_def.txt"
38
39## ToolDefClassObject
40#
41# This class defined content used in file tools_def.txt
42#
43# @param object:               Inherited from object class
44# @param Filename:             Input value for full path of tools_def.txt
45#
46# @var ToolsDefTxtDictionary:  To store keys and values defined in target.txt
47# @var MacroDictionary:        To store keys and values defined in DEFINE statement
48#
49class ToolDefClassObject(object):
50    def __init__(self, FileName=None):
51        self.ToolsDefTxtDictionary = {}
52        self.MacroDictionary = {}
53        for Env in os.environ:
54            self.MacroDictionary["ENV(%s)" % Env] = os.environ[Env]
55
56        if FileName != None:
57            self.LoadToolDefFile(FileName)
58
59    ## LoadToolDefFile
60    #
61    # Load target.txt file and parse it
62    #
63    # @param Filename:  Input value for full path of tools_def.txt
64    #
65    def LoadToolDefFile(self, FileName):
66        # set multiple workspace
67        PackagesPath = os.getenv("PACKAGES_PATH")
68        mws.setWs(GlobalData.gWorkspace, PackagesPath)
69
70        self.ToolsDefTxtDatabase = {
71            TAB_TOD_DEFINES_TARGET          :   [],
72            TAB_TOD_DEFINES_TOOL_CHAIN_TAG  :   [],
73            TAB_TOD_DEFINES_TARGET_ARCH     :   [],
74            TAB_TOD_DEFINES_COMMAND_TYPE    :   []
75        }
76
77        self.IncludeToolDefFile(FileName)
78
79        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET]))
80        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG]))
81        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH]))
82
83        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] = list(set(self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE]))
84
85        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET].sort()
86        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG].sort()
87        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH].sort()
88        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE].sort()
89
90        KeyList = [TAB_TOD_DEFINES_TARGET, TAB_TOD_DEFINES_TOOL_CHAIN_TAG, TAB_TOD_DEFINES_TARGET_ARCH, TAB_TOD_DEFINES_COMMAND_TYPE]
91        for Index in range(3, -1, -1):
92            for Key in dict(self.ToolsDefTxtDictionary):
93                List = Key.split('_')
94                if List[Index] == '*':
95                    for String in self.ToolsDefTxtDatabase[KeyList[Index]]:
96                        List[Index] = String
97                        NewKey = '%s_%s_%s_%s_%s' % tuple(List)
98                        if NewKey not in self.ToolsDefTxtDictionary:
99                            self.ToolsDefTxtDictionary[NewKey] = self.ToolsDefTxtDictionary[Key]
100                        continue
101                    del self.ToolsDefTxtDictionary[Key]
102                elif List[Index] not in self.ToolsDefTxtDatabase[KeyList[Index]]:
103                    del self.ToolsDefTxtDictionary[Key]
104
105
106    ## IncludeToolDefFile
107    #
108    # Load target.txt file and parse it as if it's contents were inside the main file
109    #
110    # @param Filename:  Input value for full path of tools_def.txt
111    #
112    def IncludeToolDefFile(self, FileName):
113        FileContent = []
114        if os.path.isfile(FileName):
115            try:
116                F = open(FileName, 'r')
117                FileContent = F.readlines()
118            except:
119                EdkLogger.error("tools_def.txt parser", FILE_OPEN_FAILURE, ExtraData=FileName)
120        else:
121            EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=FileName)
122
123        for Index in range(len(FileContent)):
124            Line = FileContent[Index].strip()
125            if Line == "" or Line[0] == '#':
126                continue
127
128            if Line.startswith("!include"):
129                IncFile = Line[8:].strip()
130                Done, IncFile = self.ExpandMacros(IncFile)
131                if not Done:
132                    EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
133                                    "Macro or Environment has not been defined",
134                                ExtraData=IncFile[4:-1], File=FileName, Line=Index+1)
135                IncFile = NormPath(IncFile)
136
137                if not os.path.isabs(IncFile):
138                    #
139                    # try WORKSPACE
140                    #
141                    IncFileTmp = PathClass(IncFile, GlobalData.gWorkspace)
142                    ErrorCode = IncFileTmp.Validate()[0]
143                    if ErrorCode != 0:
144                        #
145                        # try PACKAGES_PATH
146                        #
147                        IncFileTmp = mws.join(GlobalData.gWorkspace, IncFile)
148                        if not os.path.exists(IncFileTmp):
149                            #
150                            # try directory of current file
151                            #
152                            IncFileTmp = PathClass(IncFile, os.path.dirname(FileName))
153                            ErrorCode = IncFileTmp.Validate()[0]
154                            if ErrorCode != 0:
155                                EdkLogger.error("tools_def.txt parser", FILE_NOT_FOUND, ExtraData=IncFile)
156
157                    if type(IncFileTmp) is PathClass:
158                        IncFile = IncFileTmp.Path
159                    else:
160                        IncFile = IncFileTmp
161
162                self.IncludeToolDefFile(IncFile)
163                continue
164
165            NameValuePair = Line.split("=", 1)
166            if len(NameValuePair) != 2:
167                EdkLogger.warn("tools_def.txt parser", "Line %d: not correct assignment statement, skipped" % (Index + 1))
168                continue
169
170            Name = NameValuePair[0].strip()
171            Value = NameValuePair[1].strip()
172
173            if Name == "IDENTIFIER":
174                EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found identifier statement, skipped: %s" % ((Index + 1), Value))
175                continue
176
177            MacroDefinition = gMacroDefPattern.findall(Name)
178            if MacroDefinition != []:
179                Done, Value = self.ExpandMacros(Value)
180                if not Done:
181                    EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
182                                    "Macro or Environment has not been defined",
183                                ExtraData=Value[4:-1], File=FileName, Line=Index+1)
184
185                MacroName = MacroDefinition[0].strip()
186                self.MacroDictionary["DEF(%s)" % MacroName] = Value
187                EdkLogger.debug(EdkLogger.DEBUG_8, "Line %d: Found macro: %s = %s" % ((Index + 1), MacroName, Value))
188                continue
189
190            Done, Value = self.ExpandMacros(Value)
191            if not Done:
192                EdkLogger.error("tools_def.txt parser", ATTRIBUTE_NOT_AVAILABLE,
193                                "Macro or Environment has not been defined",
194                                ExtraData=Value[4:-1], File=FileName, Line=Index+1)
195
196            List = Name.split('_')
197            if len(List) != 5:
198                EdkLogger.verbose("Line %d: Not a valid name of definition: %s" % ((Index + 1), Name))
199                continue
200            elif List[4] == '*':
201                EdkLogger.verbose("Line %d: '*' is not allowed in last field: %s" % ((Index + 1), Name))
202                continue
203            else:
204                self.ToolsDefTxtDictionary[Name] = Value
205                if List[0] != '*':
206                    self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET] += [List[0]]
207                if List[1] != '*':
208                    self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TOOL_CHAIN_TAG] += [List[1]]
209                if List[2] != '*':
210                    self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_TARGET_ARCH] += [List[2]]
211                if List[3] != '*':
212                    self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_COMMAND_TYPE] += [List[3]]
213                if List[4] == TAB_TOD_DEFINES_FAMILY and List[2] == '*' and List[3] == '*':
214                    if TAB_TOD_DEFINES_FAMILY not in self.ToolsDefTxtDatabase:
215                        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY] = {}
216                        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
217                        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY] = {}
218                        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
219                    elif List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
220                        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] = Value
221                        self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
222                    elif self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY][List[1]] != Value:
223                        EdkLogger.verbose("Line %d: No override allowed for the family of a tool chain: %s" % ((Index + 1), Name))
224                if List[4] == TAB_TOD_DEFINES_BUILDRULEFAMILY and List[2] == '*' and List[3] == '*':
225                    if TAB_TOD_DEFINES_BUILDRULEFAMILY not in self.ToolsDefTxtDatabase \
226                       or List[1] not in self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_FAMILY]:
227                        EdkLogger.verbose("Line %d: The family is not specified, but BuildRuleFamily is specified for the tool chain: %s" % ((Index + 1), Name))
228                    self.ToolsDefTxtDatabase[TAB_TOD_DEFINES_BUILDRULEFAMILY][List[1]] = Value
229
230    ## ExpandMacros
231    #
232    # Replace defined macros with real value
233    #
234    # @param Value:   The string with unreplaced macros
235    #
236    # @retval Value:  The string which has been replaced with real value
237    #
238    def ExpandMacros(self, Value):
239        # os.environ contains all environment variables uppercase on Windows which cause the key in the self.MacroDictionary is uppercase, but Ref may not
240        EnvReference = gEnvRefPattern.findall(Value)
241        for Ref in EnvReference:
242            if Ref not in self.MacroDictionary and Ref.upper() not in self.MacroDictionary:
243                Value = Value.replace(Ref, "")
244            else:
245                if Ref in self.MacroDictionary:
246                    Value = Value.replace(Ref, self.MacroDictionary[Ref])
247                else:
248                    Value = Value.replace(Ref, self.MacroDictionary[Ref.upper()])
249
250        MacroReference = gMacroRefPattern.findall(Value)
251        for Ref in MacroReference:
252            if Ref not in self.MacroDictionary:
253                return False, Ref
254            Value = Value.replace(Ref, self.MacroDictionary[Ref])
255
256        return True, Value
257
258## ToolDefDict
259#
260# Load tools_def.txt in input Conf dir
261#
262# @param ConfDir:  Conf dir
263#
264# @retval ToolDef An instance of ToolDefClassObject() with loaded tools_def.txt
265#
266def ToolDefDict(ConfDir):
267    Target = TargetTxtDict(ConfDir)
268    ToolDef = ToolDefClassObject()
269    if DataType.TAB_TAT_DEFINES_TOOL_CHAIN_CONF in Target.TargetTxtDictionary:
270        ToolsDefFile = Target.TargetTxtDictionary[DataType.TAB_TAT_DEFINES_TOOL_CHAIN_CONF]
271        if ToolsDefFile:
272            ToolDef.LoadToolDefFile(os.path.normpath(ToolsDefFile))
273        else:
274            ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
275    else:
276        ToolDef.LoadToolDefFile(os.path.normpath(os.path.join(ConfDir, gDefaultToolsDefFile)))
277    return ToolDef
278
279##
280#
281# This acts like the main() function for the script, unless it is 'import'ed into another
282# script.
283#
284if __name__ == '__main__':
285    ToolDef = ToolDefDict(os.getenv("WORKSPACE"))
286    pass
287