diff --git a/edk2toollib/uefi/edk2/parsers/base_parser.py b/edk2toollib/uefi/edk2/parsers/base_parser.py index 2e3e7168..d246b1a6 100644 --- a/edk2toollib/uefi/edk2/parsers/base_parser.py +++ b/edk2toollib/uefi/edk2/parsers/base_parser.py @@ -42,6 +42,8 @@ def __init__(self, log="BaseParser"): self.PPs = [] self._Edk2PathUtil = None self.TargetFilePath = None # the abs path of the target file + self.FilePathStack = [] # a stack containing a list of size 2: [filepath, line_count] + self.ParsedFiles = set() self.CurrentLine = -1 self._MacroNotDefinedValue = "0" # value to used for undefined macro @@ -133,6 +135,27 @@ def SetInputVars(self, inputdict): self.InputVars = inputdict return self + def PushTargetFile(self, abs_path, line_count): + """Adds a target file to the stack.""" + self.FilePathStack.append([abs_path, line_count]) + self.ParsedFiles.add(abs_path) + + def DecrementLinesParsed(self) -> bool: + """Decrements line count for the current target file by one. + + Returns: + (bool): True if there are still lines to parse, False otherwise. + """ + if not self.FilePathStack: + return False + line_count = self.FilePathStack[-1][1] + if line_count - 1 > 0: + self.FilePathStack[-1][1] -= 1 + return True + + self.FilePathStack.pop() + return False + def FindPath(self, *p): """Given a path, it will find it relative to the root, the current target file, or the packages path. @@ -159,8 +182,8 @@ def FindPath(self, *p): return os.path.abspath(Path) # If that fails, check a path relative to the target file. - if self.TargetFilePath is not None: - Path = os.path.abspath(os.path.join(os.path.dirname(self.TargetFilePath), *p)) + if self.FilePathStack: + Path = os.path.abspath(os.path.join(os.path.dirname(self.FilePathStack[-1][0]), *p)) if os.path.exists(Path): return os.path.abspath(Path) @@ -789,6 +812,7 @@ def ResetParserState(self): self.ConditionalStack = [] self.CurrentSection = '' self.CurrentFullSection = '' + self.FilePathStack = [] self.Parsed = False diff --git a/edk2toollib/uefi/edk2/parsers/dsc_parser.py b/edk2toollib/uefi/edk2/parsers/dsc_parser.py index 435d6e63..314fb0bf 100644 --- a/edk2toollib/uefi/edk2/parsers/dsc_parser.py +++ b/edk2toollib/uefi/edk2/parsers/dsc_parser.py @@ -92,10 +92,10 @@ def __ParseLine(self, Line, file_name=None, lineno=None): if sp is None: raise FileNotFoundError(include_file) self.Logger.debug("Opening Include File %s" % sp) - self._PushTargetFile(sp) lf = open(sp, "r") loc = lf.readlines() lf.close() + self.PushTargetFile(sp, len(loc)) return ("", loc, sp) # check for new section @@ -216,10 +216,10 @@ def __ParseDefineLine(self, Line): sp = self.FindPath(include_file) if sp is None: raise FileNotFoundError(include_file) - self._PushTargetFile(sp) lf = open(sp, "r") loc = lf.readlines() lf.close() + self.PushTargetFile(sp, len(loc)) return ("", loc) # check for new section @@ -293,6 +293,7 @@ def __ProcessMore(self, lines, file_name=None): # otherwise, let the user know that we failed in the DSC self.Logger.warning(f"DSC Parser (No-Fail Mode): {raw_line}") self.Logger.warning(e) + self.DecrementLinesParsed() def __ProcessDefines(self, lines): """Goes through a file once to look for [Define] sections. @@ -315,6 +316,7 @@ def __ProcessDefines(self, lines): # otherwise, raise the exception and act normally if not self._no_fail_mode: raise + self.DecrementLinesParsed() # Reset the PcdValueDict as this was just to find any Defines. self.PcdValueDict = {} @@ -477,14 +479,14 @@ def ParseFile(self, filepath): sp = self.FindPath(filepath) if sp is None: raise FileNotFoundError(filepath) - self._PushTargetFile(sp) f = open(sp, "r") # expand all the lines and include other files file_lines = f.readlines() + self.PushTargetFile(sp, len(file_lines)) self.__ProcessDefines(file_lines) # reset the parser state before processing more self.ResetParserState() - self._PushTargetFile(sp) + self.PushTargetFile(sp, len(file_lines)) self.__ProcessMore(file_lines, file_name=sp) f.close() @@ -492,10 +494,6 @@ def ParseFile(self, filepath): self._parse_components() self.Parsed = True - def _PushTargetFile(self, targetFile): - self.TargetFilePath = os.path.abspath(targetFile) - self._dsc_file_paths.add(self.TargetFilePath) - def GetMods(self): """Returns a list with all Mods.""" return self.ThreeMods + self.SixMods @@ -517,7 +515,7 @@ def GetAllDscPaths(self): They are not all guaranteed to be DSC files """ - return self._dsc_file_paths + return self.ParsedFiles def RegisterPcds(self, line): """Reads the line and registers any PCDs found.""" diff --git a/edk2toollib/uefi/edk2/parsers/fdf_parser.py b/edk2toollib/uefi/edk2/parsers/fdf_parser.py index e4e542e0..96aca4bf 100644 --- a/edk2toollib/uefi/edk2/parsers/fdf_parser.py +++ b/edk2toollib/uefi/edk2/parsers/fdf_parser.py @@ -50,13 +50,16 @@ def GetNextLine(self): sline = self.StripComment(line) if (sline is None or len(sline) < 1): + self.DecrementLinesParsed() return self.GetNextLine() sline = self.ReplaceVariables(sline) if self.ProcessConditional(sline): # was a conditional so skip + self.DecrementLinesParsed() return self.GetNextLine() if not self.InActiveCode(): + self.DecrementLinesParsed() return self.GetNextLine() self._BracketCount += sline.count("{") @@ -67,9 +70,11 @@ def GetNextLine(self): def InsertLinesFromFile(self, file_path: str): """Adds additional lines to the Lines Attribute from the provided file.""" with open(file_path, 'r') as lines_file: - self.Lines += reversed(lines_file.readlines()) + lines = lines_file.readlines() + self.Lines += reversed(lines) # Back off the line count to ignore the include line itself. self.CurrentLine -= 1 + self.PushTargetFile(file_path, len(lines)) def ParseFile(self, filepath): """Parses the provided FDF file.""" @@ -79,12 +84,13 @@ def ParseFile(self, filepath): else: fp = filepath self.Path = fp - self.TargetFilePath = os.path.abspath(fp) + fp = os.path.abspath(fp) self.CurrentLine = 0 self._f = open(fp, "r") self.Lines = self._f.readlines() self.Lines.reverse() self._f.close() + self.PushTargetFile(fp, len(self.Lines)) self._BracketCount = 0 InDefinesSection = False InFdSection = False @@ -221,4 +227,5 @@ def ParseFile(self, filepath): elif sline.strip().lower().startswith('[rule.'): InRuleSection = True + self.DecrementLinesParsed() self.Parsed = True diff --git a/tests.unit/parsers/test_base_parser.py b/tests.unit/parsers/test_base_parser.py index 43c644f9..b6d3a05d 100644 --- a/tests.unit/parsers/test_base_parser.py +++ b/tests.unit/parsers/test_base_parser.py @@ -7,11 +7,12 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## +import os +import tempfile import unittest + from edk2toollib.uefi.edk2.parsers.base_parser import BaseParser from edk2toollib.uefi.edk2.path_utilities import Edk2Path -import tempfile -import os class TestBaseParser(unittest.TestCase): @@ -460,14 +461,14 @@ def test_emulator_conditional_not_in(self): parser.ResetParserState() def test_emulator_conditional_parens_order(self): - ''' Makes sure the parenthesis affect the order of expressions ''' + '''Makes sure the parenthesis affect the order of expressions.''' parser = BaseParser("") self.assertFalse(parser.EvaluateConditional('!if TRUE OR FALSE AND FALSE')) self.assertTrue(parser.EvaluateConditional('!if TRUE OR (FALSE AND FALSE)')) parser.ResetParserState() def test_emulator_conditional_not_or(self): - ''' Makes sure we can use the not with other operators ''' + '''Makes sure we can use the not with other operators.''' parser = BaseParser("") self.assertTrue(parser.EvaluateConditional('!if FALSE NOT OR FALSE')) self.assertFalse(parser.EvaluateConditional('!if TRUE NOT OR FALSE')) @@ -475,7 +476,7 @@ def test_emulator_conditional_not_or(self): self.assertFalse(parser.EvaluateConditional('!if TRUE NOT OR TRUE')) def test_emulator_conditional_not_it_all(self): - ''' Makes sure the parenthesis affect the order of expressions ''' + '''Makes sure the parenthesis affect the order of expressions.''' parser = BaseParser("") self.assertTrue(parser.EvaluateConditional('!if NOT FALSE OR FALSE')) self.assertFalse(parser.EvaluateConditional('!if NOT TRUE OR FALSE')) @@ -640,7 +641,7 @@ def test_find_path(self): # create target file os.makedirs(target_filedir) parser.WriteLinesToFile(target_filepath) - parser.TargetFilePath = target_filepath + parser.PushTargetFile(target_filepath, 0) # check if we can find the root root_found = parser.FindPath(root_file) self.assertEqual(root_found, root_filepath) diff --git a/tests.unit/parsers/test_dsc_parser.py b/tests.unit/parsers/test_dsc_parser.py index 7c349087..beb2a35b 100644 --- a/tests.unit/parsers/test_dsc_parser.py +++ b/tests.unit/parsers/test_dsc_parser.py @@ -7,9 +7,10 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## -import unittest -import tempfile import os +import tempfile +import unittest + from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser from edk2toollib.uefi.edk2.path_utilities import Edk2Path @@ -30,7 +31,7 @@ def write_to_file(file_path, data): f.close() def test_dsc_include_single_file(self): - ''' This tests whether includes work properly ''' + '''This tests whether includes work properly.''' workspace = tempfile.mkdtemp() file1_name = "file1.dsc" @@ -53,7 +54,7 @@ def test_dsc_include_single_file(self): self.assertEqual(len(parser.GetAllDscPaths()), 2) # make sure we have two dsc paths def test_dsc_include_missing_file(self): - ''' This tests whether includes work properly ''' + '''This tests whether includes work properly.''' workspace = tempfile.mkdtemp() file1_name = "file1.dsc" @@ -69,7 +70,7 @@ def test_dsc_include_missing_file(self): parser.ParseFile(file1_path) def test_dsc_include_missing_file_no_fail_mode(self): - ''' This tests whether includes work properly if no fail mode is on''' + '''This tests whether includes work properly if no fail mode is on.''' workspace = tempfile.mkdtemp() file1_name = "file1.dsc" @@ -85,7 +86,7 @@ def test_dsc_include_missing_file_no_fail_mode(self): parser.ParseFile(file1_path) def test_dsc_parse_file_on_package_path(self): - ''' This tests whether includes work properly if no fail mode is on''' + '''This tests whether includes work properly if no fail mode is on.''' workspace = tempfile.mkdtemp() working_dir_name = "working" working2_dir_name = "working2" @@ -113,7 +114,7 @@ def test_dsc_parse_file_on_package_path(self): self.assertEqual(parser.LocalVars["INCLUDED"], "TRUE") # make sure we got the defines def test_dsc_include_relative_path(self): - ''' This tests whether includes work properly with a relative path''' + '''This tests whether includes work properly with a relative path.''' workspace = tempfile.mkdtemp() outside_folder = os.path.join(workspace, "outside") inside_folder = os.path.join(outside_folder, "inside") @@ -154,3 +155,45 @@ def test_dsc_include_relative_path(self): self.assertEqual(parser.LocalVars["INCLUDED"], "TRUE") # make sure we got the defines finally: os.chdir(cwd) + +def test_dsc_include_relative_paths2(tmp_path): + """This tests whether includes work properly with a relative path, when includes are not nested. + + Directory setup + src + └─ Platforms + └─ PlatformPkg + ├─ PlatformPkg.dsc + └─ includes + ├─ PCDs1.dsc.inc + └─ PCDs2.dsc.inc + + Base DSC + [PcdsFixedAtBuild] + gPlatformPkgTokenSpaceGuid.PcdSomething + !include includes/PCDs1.dsc.inc + !include includes/PCDs2.dsc.inc + """ + pp = tmp_path / "Platforms" + pkg_dir = pp / "PlatformPkg" + dsc = pkg_dir / "PlatformPkg.dsc" + inc_dir = pkg_dir / "includes" + + # Create Platforms, Platforms/PlatformPkg, Platforms/PlatformPkg/includes + inc_dir.mkdir(parents=True) + + with open(inc_dir / "PCDs1.dsc.inc", "w") as f: + f.write("gPlatformPkgTokenSpaceGuid.PcdSomething1\n") + + with open(inc_dir / "PCDs2.dsc.inc", "w") as f: + f.write("gPlatformPkgTokenSpaceGuid.PcdSomething2\n") + + with open(dsc, "w") as f: + f.write("[PcdsFixedAtBuild]\n") + f.write("gPlatformPkgTokenSpaceGuid.PcdSomething\n") + f.write("!include includes/PCDs1.dsc.inc\n") + f.write("!include includes/PCDs2.dsc.inc\n") + + parser = DscParser() + parser.SetEdk2Path(Edk2Path(str(tmp_path), [str(pp)])) + parser.ParseFile(dsc) diff --git a/tests.unit/parsers/test_fdf_parser.py b/tests.unit/parsers/test_fdf_parser.py index a5e01cd3..2d82baac 100644 --- a/tests.unit/parsers/test_fdf_parser.py +++ b/tests.unit/parsers/test_fdf_parser.py @@ -7,10 +7,11 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## -import unittest import os -import textwrap import tempfile +import textwrap +import unittest + from edk2toollib.uefi.edk2.parsers.fdf_parser import FdfParser from edk2toollib.uefi.edk2.path_utilities import Edk2Path @@ -101,3 +102,54 @@ def test_section_guided(): # Then assert "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" in \ parser.FVs["MAINFV"]["Files"]["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"] + +def test_fdf_include_relative_paths(tmp_path): + """This tests whether includes work properly with a relative path. + + Tests the following scenarios: + 1. include multiple files from the same directory + 2. Properly decrement when a comment exists in the file + 3. Properly decrement when a macro exists in the file + 4. Properly decrement when in an inactive portion of a conditional macro + + Directory setup + src + └─ Platforms + └─ PlatformPkg + ├─ PlatformPkg.dsc + └─ includes + ├─ Libs1.dsc.inc + └─ Libs2.dsc.inc + + Base FDF + [LibraryClasses] + !include includes/Libs1.dsc.inc + !include includes/Libs2.dsc.inc + """ + pp = tmp_path / "Platforms" + pkg_dir = pp / "PlatformPkg" + fdf = pkg_dir / "PlatformPkg.fdf" + inc_dir = pkg_dir / "includes" + + # Create Platforms, Platforms/PlatformPkg, Platforms/PlatformPkg/includes + inc_dir.mkdir(parents=True) + + with open(inc_dir / "Libs1.dsc.inc", "w") as f: + f.write("# A Comment here.\n") + f.write("!if TRUE == TRUE\n") + f.write(" Lib1|BaseLib1.inf\n") + f.write("!else\n") + f.write(" Lib1|BaseLib3.inf\n") + f.write("!endif\n") + + with open(inc_dir / "Libs2.dsc.inc", "w") as f: + f.write("Lib2|BaseLib2.inf\n") + + with open(fdf, "w") as f: + f.write("[LibraryClasses]\n") + f.write("!include includes/Libs1.dsc.inc\n") + f.write("!include includes/Libs2.dsc.inc\n") + + parser = FdfParser() + parser.SetEdk2Path(Edk2Path(str(tmp_path), [str(pp)])) + parser.ParseFile(fdf)