base_parser: support relative path include files

edk2toollib/uefi/edk2/parsers/base_parser.py

@@ -42,6 +42,8 @@
         self.PPs = []
         self._Edk2PathUtil = None
         self.TargetFilePath = None  # the abs path of the target file
+        self.FilePathStack = []  # a stack containing a list of size 2: [filepath, line_count]
+        self.ParsedFiles = set()
         self.CurrentLine = -1
         self._MacroNotDefinedValue = "0"  # value to used for undefined macro
 
@@ -133,6 +135,27 @@
         self.InputVars = inputdict
         return self
 
+    def PushTargetFile(self, abs_path, line_count):
+        """Adds a target file to the stack."""
+        self.FilePathStack.append([abs_path, line_count])
+        self.ParsedFiles.add(abs_path)
+
+    def DecrementLinesParsed(self) -> bool:
+        """Decrements line count for the current target file by one.
+
+        Returns:
+            (bool): True if there are still lines to parse, False otherwise.
+        """
+        if not self.FilePathStack:
+            return False
+        line_count = self.FilePathStack[-1][1]
+        if line_count - 1 > 0:
+            self.FilePathStack[-1][1] -= 1
+            return True
+
+        self.FilePathStack.pop()
+        return False
+
     def FindPath(self, *p):
         """Given a path, it will find it relative to the root, the current target file, or the packages path.
 
@@ -159,8 +182,8 @@
             return os.path.abspath(Path)
 
         # If that fails, check a path relative to the target file.
-        if self.TargetFilePath is not None:
-            Path = os.path.abspath(os.path.join(os.path.dirname(self.TargetFilePath), *p))
+        if self.FilePathStack:
+            Path = os.path.abspath(os.path.join(os.path.dirname(self.FilePathStack[-1][0]), *p))
             if os.path.exists(Path):
                 return os.path.abspath(Path)
 
@@ -789,6 +812,7 @@
         self.ConditionalStack = []
         self.CurrentSection = ''
         self.CurrentFullSection = ''
+        self.FilePathStack = []
         self.Parsed = False
 
 

edk2toollib/uefi/edk2/parsers/dsc_parser.py

@@ -92,10 +92,10 @@ def __ParseLine(self, Line, file_name=None, lineno=None):
             if sp is None:
                 raise FileNotFoundError(include_file)
             self.Logger.debug("Opening Include File %s" % sp)
-            self._PushTargetFile(sp)
             lf = open(sp, "r")
             loc = lf.readlines()
             lf.close()
+            self.PushTargetFile(sp, len(loc))
             return ("", loc, sp)
 
         # check for new section
@@ -216,10 +216,10 @@ def __ParseDefineLine(self, Line):
             sp = self.FindPath(include_file)
             if sp is None:
                 raise FileNotFoundError(include_file)
-            self._PushTargetFile(sp)
             lf = open(sp, "r")
             loc = lf.readlines()
             lf.close()
+            self.PushTargetFile(sp, len(loc))
             return ("", loc)
 
         # check for new section
@@ -293,6 +293,7 @@ def __ProcessMore(self, lines, file_name=None):
                     # otherwise, let the user know that we failed in the DSC
                     self.Logger.warning(f"DSC Parser (No-Fail Mode): {raw_line}")
                     self.Logger.warning(e)
+            self.DecrementLinesParsed()
 
     def __ProcessDefines(self, lines):
         """Goes through a file once to look for [Define] sections.
@@ -315,6 +316,7 @@ def __ProcessDefines(self, lines):
                 # otherwise, raise the exception and act normally
                 if not self._no_fail_mode:
                     raise
+            self.DecrementLinesParsed()
         # Reset the PcdValueDict as this was just to find any Defines.
         self.PcdValueDict = {}
 
@@ -477,25 +479,21 @@ def ParseFile(self, filepath):
         sp = self.FindPath(filepath)
         if sp is None:
             raise FileNotFoundError(filepath)
-        self._PushTargetFile(sp)
         f = open(sp, "r")
         # expand all the lines and include other files
         file_lines = f.readlines()
+        self.PushTargetFile(sp, len(file_lines))
         self.__ProcessDefines(file_lines)
         # reset the parser state before processing more
         self.ResetParserState()
-        self._PushTargetFile(sp)
+        self.PushTargetFile(sp, len(file_lines))
         self.__ProcessMore(file_lines, file_name=sp)
         f.close()
 
         self._parse_libraries()
         self._parse_components()
         self.Parsed = True
 
-    def _PushTargetFile(self, targetFile):
-        self.TargetFilePath = os.path.abspath(targetFile)
-        self._dsc_file_paths.add(self.TargetFilePath)
-
     def GetMods(self):
         """Returns a list with all Mods."""
         return self.ThreeMods + self.SixMods
@@ -517,7 +515,7 @@ def GetAllDscPaths(self):
 
         They are not all guaranteed to be DSC files
         """
-        return self._dsc_file_paths
+        return self.ParsedFiles
 
     def RegisterPcds(self, line):
         """Reads the line and registers any PCDs found."""

edk2toollib/uefi/edk2/parsers/fdf_parser.py

@@ -50,13 +50,16 @@ def GetNextLine(self):
         sline = self.StripComment(line)
 
         if (sline is None or len(sline) < 1):
+            self.DecrementLinesParsed()
             return self.GetNextLine()
 
         sline = self.ReplaceVariables(sline)
         if self.ProcessConditional(sline):
             # was a conditional so skip
+            self.DecrementLinesParsed()
             return self.GetNextLine()
         if not self.InActiveCode():
+            self.DecrementLinesParsed()
             return self.GetNextLine()
 
         self._BracketCount += sline.count("{")
@@ -67,9 +70,11 @@ def GetNextLine(self):
     def InsertLinesFromFile(self, file_path: str):
         """Adds additional lines to the Lines Attribute from the provided file."""
         with open(file_path, 'r') as lines_file:
-            self.Lines += reversed(lines_file.readlines())
+            lines = lines_file.readlines()
+            self.Lines += reversed(lines)
             # Back off the line count to ignore the include line itself.
             self.CurrentLine -= 1
+            self.PushTargetFile(file_path, len(lines))
 
     def ParseFile(self, filepath):
         """Parses the provided FDF file."""
@@ -79,12 +84,13 @@ def ParseFile(self, filepath):
         else:
             fp = filepath
         self.Path = fp
-        self.TargetFilePath = os.path.abspath(fp)
+        fp = os.path.abspath(fp)
         self.CurrentLine = 0
         self._f = open(fp, "r")
         self.Lines = self._f.readlines()
         self.Lines.reverse()
         self._f.close()
+        self.PushTargetFile(fp, len(self.Lines))
         self._BracketCount = 0
         InDefinesSection = False
         InFdSection = False
@@ -221,4 +227,5 @@ def ParseFile(self, filepath):
             elif sline.strip().lower().startswith('[rule.'):
                 InRuleSection = True
 
+            self.DecrementLinesParsed()
         self.Parsed = True

tests.unit/parsers/test_base_parser.py

@@ -7,11 +7,12 @@
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 ##
 
+import os
+import tempfile
 import unittest
+
 from edk2toollib.uefi.edk2.parsers.base_parser import BaseParser
 from edk2toollib.uefi.edk2.path_utilities import Edk2Path
-import tempfile
-import os
 
 
 class TestBaseParser(unittest.TestCase):
@@ -460,22 +461,22 @@ def test_emulator_conditional_not_in(self):
         parser.ResetParserState()
 
     def test_emulator_conditional_parens_order(self):
-        ''' Makes sure the parenthesis affect the order of expressions '''
+        '''Makes sure the parenthesis affect the order of expressions.'''
         parser = BaseParser("")
         self.assertFalse(parser.EvaluateConditional('!if TRUE OR FALSE AND FALSE'))
         self.assertTrue(parser.EvaluateConditional('!if TRUE OR (FALSE AND FALSE)'))
         parser.ResetParserState()
 
     def test_emulator_conditional_not_or(self):
-        ''' Makes sure we can use the not with other operators '''
+        '''Makes sure we can use the not with other operators.'''
         parser = BaseParser("")
         self.assertTrue(parser.EvaluateConditional('!if FALSE NOT OR FALSE'))
         self.assertFalse(parser.EvaluateConditional('!if TRUE NOT OR FALSE'))
         self.assertFalse(parser.EvaluateConditional('!if FALSE NOT OR TRUE'))
         self.assertFalse(parser.EvaluateConditional('!if TRUE NOT OR TRUE'))
 
     def test_emulator_conditional_not_it_all(self):
-        ''' Makes sure the parenthesis affect the order of expressions '''
+        '''Makes sure the parenthesis affect the order of expressions.'''
         parser = BaseParser("")
         self.assertTrue(parser.EvaluateConditional('!if NOT FALSE OR FALSE'))
         self.assertFalse(parser.EvaluateConditional('!if NOT TRUE OR FALSE'))
@@ -640,7 +641,7 @@ def test_find_path(self):
         # create target file
         os.makedirs(target_filedir)
         parser.WriteLinesToFile(target_filepath)
-        parser.TargetFilePath = target_filepath
+        parser.PushTargetFile(target_filepath, 0)
         # check if we can find the root
         root_found = parser.FindPath(root_file)
         self.assertEqual(root_found, root_filepath)

tests.unit/parsers/test_dsc_parser.py

@@ -7,9 +7,10 @@
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 ##
 
-import unittest
-import tempfile
 import os
+import tempfile
+import unittest
+
 from edk2toollib.uefi.edk2.parsers.dsc_parser import DscParser
 from edk2toollib.uefi.edk2.path_utilities import Edk2Path
 
@@ -30,7 +31,7 @@ def write_to_file(file_path, data):
         f.close()
 
     def test_dsc_include_single_file(self):
-        ''' This tests whether includes work properly '''
+        '''This tests whether includes work properly.'''
         workspace = tempfile.mkdtemp()
 
         file1_name = "file1.dsc"
@@ -53,7 +54,7 @@ def test_dsc_include_single_file(self):
         self.assertEqual(len(parser.GetAllDscPaths()), 2)  # make sure we have two dsc paths
 
     def test_dsc_include_missing_file(self):
-        ''' This tests whether includes work properly '''
+        '''This tests whether includes work properly.'''
         workspace = tempfile.mkdtemp()
 
         file1_name = "file1.dsc"
@@ -69,7 +70,7 @@ def test_dsc_include_missing_file(self):
             parser.ParseFile(file1_path)
 
     def test_dsc_include_missing_file_no_fail_mode(self):
-        ''' This tests whether includes work properly if no fail mode is on'''
+        '''This tests whether includes work properly if no fail mode is on.'''
         workspace = tempfile.mkdtemp()
 
         file1_name = "file1.dsc"
@@ -85,7 +86,7 @@ def test_dsc_include_missing_file_no_fail_mode(self):
         parser.ParseFile(file1_path)
 
     def test_dsc_parse_file_on_package_path(self):
-        ''' This tests whether includes work properly if no fail mode is on'''
+        '''This tests whether includes work properly if no fail mode is on.'''
         workspace = tempfile.mkdtemp()
         working_dir_name = "working"
         working2_dir_name = "working2"
@@ -113,7 +114,7 @@ def test_dsc_parse_file_on_package_path(self):
         self.assertEqual(parser.LocalVars["INCLUDED"], "TRUE")  # make sure we got the defines
 
     def test_dsc_include_relative_path(self):
-        ''' This tests whether includes work properly with a relative path'''
+        '''This tests whether includes work properly with a relative path.'''
         workspace = tempfile.mkdtemp()
         outside_folder = os.path.join(workspace, "outside")
         inside_folder = os.path.join(outside_folder, "inside")
@@ -154,3 +155,45 @@ def test_dsc_include_relative_path(self):
             self.assertEqual(parser.LocalVars["INCLUDED"], "TRUE")  # make sure we got the defines
         finally:
             os.chdir(cwd)
+
+def test_dsc_include_relative_paths2(tmp_path):
+    """This tests whether includes work properly with a relative path, when includes are not nested.
+
+    Directory setup
+    src
+    └─ Platforms
+       └─ PlatformPkg
+          ├─ PlatformPkg.dsc
+          └─ includes
+             ├─ PCDs1.dsc.inc
+             └─ PCDs2.dsc.inc
+
+    Base DSC
+    [PcdsFixedAtBuild]
+    gPlatformPkgTokenSpaceGuid.PcdSomething
+    !include includes/PCDs1.dsc.inc
+    !include includes/PCDs2.dsc.inc
+    """
+    pp = tmp_path / "Platforms"
+    pkg_dir = pp / "PlatformPkg"
+    dsc = pkg_dir / "PlatformPkg.dsc"
+    inc_dir = pkg_dir / "includes"
+
+    # Create Platforms, Platforms/PlatformPkg, Platforms/PlatformPkg/includes
+    inc_dir.mkdir(parents=True)
+
+    with open(inc_dir / "PCDs1.dsc.inc", "w") as f:
+        f.write("gPlatformPkgTokenSpaceGuid.PcdSomething1\n")
+
+    with open(inc_dir / "PCDs2.dsc.inc", "w") as f:
+        f.write("gPlatformPkgTokenSpaceGuid.PcdSomething2\n")
+
+    with open(dsc, "w") as f:
+        f.write("[PcdsFixedAtBuild]\n")
+        f.write("gPlatformPkgTokenSpaceGuid.PcdSomething\n")
+        f.write("!include includes/PCDs1.dsc.inc\n")
+        f.write("!include includes/PCDs2.dsc.inc\n")
+
+    parser = DscParser()
+    parser.SetEdk2Path(Edk2Path(str(tmp_path), [str(pp)]))
+    parser.ParseFile(dsc)