Merge branch 'master' into mvafin/pt_fe/perf_refactor

.github/actions/common/action_utils.py

@@ -0,0 +1,17 @@
+import logging
+import os
+
+
+def init_logger():
+    logging.basicConfig(level=logging.DEBUG,
+                        format='%(asctime)s %(name)-15s %(levelname)-8s %(message)s',
+                        datefmt='%m-%d-%Y %H:%M:%S')
+
+
+def set_github_output(name: str, value: str, github_output_var_name: str = 'GITHUB_OUTPUT'):
+    """Sets output variable for a GitHub Action"""
+    logger = logging.getLogger(__name__)
+    # In an environment variable "GITHUB_OUTPUT" GHA stores path to a file to write outputs to
+    with open(os.environ.get(github_output_var_name), 'a+') as file:
+        logger.info(f"Add {name}={value} to {github_output_var_name}")
+        print(f'{name}={value}', file=file)

.github/actions/common/artifact_utils.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+from .constants import EventType
+
+
+def add_common_args(parser: argparse.ArgumentParser):
+    parser.add_argument('-s', '--commit_sha', help='Commit hash for which artifacts were generated', required=True)
+    parser.add_argument('-b', '--branch_name', help='Name of GitHub branch', required=False,
+                        default=os.getenv('GITHUB_BASE_REF') or os.getenv('GITHUB_REF_NAME'))
+    parser.add_argument('-e', '--event_name', help='Name of GitHub event', required=False,
+                        default=os.getenv('GITHUB_EVENT_NAME'))
+    parser.add_argument('--storage_dir', help='Subdirectory name for artifacts, same as product type', required=True)
+    parser.add_argument('--storage_root', help='Root path of the artifacts storage', required=False,
+                        default=os.getenv('ARTIFACTS_SHARE'))
+
+
+def get_event_type(event_name: str = os.getenv('GITHUB_EVENT_NAME')) -> str:
+    return EventType.pre_commit.value if event_name == 'pull_request' else EventType.commit.value
+
+
+def get_storage_event_dir(storage_root: str | Path, branch_name: str, event_name: str,
+                          product_name: str = 'dldt') -> Path:
+    """ Returns path to stored artifacts for a given branch and event """
+    event_type = get_event_type(event_name)
+    storage_branch_dir = Path(storage_root) / product_name / branch_name / event_type
+    return storage_branch_dir
+
+
+def get_storage_dir(product_type: str, commit_hash: str, storage_root: str | Path, branch_name: str, event_name: str,
+                    product_name: str = 'dldt') -> Path:
+    """ Returns full path to stored artifacts for a given product type """
+
+    # TODO: return, once we decide to get rid of post-commit and choose artifacts generated for a merged PR in queue?
+    # merge_queue_matcher = re.search(r'gh-readonly-queue/(.*?)/pr-', branch_name)
+    # if merge_queue_matcher:
+    #     branch_name = merge_queue_matcher.group(1)
+
+    storage_event_dir = get_storage_event_dir(storage_root, branch_name, event_name, product_name)
+    storage = storage_event_dir / commit_hash / product_type
+    return storage
+
+
+def get_latest_artifacts_link(product_type: str, storage_root: str | Path, branch_name: str, event_name: str,
+                              product_name: str = 'dldt') -> Path:
+    """ Returns path to latest available artifacts for a given branch, event and product type """
+    storage_branch_dir = get_storage_event_dir(storage_root, branch_name, event_name, product_name)
+    latest_artifacts_for_branch = storage_branch_dir / f"latest_{product_type}.txt"
+    return Path(latest_artifacts_for_branch)

.github/actions/common/constants.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class EventType(Enum):
+    pre_commit = 'pre_commit'
+    commit = 'commit'
+
+# TODO: add enum for product type to validate it

.github/actions/create_manifest/action.yml

@@ -19,6 +19,9 @@ inputs:
   build_type:
     description: "Build type: release | debug | release_with_debug"
     required: true
+  trigger_repo_sha:
+    description: "Commit hash of a trigger repo. If not set - taken from github context"
+    required: false
 
 
 runs:
@@ -40,5 +43,5 @@ runs:
         --save_to "${{ inputs.save_to }}" --product_type "${{ inputs.product_type }}" -r "${{ inputs.repos }}"
       env:
         BASE_SHA: ${{ github.event.pull_request.base.sha }}
-        PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        TRIGGER_REPO_SHA: ${{ inputs.trigger_repo_sha || github.event.pull_request.head.sha || github.sha }}
         ACTION_PATH: ${{ runner.os == 'Windows' && '$env:GITHUB_ACTION_PATH' || '$GITHUB_ACTION_PATH' }}

.github/actions/create_manifest/create_manifest.py

@@ -7,9 +7,13 @@
 from pathlib import Path
 import re
 import git
+import sys
 
 from manifest_manager import Manifest, Repository, Component
 
+sys.path.append(str(Path(__file__).parents[1]))
+from common import artifact_utils
+
 
 def parse_args():
     parser = argparse.ArgumentParser(description='Creates manifest with product and repositories version')
@@ -50,7 +54,7 @@ def get_repo_data(repo_dir: str | Path) -> dict:
 
     branch = os.getenv('GITHUB_REF') if is_trigger_repo else repo.references[0].name
     target_branch = os.getenv('GITHUB_BASE_REF') if is_trigger_repo else None
-    revision = os.getenv('PR_HEAD_SHA') or os.getenv('GITHUB_SHA') if is_trigger_repo else repo.head.commit.hexsha
+    revision = os.getenv('TRIGGER_REPO_SHA') if is_trigger_repo else repo.head.commit.hexsha
     target_revision = os.getenv('BASE_SHA') if is_trigger_repo else None
     # Commit time of a merge commit (in case of PR merged to target)
     # TODO: Save commit time of a head commit in PR as well?
@@ -115,7 +119,7 @@ def main():
     args = parse_args()
 
     event_name = args.event_name or os.getenv('GITHUB_EVENT_NAME')
-    event_type = 'pre_commit' if event_name == 'pull_request' else 'commit'
+    event_type = artifact_utils.get_event_type(event_name)
 
     repos = args.repos.split()
     manifest = generate_manifest(repos, args.product_type, event_type, args.build_type, args.target_arch)

.github/actions/restore_artifacts/action.yml

@@ -0,0 +1,58 @@
+name: 'Restore artifacts'
+description: 'Take artifacts from a shared drive and load them back to GitHub'
+inputs:
+  storage_dir:
+    description: "Subdirectory name for artifacts, same as product type"
+    required: true
+  storage_root:
+    description: "Root path of the artifacts storage"
+    required: true
+  event_name:
+    description: "GitHub event name. If not set - taken from github context"
+    required: false
+  branch_name:
+    description: "Target branch name. If not set - taken from github context"
+    required: false
+  trigger_repo_sha:
+    description: "Commit hash of a trigger repo. If not set - taken from github context"
+    required: false
+  artifacts_key:
+    description: "A key under which to upload the artifacts to storage, product type by default"
+    required: false
+
+
+outputs:
+  artifacts_storage_path:
+    description: "Path on a shared drive where the artifacts are stored"
+    value: ${{ steps.restore.outputs.artifacts_storage_path }}
+  restored_artifacts_key:
+    description: "Key under which the restored artifacts, if any, were uploaded to GitHub"
+    value: ${{ steps.restore.outputs.restored_artifacts_key }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Restore artifacts
+      id: restore
+      if: "!inputs.artifacts"
+      shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }}
+      run: >-
+        python3 ${{ env.ACTION_PATH }}/restore_artifacts.py
+        -t "${{ env.GITHUB_WORKSPACE }}/${{ inputs.storage_dir }}"
+        --storage_dir "${{ inputs.storage_dir }}" --storage_root "${{ inputs.storage_root }}"
+        -s "${{ inputs.trigger_repo_sha || github.event.pull_request.head.sha || github.sha }}"
+        ${{ inputs.branch_name && env.BRANCH_PARAM || '' }}
+        ${{ inputs.event_name && env.EVENT_PARAM || '' }}
+      env:
+        ACTION_PATH: ${{ runner.os == 'Windows' && '$env:GITHUB_ACTION_PATH' || '$GITHUB_ACTION_PATH' }}
+        GITHUB_WORKSPACE: ${{ runner.os == 'Windows' && '$env:GITHUB_WORKSPACE' || '$GITHUB_WORKSPACE' }}
+        BRANCH_PARAM: "-b ${{ inputs.branch_name }}"
+        EVENT_PARAM: "-e ${{ inputs.event_name }}"
+
+    - name: Upload artifacts
+      if: "!inputs.artifacts"
+      uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+      with:
+        name: ${{ inputs.artifacts_key || steps.restore.outputs.restored_artifacts_key }}
+        path: ${{ steps.restore.outputs.artifacts_workspace_path }}
+        if-no-files-found: 'error'

.github/actions/restore_artifacts/restore_artifacts.py

@@ -0,0 +1,52 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import argparse
+import logging
+import shutil
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parents[1]))
+from common import artifact_utils, action_utils
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Returns a path to artifacts for a given revision on a shared drive')
+    artifact_utils.add_common_args(parser)
+    parser.add_argument('-t', '--target_dir', type=str, help='Path to a dir in a workspace to download artifacts into',
+                        required=True)
+    parser.add_argument('-k', '--artifact_key', type=str,
+                        help='A key under which to upload the artifacts to storage, product type by default',
+                        required=False)
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    action_utils.init_logger()
+    logger = logging.getLogger(__name__)
+    args = parse_args()
+
+    if args.commit_sha == 'latest':
+        latest_artifacts_link = artifact_utils.get_latest_artifacts_link(args.storage_dir, args.storage_root,
+                                                                         args.branch_name, args.event_name)
+        storage = Path(args.storage_root) / latest_artifacts_link.read_text()
+    else:
+        storage = artifact_utils.get_storage_dir(args.storage_dir, args.commit_sha, args.storage_root, args.branch_name,
+                                                 args.event_name)
+
+    action_utils.set_github_output("artifacts_storage_path", str(storage))
+    logger.info(f"Artifacts are taken from here: {storage}")
+
+    shutil.copytree(storage, args.target_dir, dirs_exist_ok=True)
+    logger.info(f"Artifacts are copied here: {args.target_dir}")
+
+    action_utils.set_github_output("artifacts_workspace_path", args.target_dir)
+    action_utils.set_github_output("restored_artifacts_key", args.artifact_key or args.storage_dir)
+
+
+if __name__ == '__main__':
+    main()

.github/actions/store_artifacts/action.yml

@@ -5,17 +5,29 @@ inputs:
     description: "Multi-line list of artifacts to store"
     required: true
   storage_dir:
-    description: "Directory name to store artifacts in"
+    description: "Subdirectory name for stored artifacts, same as product type"
     required: true
   storage_root:
-    description: "Root path of the storage to place artifacts to"
+    description: "Root path to place artifacts to"
     required: true
+  event_name:
+    description: "GitHub event name. If not set - taken from github context"
+    required: false
+  branch_name:
+    description: "Target branch name. If not set - taken from github context"
+    required: false
+  trigger_repo_sha:
+    description: "Commit hash of a trigger repo. If not set - taken from github context"
+    required: false
 
 
 outputs:
   artifacts_storage_path:
-    description: "Path where the artifacts are stored"
-    value: ${{ steps.copy_artifacts.outputs.artifacts_storage_path }}
+    description: "Path on a shared drive where the artifacts are stored"
+    value: ${{ steps.store.outputs.artifacts_storage_path || steps.restore.outputs.artifacts_storage_path }}
+  restored_artifacts_key:
+    description: "Key under which the restored artifacts, if any, were uploaded to GitHub"
+    value: ${{ steps.restore.outputs.restored_artifacts_key }}
 
 runs:
   using: "composite"
@@ -27,13 +39,18 @@ runs:
       env:
         ACTION_PATH: ${{ runner.os == 'Windows' && '$env:GITHUB_ACTION_PATH' || '$GITHUB_ACTION_PATH' }}
 
-    - name: 'Copy artifacts'
-      id: copy_artifacts
+    - name: Store artifacts
+      id: store
+      if: inputs.artifacts
       shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }}
       run: >-
         python3 ${{ env.ACTION_PATH }}/store_artifacts.py
         --storage_dir "${{ inputs.storage_dir }}" --storage_root "${{ inputs.storage_root }}"
-        -a "${{ inputs.artifacts }}"
+        -a "${{ inputs.artifacts }}" 
+        -s "${{ inputs.trigger_repo_sha || github.event.pull_request.head.sha || github.sha }}"
+        ${{ inputs.branch_name && env.BRANCH_PARAM || '' }}
+        ${{ inputs.event_name && env.EVENT_PARAM || '' }}
       env:
-        PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         ACTION_PATH: ${{ runner.os == 'Windows' && '$env:GITHUB_ACTION_PATH' || '$GITHUB_ACTION_PATH' }}
+        BRANCH_PARAM: "-b ${{ inputs.branch_name }}"
+        EVENT_PARAM: "-e ${{ inputs.event_name }}"

.github/actions/store_artifacts/store_artifacts.py

@@ -6,41 +6,23 @@
 import argparse
 import logging
 import os
-import re
 import sys
-import git
 import shutil
 from contextlib import contextmanager
 from pathlib import Path
 
+sys.path.append(str(Path(__file__).parents[1]))
+from common import artifact_utils, action_utils
+
 
 def parse_args():
-    parser = argparse.ArgumentParser(description='Returns product components changed in a given PR or commit')
-    parser.add_argument('-e', '--event_name', help='Name of GitHub event', required=False)
-    parser.add_argument('-b', '--branch_name', help='Name of GitHub branch', required=False)
-    parser.add_argument('-s', '--commit_sha', help='Commit hash for which artifacts were generated', required=False)
+    parser = argparse.ArgumentParser(description='Stores given artifacts on a shared drive')
     parser.add_argument('-a', '--artifacts', type=str, help='Paths to artifacts to store (files/dirs)', required=True)
-    parser.add_argument('--storage_dir', help='Directory name to store artifacts in', required=True)
-    parser.add_argument('--storage_root', help='Root path of the storage to place artifacts to', required=True)
+    artifact_utils.add_common_args(parser)
     args = parser.parse_args()
     return args
 
 
-def init_logger():
-    logging.basicConfig(level=logging.DEBUG,
-                        format='%(asctime)s %(name)-15s %(levelname)-8s %(message)s',
-                        datefmt='%m-%d-%Y %H:%M:%S')
-
-
-def set_github_output(name: str, value: str, github_output_var_name: str = 'GITHUB_OUTPUT'):
-    """Sets output variable for a GitHub Action"""
-    logger = logging.getLogger(__name__)
-    # In an environment variable "GITHUB_OUTPUT" GHA stores path to a file to write outputs to
-    with open(os.environ.get(github_output_var_name), 'a+') as file:
-        logger.info(f"Add {name}={value} to {github_output_var_name}")
-        print(f'{name}={value}', file=file)
-
-
 @contextmanager
 def preserve_stats_context():
     """
@@ -80,24 +62,14 @@ def rotate_dir(directory: Path) -> bool:
 
 
 def main():
-    init_logger()
+    action_utils.init_logger()
     logger = logging.getLogger(__name__)
     args = parse_args()
 
-    event_name = args.event_name or os.getenv('GITHUB_EVENT_NAME')
-    branch_name = args.branch_name or os.getenv('GITHUB_BASE_REF') or os.getenv('GITHUB_REF_NAME')
-
-    # TODO: return, once we decide to get rid of post-commit and choose artifacts generated for a merged PR in queue?
-    # merge_queue_matcher = re.search(r'gh-readonly-queue/(.*?)/pr-', branch_name)
-    # if merge_queue_matcher:
-    #     branch_name = merge_queue_matcher.group(1)
-
-    commit_hash = args.commit_sha or os.getenv('PR_HEAD_SHA') or os.getenv('GITHUB_SHA')
-    event_type = 'pre_commit' if event_name == 'pull_request' else 'commit'
     storage_root = args.storage_root or os.getenv('ARTIFACTS_SHARE')
-
-    storage = Path(storage_root) / 'dldt' / branch_name / event_type / commit_hash / args.storage_dir
-    set_github_output("artifacts_storage_path", str(storage))
+    storage = artifact_utils.get_storage_dir(args.storage_dir, args.commit_sha, args.storage_root, args.branch_name,
+                                             args.event_name)
+    action_utils.set_github_output("artifacts_storage_path", str(storage))
 
     logger.info(f"Storing artifacts to {storage}")
     rotate_dir(storage)  # TODO: use more stable approach to handle storing artifacts from re-runs
@@ -124,6 +96,14 @@ def main():
         with open(storage / 'workflow_link.txt', 'w') as file:
             file.write(workflow_link)
 
+    latest_artifacts_for_branch = artifact_utils.get_latest_artifacts_link(args.storage_dir, args.storage_root,
+                                                                           args.branch_name, args.event_name)
+    # Overwrite path to "latest" built artifacts only if a given commit is the head of a given branch
+    if args.event_name != 'pull_request' and args.commit_sha == os.getenv('GITHUB_SHA'):
+        # TODO: lock to avoid corruption in case of a parallel build (unlikely event for now, but still)
+        with open(latest_artifacts_for_branch, 'w') as file:
+            file.write(str(storage.relative_to(storage_root)))
+
     logger.debug(f"Copying finished")
     (storage / 'copying_finished').touch()
     if error_found:

.github/dockerfiles/docker_tag

@@ -1 +1 @@
-pr-25234
+pr-25992