Fix BytePair special tokens tokenization

.github/dependabot.yml

@@ -21,3 +21,6 @@ updates:
       python:
         patterns:
           - "*"
+    ignore:
+        # ignore all updates for JAX GPU due to cuda version issue
+      - dependency-name: "jax[cuda12_pip]"

.github/workflows/auto-assignment.yml

@@ -0,0 +1,21 @@
+name: auto-assignment
+on:
+  issues:
+    types:
+      - opened
+
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+
+jobs:
+  welcome:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            const script = require('./\.github/workflows/scripts/auto-assignment.js')
+            script({github, context})

.github/workflows/labeler.yaml

@@ -0,0 +1,42 @@
+# Copyright 2024 Google LLC. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This workflow automatically identifies issues and pull requests (PRs)
+# related to Gemma. It searches for the keyword "Gemma" (case-insensitive)
+# in both the title and description of the issue/PR.  If a match is found,
+# the workflow adds the label 'Gemma' to the issue/PR.
+
+name: 'Labeler'
+on:
+  issues:
+    types: [edited, opened]
+  pull_request_target:
+    types: [opened, edited]
+
+permissions:
+  contents: read
+  issues: write
+  pull-requests: write
+
+jobs:
+  welcome:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            const script = require('./\.github/workflows/scripts/labeler.js')
+            script({github, context})

.github/workflows/scripts/auto-assignment.js

@@ -0,0 +1,43 @@
+/** Automatically assign issues and PRs to users in the `assigneesList` 
+ *  on a rotating basis.
+
+  @param {!object}
+    GitHub objects can call GitHub APIs using their built-in library functions.
+    The context object contains issue and PR details.
+*/
+
+module.exports = async ({ github, context }) => {
+  let issueNumber;
+  let assigneesList;
+  // Is this an issue? If so, assign the issue number. Otherwise, assign the PR number.
+  if (context.payload.issue) {
+    //assignee List for issues. 
+    assigneesList = ["SuryanarayanaY", "sachinprasadhs"];
+    issueNumber = context.payload.issue.number;
+  } else {
+    //assignee List for PRs. 
+    assigneesList = [mattdangerw];
+    issueNumber = context.payload.number;
+  }
+  console.log("assignee list", assigneesList);
+  console.log("entered auto assignment for this issue:  ", issueNumber);
+  if (!assigneesList.length) {
+    console.log("No assignees found for this repo.");
+    return;
+  }
+  let noOfAssignees = assigneesList.length;
+  let selection = issueNumber % noOfAssignees;
+  let assigneeForIssue = assigneesList[selection];
+
+  console.log(
+    "issue Number = ",
+    issueNumber + " , assigning to: ",
+    assigneeForIssue
+  );
+  return github.rest.issues.addAssignees({
+    issue_number: context.issue.number,
+    owner: context.repo.owner,
+    repo: context.repo.repo,
+    assignees: [assigneeForIssue],
+  });
+};

.github/workflows/scripts/labeler.js

@@ -0,0 +1,53 @@
+/*
+Copyright 2024 Google LLC. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+
+/**
+ * Invoked from labeler.yaml file to add
+ * label 'Gemma' to the issue and PR for which have gemma keyword present.
+ * @param {!Object.<string,!Object>} github contains pre defined functions.
+ *  context Information about the workflow run.
+ */
+
+module.exports = async ({ github, context }) => {
+    const issue_title = context.payload.issue ?  context.payload.issue.title : context.payload.pull_request.title
+    let issue_description = context.payload.issue ? context.payload.issue.body : context.payload.pull_request.body
+    const issue_number = context.payload.issue ? context.payload.issue.number : context.payload.pull_request.number
+    const keyword_label =  {
+         gemma:'Gemma'
+    }
+    const labelsToAdd = []
+    console.log(issue_title,issue_description,issue_number)
+    if (issue_description==null)
+    {
+      issue_description = ''
+    }
+
+    for(const [keyword, label] of Object.entries(keyword_label)){
+     if(issue_title.toLowerCase().indexOf(keyword) !=-1 || issue_description.toLowerCase().indexOf(keyword) !=-1 ){
+        console.log(`'${keyword}'keyword is present inside the title or description. Pushing label '${label}' to row.`)
+        labelsToAdd.push(label)
+    }
+   }
+   if(labelsToAdd.length > 0){
+    console.log(`Adding labels ${labelsToAdd} to the issue '#${issue_number}'.`)
+     github.rest.issues.addLabels({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        issue_number: context.issue.number,
+        labels: labelsToAdd
+     })
+   }
+};

.github/workflows/stale-issue-pr.yml

@@ -0,0 +1,50 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "30 1 * * *"
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - name: Awaiting response issues
+        uses: actions/stale@v9
+        with:
+          days-before-issue-stale: 14
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          # reason for closed the issue default value is not_planned
+          close-issue-reason: completed
+          only-labels: "stat:awaiting response from contributor"
+          stale-issue-message: > 
+            This issue is stale because it has been open for 14 days with no activity.
+            It will be closed if no further activity occurs. Thank you.
+          # List of labels to remove when issues/PRs unstale. 
+          labels-to-remove-when-unstale: "stat:awaiting response from contributor"
+          close-issue-message: >
+            This issue was closed because it has been inactive for 28 days.
+            Please reopen if you'd like to work on this further.
+          days-before-pr-stale: 14
+          days-before-pr-close: 14
+          stale-pr-message: "This PR is stale because it has been open for 14 days with no activity. It will be closed if no further activity occurs. Thank you."
+          close-pr-message: "This PR was closed because it has been inactive for 28 days. Please reopen if you'd like to work on this further."
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Contribution issues
+        uses: actions/stale@v9
+        with:
+          days-before-issue-stale: 180
+          days-before-issue-close: 365
+          stale-issue-label: "stale"
+          # reason for closed the issue default value is not_planned
+          close-issue-reason: not_planned
+          any-of-labels: "stat:contributions welcome,good first issue"
+          # List of labels to remove when issues/PRs unstale. 
+          labels-to-remove-when-unstale: "stat:contributions welcome,good first issue"
+          stale-issue-message: > 
+            This issue is stale because it has been open for 180 days with no activity.
+            It will be closed if no further activity occurs. Thank you.
+          close-issue-message: >
+            This issue was closed because it has been inactive for more than 1 year.
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.kokoro/github/ubuntu/gpu/build.sh

@@ -1,9 +1,20 @@
 set -e
-set -x
 
-cd "${KOKORO_ROOT}/"
+export KAGGLE_KEY="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_secret_key)"
+export KAGGLE_USERNAME="$(cat ${KOKORO_KEYSTORE_DIR}/73361_keras_kaggle_username)"
+
+if [[ -z "${KAGGLE_KEY}" ]]; then
+   echo "KAGGLE_KEY is NOT set"
+   exit 1
+fi
 
-sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1
+if [[ -z "${KAGGLE_USERNAME}" ]]; then
+   echo "KAGGLE_USERNAME is NOT set"
+   exit 1
+fi
+
+set -x
+cd "${KOKORO_ROOT}/"
 
 PYTHON_BINARY="/usr/bin/python3.9"
 
@@ -24,23 +35,25 @@ pip install -U pip setuptools psutil
 if [ "${KERAS2:-0}" == "1" ]
 then
    echo "Keras2 detected."
-   pip install -r requirements-common.txt --progress-bar off
-   pip install tensorflow-text==2.15 tensorflow[and-cuda]~=2.15 keras-core
+   pip install -r requirements-common.txt --progress-bar off --timeout 1000
+   pip install tensorflow-text==2.15 tensorflow[and-cuda]~=2.15 keras-core \
+      --timeout 1000
 
 elif [ "$KERAS_BACKEND" == "tensorflow" ]
 then
    echo "TensorFlow backend detected."
-   pip install -r requirements-tensorflow-cuda.txt --progress-bar off
+   pip install -r requirements-tensorflow-cuda.txt --progress-bar off \
+      --timeout 1000
 
 elif [ "$KERAS_BACKEND" == "jax" ]
 then
    echo "JAX backend detected."
-   pip install -r requirements-jax-cuda.txt --progress-bar off
+   pip install -r requirements-jax-cuda.txt --progress-bar off --timeout 1000
 
 elif [ "$KERAS_BACKEND" == "torch" ]
 then
    echo "PyTorch backend detected."
-   pip install -r requirements-torch-cuda.txt --progress-bar off
+   pip install -r requirements-torch-cuda.txt --progress-bar off --timeout 1000
 fi
 
 pip install --no-deps -e "." --progress-bar off

.kokoro/github/ubuntu/gpu/jax/continuous.cfg

@@ -12,5 +12,23 @@ env_vars: {
    value: "jax"
 }
 
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73361
+      keyname: "keras_kaggle_username"
+    }
+  }
+}
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73361
+      keyname: "keras_kaggle_secret_key"
+    }
+  }
+}
+
 # Set timeout to 60 mins from default 180 mins
 timeout_mins: 60

.kokoro/github/ubuntu/gpu/jax/presubmit.cfg

@@ -12,5 +12,23 @@ env_vars: {
    value: "jax"
 }
 
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73361
+      keyname: "keras_kaggle_username"
+    }
+  }
+}
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73361
+      keyname: "keras_kaggle_secret_key"
+    }
+  }
+}
+
 # Set timeout to 60 mins from default 180 mins
 timeout_mins: 60

.kokoro/github/ubuntu/gpu/keras2/continuous.cfg

@@ -12,5 +12,23 @@ env_vars: {
    value: "1"
 }
 
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73361
+      keyname: "keras_kaggle_username"
+    }
+  }
+}
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73361
+      keyname: "keras_kaggle_secret_key"
+    }
+  }
+}
+
 # Set timeout to 60 mins from default 180 mins
 timeout_mins: 60

.kokoro/github/ubuntu/gpu/keras2/presubmit.cfg

@@ -12,5 +12,23 @@ env_vars: {
    value: "1"
 }
 
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73361
+      keyname: "keras_kaggle_username"
+    }
+  }
+}
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73361
+      keyname: "keras_kaggle_secret_key"
+    }
+  }
+}
+
 # Set timeout to 60 mins from default 180 mins
 timeout_mins: 60