Add the helper function in Keras setup to clean up special tokens in GTP2 and SentencePiece tokenizer. #3871
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2021 Google LLC | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # ============================================================================== | |
| # Config file for GitHub Actions for CI. | |
| name: CI | |
| # Run on pushes and pull requests on main and dev branches. | |
| on: | |
| push: | |
| branches: [main, dev] | |
| pull_request: | |
| branches: [main, dev] | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: null | |
| jobs: | |
| build-and-test: | |
| name: Build and test (Python ${{ matrix.python-version }}, Node ${{ matrix.node-version }}) | |
| strategy: | |
| matrix: | |
| node-version: [18] | |
| python-version: ["3.10", "3.11"] | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Python dependencies | |
| run: python -m pip install -r requirements.txt | |
| - name: Test Python library | |
| run: | | |
| python -m pip install pytest | |
| pytest -v | |
| - name: Setup Node ${{ matrix.node-version }} | |
| uses: actions/setup-node@v2 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Build LIT client | |
| run: pushd lit_nlp && yarn && yarn build && popd | |
| # TODO(b/235626637): Add front-end unit and e2e tests | |
| docker-build-and-test: | |
| name: Build docker image | |
| needs: build-and-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build Docker image | |
| uses: docker/build-push-action@v4 | |
| with: | |
| tags: lit-nlp:ci-${{ github.sha }} |