Adding app scaffolding for scrapein_ #8103
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| push: | |
| branches: | |
| - master | |
| paths: | |
| - 'components/**' | |
| jobs: | |
| publish-components: | |
| name: Publish Components to Pipedream Registry | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| - uses: pnpm/[email protected] | |
| with: | |
| version: 7.33.6 | |
| - name: Get pnpm store directory | |
| id: pnpm-cache | |
| run: | | |
| echo "::set-output name=pnpm_cache_dir::$(pnpm store path)" | |
| - uses: actions/cache@v4 | |
| name: Setup pnpm cache | |
| with: | |
| path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pnpm-store- | |
| - name: Install dependencies | |
| run: pnpm install -r --no-frozen-lockfile | |
| - name: Setup Node Env | |
| uses: actions/[email protected] | |
| with: | |
| node-version: 14 | |
| registry-url: https://registry.npmjs.org/ | |
| cache: 'pnpm' | |
| - name: Compile TypeScript | |
| run: npm run build | |
| - name: Install pd cli | |
| env: | |
| PD_API_KEY: ${{ secrets.PD_API_KEY }} | |
| PD_ORG_ID: ${{ secrets.PD_ORG_ID }} | |
| run: | | |
| curl -O https://cli.pipedream.com/linux/amd64/latest/pd.zip | |
| unzip pd.zip | |
| mkdir -p $HOME/.config/pipedream | |
| echo "api_key = $PD_API_KEY" > $HOME/.config/pipedream/config | |
| echo "org_id = $PD_ORG_ID" >> $HOME/.config/pipedream/config | |
| - name: Get Changed Files | |
| id: files | |
| uses: jitterbit/get-changed-files@v1 | |
| with: | |
| format: 'csv' | |
| - name: Publish and add to registry components/*.*js (that aren't .app.js files) | |
| id: publish | |
| env: | |
| PD_API_KEY: ${{ secrets.PD_API_KEY }} | |
| PD_ORG_ID: ${{ secrets.PD_ORG_ID }} | |
| ENDPOINT: ${{ secrets.ENDPOINT }} | |
| shell: bash {0} # don't fast fail | |
| run: | | |
| PD_URL=https://api.pipedream.com/v1/components/registry | |
| UNPUBLISHED="" | |
| PUBLISHED="" | |
| ERRORS="" | |
| SKIPPED="" | |
| mapfile -d ',' -t added_modified_renamed_files < <(printf '%s,%s' '${{ steps.files.outputs.added_modified }}' '${{ steps.files.outputs.renamed }}') | |
| for added_modified_file in "${added_modified_renamed_files[@]}"; do | |
| # starts with components, ends with .*js (e.g. .js and .mjs) and not app.js, | |
| # doesn't end with /common*.*js, and doesn't follow */common/ | |
| if [[ $added_modified_file == components/* ]] && [[ $added_modified_file == *.*js ]] && [[ $added_modified_file != *.app.*js ]] \ | |
| && [[ $added_modified_file != */common*.*js ]] && [[ $added_modified_file != */common/* ]] && [[ $added_modified_file != components/*/test-event.mjs ]] | |
| then | |
| echo "retrieving previous version for ${added_modified_file}" | |
| KEY=`echo $added_modified_file | tr '/' ' ' | awk '{ print $2 "-" $4 }'` | |
| GLOBAL_REGISTRY_COMP=`curl --silent "$PD_URL/$KEY" -H "Authorization: Bearer $PD_API_KEY" -H "Accept: application/json"` | |
| PREV_VERSION=`echo $GLOBAL_REGISTRY_COMP | jq -r ".data.version"` | |
| echo "attempting to publish ${added_modified_file}" | |
| PD_OUTPUT=`./pd publish ${added_modified_file} --json` | |
| if [ $? -eq 0 ] | |
| then | |
| KEY=`echo $PD_OUTPUT | jq -r ".key"` | |
| echo "published ${KEY}" | |
| echo "${KEY} will be added to the registry" | |
| curl "https://api.pipedream.com/graphql" -H "Content-Type: application/json" -H "Authorization: Bearer ${PD_API_KEY}" --data-binary $'{"query":"mutation setComponentRegistry($key: String!, $registry: Boolean!, $gitPath: String, $orgId: String){\\n setComponentRegistry(key: $key, registry: $registry, gitPath: $gitPath, orgId: $orgId){\\n savedComponent{\\n id\\n key\\n gitPath\\n }\\n }\\n}","variables":{"key":"'${KEY}'","registry":'true',"gitPath":"'${added_modified_file}'","orgId":"'${PD_ORG_ID}'"}}' | |
| [[ "$PREV_VERSION" == "null" ]] && PUBLISHED+="*${added_modified_file}" || PUBLISHED+="*${added_modified_file}~$PREV_VERSION" | |
| else | |
| ERROR=`echo $PD_OUTPUT | jq -r ".error" || $PD_OUTPUT` | |
| ERROR_MESSAGE="${ERROR} with ${added_modified_file}" | |
| echo $ERROR_MESSAGE | |
| ERRORS+="*${ERROR_MESSAGE}" | |
| UNPUBLISHED+="*${added_modified_file}" | |
| # add to array to spit out later | |
| fi | |
| else | |
| echo "${added_modified_file} will not be added to the registry" | |
| SKIPPED+="*${added_modified_file}" | |
| fi | |
| done | |
| # print out everything that didn't publish | |
| if [ ${#UNPUBLISHED[@]} -ne 0 ]; then | |
| echo "the following files were not published" | |
| printf '%s\n' "${UNPUBLISHED[@]}" | |
| fi | |
| # curl with form | |
| curl -X POST -d "type=javascript" --data-urlencode event@$GITHUB_EVENT_PATH -d "github=$(printenv | grep "GITHUB_*")" -d "skipped=${SKIPPED}" -d "errors=${ERRORS}" -d "unpublished=${UNPUBLISHED}" -d "published=${PUBLISHED}" $ENDPOINT | |
| if [ -n "$ERRORS" ]; then | |
| exit 1 | |
| fi | |
| - name: Remove pd cli config | |
| run: | | |
| rm $HOME/.config/pipedream/config | |
| publish-typescript-components: | |
| name: Publish TypeScript components | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/[email protected] | |
| - uses: pnpm/[email protected] | |
| with: | |
| version: 7.33.6 | |
| - name: Get pnpm store directory | |
| id: pnpm-cache | |
| run: | | |
| echo "::set-output name=pnpm_cache_dir::$(pnpm store path)" | |
| - uses: actions/cache@v4 | |
| name: Setup pnpm cache | |
| with: | |
| path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pnpm-store- | |
| - name: Setup Node Env | |
| uses: actions/[email protected] | |
| with: | |
| node-version: 14 | |
| registry-url: https://registry.npmjs.org/ | |
| cache: 'pnpm' | |
| - name: Install Dependencies | |
| run: pnpm install -r --no-frozen-lockfile | |
| - name: Install pd cli | |
| env: | |
| PD_API_KEY: ${{ secrets.PD_API_KEY }} | |
| PD_ORG_ID: ${{ secrets.PD_ORG_ID }} | |
| run: | | |
| curl -O https://cli.pipedream.com/linux/amd64/latest/pd.zip | |
| unzip pd.zip | |
| mkdir -p $HOME/.config/pipedream | |
| echo "api_key = $PD_API_KEY" > $HOME/.config/pipedream/config | |
| echo "org_id = $PD_ORG_ID" >> $HOME/.config/pipedream/config | |
| - name: Compile TypeScript | |
| id: compile | |
| run: npm run build > files.txt | |
| - name: Get Changed Files | |
| id: files | |
| uses: jitterbit/get-changed-files@v1 | |
| with: | |
| format: 'csv' | |
| - name: Publish TypeScript components (dry run) | |
| env: | |
| PD_API_KEY: ${{ secrets.PD_API_KEY }} | |
| PD_ORG_ID: ${{ secrets.PD_ORG_ID }} | |
| ENDPOINT: ${{ secrets.ENDPOINT }} | |
| shell: bash {0} # don't fast fail | |
| run: | | |
| IFS=$'\n' | |
| echo "The following files will be published on merge:" | |
| # Remove initial tsc output | |
| output_files=$(cat files.txt | sed 1,3d) | |
| for f in $output_files | |
| do | |
| echo "$f" | |
| done | |
| PD_URL=https://api.pipedream.com/v1/components/registry | |
| UNPUBLISHED="" | |
| PUBLISHED="" | |
| ERRORS="" | |
| SKIPPED="" | |
| mapfile -d ',' -t added_modified_renamed_files < <(printf '%s,%s' '${{ steps.files.outputs.added_modified }}' '${{ steps.files.outputs.renamed }}') | |
| # included in the components dir, ends with .ts and not app.ts, | |
| # doesn't end with /common*.ts, and doesn't follow */common/ | |
| for added_modified_file in "${added_modified_renamed_files[@]}"; | |
| do | |
| echo "Checking if $added_modified_file is a publishable ts file" | |
| if [[ $added_modified_file == components/* ]] && [[ $added_modified_file == *.ts ]] && [[ $added_modified_file != *.app.ts ]] \ | |
| && [[ $added_modified_file != */common*.ts ]] && [[ $added_modified_file != */common/* ]] | |
| then | |
| # XXX This is a hacky way to publish only TS components with changes. If a changed | |
| # file "path-a/path-b/c.ts" has a corresponding compiled output file | |
| # "path-a/dist/path-b/c.mjs", attempt to publish the output `.mjs` file. | |
| changed_output_file="" | |
| for f in $output_files; # check each output file for a match | |
| do | |
| # Replaces /dist/path/filename.mjs with /path/filename.ts | |
| maybe_source_file=$(echo "$f" | sed 's/\/dist\//\//;s/.mjs/\.ts/') | |
| if [[ ${maybe_source_file} == **/${added_modified_file} ]] | |
| then | |
| changed_output_file=${f} | |
| break | |
| fi | |
| done | |
| if [[ $changed_output_file == "" ]] | |
| then | |
| ERROR_MESSAGE="cannot find an output .mjs file with ${added_modified_file}" | |
| echo $ERROR_MESSAGE | |
| ERRORS+="*${ERROR_MESSAGE}" | |
| UNPUBLISHED+="*${added_modified_file}" | |
| else | |
| echo "retrieving previous version for ${added_modified_file}" | |
| KEY=`echo $added_modified_file | tr '/' ' ' | awk '{ print $2 "-" $4 }'` | |
| GLOBAL_REGISTRY_COMP=`curl --silent "$PD_URL/$KEY" -H "Authorization: Bearer $PD_API_KEY" -H "Accept: application/json"` | |
| PREV_VERSION=`echo $GLOBAL_REGISTRY_COMP | jq -r ".data.version"` | |
| echo "attempting to publish ${changed_output_file}" | |
| PD_OUTPUT=`./pd publish ${changed_output_file} --json` | |
| if [ $? -eq 0 ] | |
| then | |
| KEY=`echo $PD_OUTPUT | jq -r ".key"` | |
| echo "published ${KEY}" | |
| echo "${KEY} will be added to the registry" | |
| curl "https://api.pipedream.com/graphql" -H "Content-Type: application/json" -H "Authorization: Bearer ${PD_API_KEY}" --data-binary $'{"query":"mutation setComponentRegistry($key: String!, $registry: Boolean!, $gitPath: String, $orgId: String){\\n setComponentRegistry(key: $key, registry: $registry, gitPath: $gitPath, orgId: $orgId){\\n savedComponent{\\n id\\n key\\n gitPath\\n }\\n }\\n}","variables":{"key":"'${KEY}'","registry":'true',"gitPath":"'${added_modified_file}'","orgId":"'${PD_ORG_ID}'"}}' | |
| [[ "$PREV_VERSION" == "null" ]] && PUBLISHED+="*${changed_output_file}" || PUBLISHED+="*${changed_output_file}~$PREV_VERSION" | |
| else | |
| ERROR=`echo $PD_OUTPUT | jq -r ".error"` | |
| ERROR_MESSAGE="${ERROR} with ${changed_output_file}" | |
| echo $ERROR_MESSAGE | |
| ERRORS+="*${ERROR_MESSAGE}" | |
| UNPUBLISHED+="*${changed_output_file}" | |
| # add to array to spit out later | |
| fi | |
| fi | |
| else | |
| echo "${f} will not be added to the registry" | |
| SKIPPED+="*${f}" | |
| fi | |
| done | |
| # print out everything that didn't publish | |
| if [ ${#UNPUBLISHED[@]} -ne 0 ]; then | |
| echo "the following files were not published" | |
| printf '%s\n' "${UNPUBLISHED[@]}" | |
| fi | |
| # curl with form | |
| curl -X POST -d "type=typescript" --data-urlencode event@$GITHUB_EVENT_PATH -d "github=$(printenv | grep "GITHUB_*")" -d "skipped=${SKIPPED}" -d "errors=${ERRORS}" -d "unpublished=${UNPUBLISHED}" -d "published=${PUBLISHED}" $ENDPOINT | |
| if [ -n "$ERRORS" ]; then | |
| exit 1 | |
| fi | |
| unset IFS | |
| - name: Remove pd cli config | |
| run: | | |
| rm $HOME/.config/pipedream/config |