update to new embedding model #43
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow build and push a Docker container to Google Artifact Registry and deploy it on Google Kubernetes Engine when a commit is pushed to the "develop" branch | |
| # | |
| # To configure this workflow: | |
| # | |
| # 1. Ensure the required Google Cloud APIs are enabled in the project: | |
| # | |
| # Cloud Build cloudbuild.googleapis.com | |
| # Kubernetes Engine API container.googleapis.com | |
| # Artifact Registry artifactregistry.googleapis.com | |
| # | |
| # 2. Create a service account (if you don't have one) with the following fields: | |
| # | |
| # Service Account Name <PROJECT-NAME>-github-actions | |
| # Service Account ID <PROJECT-NAME>-github-actions | |
| # | |
| # 3. Ensure the service account have the required IAM permissions granted: | |
| # | |
| # Kubernetes Engine Developer | |
| # roles/container.developer (kubernetes engine developer) | |
| # | |
| # Artifact Registry | |
| # roles/artifactregistry.repoAdmin (artifact registry repository administrator) | |
| # roles/artifactregistry.admin (artifact registry administrator) | |
| # | |
| # Service Account | |
| # roles/iam.serviceAccountUser (act as the Cloud Run runtime service account) | |
| # | |
| # Basic Roles | |
| # roles/viewer (viewer) | |
| # | |
| # NOTE: You should always follow the principle of least privilege when assigning IAM roles | |
| # | |
| # 4. Ensure you have the following GitHub Secrets and Variables: | |
| # | |
| # GitHub Secrets | |
| # GCP_SA_KEY (Google Cloud Project Service Account Key) ref visit https://github.com/Datawheel/company/wiki/Setting-Up-a-Service-Account-for-Workflows#use-the-service-account-on-github-secrets | |
| # | |
| # GitHub Variables | |
| # GCP_PROJECT_ID (Google Cloud Project ID) | |
| # GCP_ARTIFACT_REGISTRY_NAME (Google Cloud Articaft Registry Repository Name) | |
| # GCP_ARTIFACT_REGISTRY_LOCATION (Google Cloud Artifact Registry Reposotiry Location) | |
| # | |
| # 5. Ensure you have the following GitHub Variables for each environment that you will set up: | |
| # | |
| # GitHub Variables | |
| # GCP_IMAGE_NAME (Docker Image Name) | |
| # GKE_APP_NAME (Google Kubernetes Engine Deployment Name) | |
| # GKE_APP_NAMESPACE (Google Kubernetes Engine Deployment Namespace) | |
| # GKE_CLUSTER (Google Kubernetes Engine Cluster Name) | |
| # GKE_ZONE (Google Kubernetes Engine Cluster Zone) | |
| # | |
| # Further reading: | |
| # Kubernetes Developer - https://cloud.google.com/iam/docs/understanding-roles#container.developer | |
| # Artifact Registry IAM permissions - https://cloud.google.com/artifact-registry/docs/access-control#roles | |
| # Container Registry vs Artifact Registry - https://cloud.google.com/blog/products/application-development/understanding-artifact-registry-vs-container-registry | |
| # Principle of least privilege - https://cloud.google.com/blog/products/identity-security/dont-get-pwned-practicing-the-principle-of-least-privilege | |
| # Deploy CloudRun Github Actions - https://github.com/google-github-actions/deploy-cloudrun | |
| name: "[GCP] Build API to Registry and Deploy via Helm" | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| paths: | |
| - .github/workflows/google-registry-api.yaml | |
| - helm/values-api.yaml | |
| - api/** | |
| env: | |
| GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| GCP_ARTIFACT_REGISTRY_NAME: ${{ vars.GCP_ARTIFACT_REGISTRY_NAME }} | |
| GCP_ARTIFACT_REGISTRY_LOCATION: ${{ vars.GCP_ARTIFACT_REGISTRY_LOCATION }} | |
| GCP_IMAGE_NAME: ${{ vars.GCP_IMAGE_NAME }} | |
| GKE_APP_NAME: ${{ vars.GKE_APP_NAME }} | |
| GKE_APP_NAMESPACE: ${{ vars.GKE_APP_NAMESPACE }} | |
| GKE_CLUSTER: ${{ vars.GKE_CLUSTER }} | |
| GKE_ZONE: ${{ vars.GKE_ZONE }} | |
| ACTIONS_ALLOW_UNSECURE_COMMANDS: true | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| environment: api-dev | |
| if: ${{ !contains(github.event.head_commit.message, '#update') }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| # Authentication via credentials json | |
| - name: Google Auth | |
| id: auth | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| project_id: '${{ env.GCP_PROJECT_ID }}' | |
| credentials_json: '${{ secrets.GCP_SA_KEY }}' | |
| # Install Cloud SDK | |
| - name: Set up Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| install_components: "beta" | |
| # Build image on Google Cloud Artifact Registry | |
| - name: Build Docker Image | |
| run: |- | |
| gcloud builds submit \ | |
| --quiet \ | |
| --timeout=40m \ | |
| --config=cloudbuild.yml \ | |
| --substitutions=_GCP_PROJECT_ID=${{ env.GCP_PROJECT_ID }},_GCP_ARTIFACT_REGISTRY_NAME=${{ env.GCP_ARTIFACT_REGISTRY_NAME }},_GCP_ARTIFACT_REGISTRY_LOCATION=${{ env.GCP_ARTIFACT_REGISTRY_LOCATION }},_GCP_IMAGE_NAME=${{ env.GCP_IMAGE_NAME }},_GCP_IMAGE_TAG=${{ github.sha }},_GCP_IMAGE_ENVIRONMENT=${{ env.GKE_APP_NAMESPACE }} | |
| working-directory: ./api | |
| deploy: | |
| needs: build | |
| runs-on: ubuntu-latest | |
| environment: api-dev | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| # Authentication via credentials json | |
| - name: Google Auth | |
| id: auth | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| project_id: '${{ vars.GCP_PROJECT_ID }}' | |
| credentials_json: '${{ secrets.GCP_SA_KEY }}' | |
| # Get google kubernetes engine credentials | |
| - name: Get GKE Credentials | |
| uses: google-github-actions/get-gke-credentials@v2 | |
| with: | |
| cluster_name: ${{ env.GKE_CLUSTER }} | |
| location: ${{ env.GKE_ZONE }} | |
| # Transform GitHub secrets to base64 encoded | |
| - name: Set encoded secret values | |
| run: | | |
| echo "ENCODED_OPENAI_KEY=$(echo -n "${{ secrets.OPENAI_KEY }}" | base64 | tr -d '\n')" >> $GITHUB_ENV | |
| echo "ENCODED_POSTGRES_HOST=$(echo -n "${{ secrets.POSTGRES_HOST }}" | base64 | tr -d '\n')" >> $GITHUB_ENV | |
| echo "ENCODED_POSTGRES_PASSWORD=$(echo -n "${{ secrets.POSTGRES_PASSWORD }}" | base64 | tr -d '\n')" >> $GITHUB_ENV | |
| echo "ENCODED_TESSERACT_API_SECRET=$(echo -n "${{ secrets.TESSERACT_API_SECRET }}" | base64 | tr -d '\n')" >> $GITHUB_ENV | |
| # Install Helm chart | |
| - name: Helm install | |
| uses: WyriHaximus/github-action-helm3@v2 | |
| with: | |
| exec: | | |
| helm upgrade --install --create-namespace \ | |
| --namespace ${{ env.GKE_APP_NAMESPACE }} \ | |
| --set app.environment=${{ env.GKE_APP_NAMESPACE }} \ | |
| --set app.release=${{ env.GKE_APP_NAMESPACE }} \ | |
| --set image.repository=${{ env.GCP_ARTIFACT_REGISTRY_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GCP_ARTIFACT_REGISTRY_NAME }}/${{ env.GCP_IMAGE_NAME }} \ | |
| --set image.tag=${{ github.sha }} \ | |
| --set nameOverride=${{ env.GKE_APP_NAME }} \ | |
| --set fullnameOverride=${{ env.GKE_APP_NAME }} \ | |
| --set configMap.DATA_PATH=${{ vars.DATA_PATH }} \ | |
| --set configMap.OLLAMA_API=${{ vars.OLLAMA_API }} \ | |
| --set configMap.POSTGRES_DB=${{ vars.POSTGRES_DB }} \ | |
| --set configMap.POSTGRES_USER=${{ vars.POSTGRES_USER }} \ | |
| --set configMap.TESSERACT_API=${{ vars.TESSERACT_API }} \ | |
| --set secrets.OPENAI_KEY=$ENCODED_OPENAI_KEY \ | |
| --set secrets.POSTGRES_HOST=$ENCODED_POSTGRES_HOST \ | |
| --set secrets.POSTGRES_PASSWORD=$ENCODED_POSTGRES_PASSWORD \ | |
| --set secrets.TESSERACT_API_SECRET=$ENCODED_TESSERACT_API_SECRET \ | |
| ${{ env.GKE_APP_NAME }} --values=./helm/values-api.yaml ./helm | |
| update: | |
| runs-on: ubuntu-latest | |
| environment: api-dev | |
| if: ${{ contains(github.event.head_commit.message, '#update') }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| # Authentication via credentials json | |
| - name: Google Auth | |
| id: auth | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| project_id: '${{ vars.GCP_PROJECT_ID }}' | |
| credentials_json: '${{ secrets.GCP_SA_KEY }}' | |
| # Get google kubernetes engine credentials | |
| - name: Get GKE Credentials | |
| uses: google-github-actions/get-gke-credentials@v2 | |
| with: | |
| cluster_name: ${{ env.GKE_CLUSTER }} | |
| location: ${{ env.GKE_ZONE }} | |
| # Transform GitHub secrets to base64 encoded | |
| - name: Set encoded secret values | |
| run: | | |
| echo "ENCODED_OPENAI_KEY=$(echo -n "${{ secrets.OPENAI_KEY }}" | base64 | tr -d '\n')" >> $GITHUB_ENV | |
| echo "ENCODED_POSTGRES_HOST=$(echo -n "${{ secrets.POSTGRES_HOST }}" | base64 | tr -d '\n')" >> $GITHUB_ENV | |
| echo "ENCODED_POSTGRES_PASSWORD=$(echo -n "${{ secrets.POSTGRES_PASSWORD }}" | base64 | tr -d '\n')" >> $GITHUB_ENV | |
| echo "ENCODED_TESSERACT_API_SECRET=$(echo -n "${{ secrets.TESSERACT_API_SECRET }}" | base64 | tr -d '\n')" >> $GITHUB_ENV | |
| # Install Helm chart | |
| - name: Helm install | |
| uses: WyriHaximus/github-action-helm3@v2 | |
| with: | |
| exec: | | |
| helm upgrade --install --create-namespace \ | |
| --namespace ${{ env.GKE_APP_NAMESPACE }} \ | |
| --set app.environment=${{ env.GKE_APP_NAMESPACE }} \ | |
| --set app.release=${{ env.GKE_APP_NAMESPACE }} \ | |
| --set image.repository=${{ env.GCP_ARTIFACT_REGISTRY_LOCATION }}-docker.pkg.dev/${{ env.GCP_PROJECT_ID }}/${{ env.GCP_ARTIFACT_REGISTRY_NAME }}/${{ env.GCP_IMAGE_NAME }} \ | |
| --set image.tag=${{ env.GKE_APP_NAMESPACE }} \ | |
| --set nameOverride=${{ env.GKE_APP_NAME }} \ | |
| --set fullnameOverride=${{ env.GKE_APP_NAME }} \ | |
| --set configMap.DATA_PATH=${{ vars.DATA_PATH }} \ | |
| --set configMap.OLLAMA_API=${{ vars.OLLAMA_API }} \ | |
| --set configMap.POSTGRES_DB=${{ vars.POSTGRES_DB }} \ | |
| --set configMap.POSTGRES_USER=${{ vars.POSTGRES_USER }} \ | |
| --set configMap.TESSERACT_API=${{ vars.TESSERACT_API }} \ | |
| --set secrets.OPENAI_KEY=$ENCODED_OPENAI_KEY \ | |
| --set secrets.POSTGRES_HOST=$ENCODED_POSTGRES_HOST \ | |
| --set secrets.POSTGRES_PASSWORD=$ENCODED_POSTGRES_PASSWORD \ | |
| --set secrets.TESSERACT_API_SECRET=$ENCODED_TESSERACT_API_SECRET \ | |
| ${{ env.GKE_APP_NAME }} --values=./helm/values-api.yaml ./helm |