chore: addint robots no index by host #237

	name: Documentation development pipeline
	on:
	push:
	branches:
	- dev
	jobs:
	deploy:
	name: Development publising
	runs-on: ubuntu-latest
	steps:
	- name: CHECKOUT project
	uses: actions/checkout@v4
	with:
	repository: aziontech/docs
	ref: dev
	path: '.'
	clean: true
	fetch-depth: '0'
	lfs: 'false'
	- name: SETTING Node and Github Packages
	uses: actions/setup-node@v4
	with:
	node-version: 20.13.1
	- name: INSTALLING DEPENDENCIES
	run: \|
	npm install
	- name: CHECKER MEMORY
	run: \|
	npm run check-memory
	- name: BUILD
	run: \|
	export NODE_OPTIONS="--max-old-space-size=8120"
	export PUBLIC_GITHUB_TOKEN="${{ secrets.GH_PACKAGES_SECRET }}"
	npm run build-development
	- name: 'AUTH Google Cloud'
	uses: 'google-github-actions/auth@v2'
	with:
	credentials_json: ${{ secrets.GCP_EXP_ENG }}
	- uses: 'google-github-actions/setup-gcloud@v2'
	with:
	version: '>= 363.0.0'
	- name: PUBLISHING
	run: \|
	gcloud storage rsync -r dist gs://azdocs-dev
	- name: UPDATING Google Cloud Storage files Cache-Control header
	run: \|
	gcloud storage objects update -r gs://azdocs-dev --cache-control="public, max-age=120"
	- name: PURGE Edge Cache
	run: \|
	npm install axios
	node ./cicd/cdn-purge.js basic_token=${{ secrets.MKT_BASE64_TOKEN }}

Provide feedback