Merge pull request #133 from dadosjusbr/modificando-separador-de-colunas

modificando o separador de colunas e decimal do dump
dadosjusbr · Apr 16, 2024 · c3cdbf1 · c3cdbf1
2 parents 5da10c8 + 107eef6
commit c3cdbf1
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 5 deletions.
diff --git a/.github/workflows/dump.yaml b/.github/workflows/dump.yaml
@@ -18,16 +18,16 @@ jobs:
             - uses: actions/checkout@v3
 
             - name: Criando coleta.csv...
-              run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id as chave_coleta, id_orgao as orgao, mes, ano, timestamp as timestampb_coleta, repositorio_coletor, versao_coletor, repositorio_parser, versao_parser FROM coletas WHERE (procinfo is null or procinfo::text = 'null') and atual=true) TO 'dump/coleta.csv' csv header;"
+              run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id as chave_coleta, id_orgao as orgao, mes, ano, TO_CHAR(timestamp, 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') as timestampb_coleta, repositorio_coletor, versao_coletor, repositorio_parser, versao_parser FROM coletas WHERE (procinfo is null or procinfo::text = 'null') and atual=true) TO 'dump/coleta.csv' csv header delimiter ';';"
 
             - name: Criando metadados.csv...
-              run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id_orgao as orgao, mes, ano, formato_aberto, acesso, extensao, estritamente_tabular, formato_consistente, tem_matricula, tem_lotacao, tem_cargo, detalhamento_receita_base, detalhamento_outras_receitas, detalhamento_descontos, indice_completude, indice_facilidade, indice_transparencia FROM coletas WHERE (procinfo is null or procinfo::text = 'null') and atual=true) TO 'dump/metadados.csv' csv header;"
+              run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id_orgao as orgao, mes, ano, formato_aberto, acesso, extensao, estritamente_tabular, formato_consistente, tem_matricula, tem_lotacao, tem_cargo, detalhamento_receita_base, detalhamento_outras_receitas, detalhamento_descontos, replace(indice_completude::text, '.', ',') as indice_completude, replace(indice_facilidade::text, '.', ',') as indice_facilidade, replace(indice_transparencia::text, '.', ',') as indice_transparencia FROM coletas WHERE (procinfo is null or procinfo::text = 'null') and atual=true) TO 'dump/metadados.csv' csv header delimiter ';';"
 
             - name: Criando contracheque.csv...
-              run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT  id as id_contracheque, orgao, mes, ano, nome, matricula, funcao, local_trabalho, salario, beneficios, descontos, remuneracao FROM contracheques) TO 'dump/contracheque.csv' csv header;"
+              run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT  id as id_contracheque, orgao, mes, ano, nome, matricula, funcao, local_trabalho, replace(salario::text, '.', ',') as salario, replace(beneficios::text, '.', ',') as beneficios, replace(descontos::text, '.', ',') as descontos, replace(remuneracao::text, '.', ',') as remuneracao, situacao FROM contracheques) TO 'dump/contracheque.csv' csv header delimiter ';';"
 
             - name: Criando remuneracao.csv...
-              run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id_contracheque, orgao, mes, ano, tipo, categoria, item, valor FROM remuneracoes) TO 'dump/remuneracao.csv' csv header;"
+              run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id_contracheque, orgao, mes, ano, tipo, categoria, item, replace(valor::text, '.', ',') as valor FROM remuneracoes) TO 'dump/remuneracao.csv' csv header delimiter ';';"
 
             - name: Realizando dump...
               run: |

diff --git a/dump/dump.go b/dump/dump.go
@@ -50,17 +50,24 @@ func main() {
 	year, month, _ := time.Now().Date()
 	pkgName := fmt.Sprintf("dadosjusbr-%d-%d.zip", year, month)
 	desc, err := datapackage.DescriptorMapV2()
+
 	if err != nil {
 		log.Fatalf("error DescriptorMapV2(): %v", err)
 	}
 	pkg, err := dpkg.New(desc, ".")
 	if err != nil {
 		log.Fatalf("error create datapackage: %v", err)
 	}
+
 	if err := pkg.Zip(pkgName); err != nil {
 		log.Fatalf("error zipping datapackage: %v", err)
 	}
 
+	// Testando o pacote
+	if _, err = datapackage.LoadV2(pkgName); err != nil {
+		log.Fatalf("error loading datapackage: %v", err)
+	}
+
 	// Armazenando no S3
 	_, err = pgS3Client.Cloud.UploadFile(pkgName, "dumps/"+pkgName)
 	if err != nil {

diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.14
 
 require (
 	github.com/aws/aws-sdk-go v1.44.127
-	github.com/dadosjusbr/datapackage v0.0.0-20230904162108-6e2264aafb68
+	github.com/dadosjusbr/datapackage v0.0.0-20240320203926-5f369dadd3a5
 	github.com/dadosjusbr/proto v0.0.0-20221212025627-91c60aa3cd12
 	github.com/frictionlessdata/datapackage-go v1.0.4
 	github.com/golang/mock v1.6.0

diff --git a/go.sum b/go.sum
@@ -26,6 +26,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/dadosjusbr/datapackage v0.0.0-20230904162108-6e2264aafb68 h1:DLea5zdEkGQkhwALQl2b+cpld7rVHZgeXKv+SXQ1/6o=
 github.com/dadosjusbr/datapackage v0.0.0-20230904162108-6e2264aafb68/go.mod h1:o1FghJC46mR+lgHcQWVve/q84/SAd0WVdYMcgq88BBU=
+github.com/dadosjusbr/datapackage v0.0.0-20240320203926-5f369dadd3a5 h1:Oh8goNtBxzPls8mvFt1GZ0ISSOcpINyip3eRknSMkK0=
+github.com/dadosjusbr/datapackage v0.0.0-20240320203926-5f369dadd3a5/go.mod h1:xDZc4jX8VDOP7SFo793x0Q8gjUjuMSOmNEUG/wqLKAw=
 github.com/dadosjusbr/proto v0.0.0-20221212025627-91c60aa3cd12 h1:ufl8nbCEo6g2VHUbedGy0gYk9Sgrynf9rcnzuSw4TEg=
 github.com/dadosjusbr/proto v0.0.0-20221212025627-91c60aa3cd12/go.mod h1:gPA7VxjEmyez/xtln4qBj+tM1NO0/zcw3ryjxTRNSco=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -44,6 +46,8 @@ github.com/frictionlessdata/datapackage-go v1.0.4 h1:B2td8AuRBezuxCtbHStFL9+Qoud
 github.com/frictionlessdata/datapackage-go v1.0.4/go.mod h1:bqt5SzFjnpsZNw4kXO4CFIwH9vV5LhEdQikftJ0uJ5M=
 github.com/frictionlessdata/tableschema-go v1.1.4-0.20220401172006-6cc5f3b2411c h1:7S5F4VDf8vkLL3egYLWobmq1FbZb7ig33IbliL7Tr/M=
 github.com/frictionlessdata/tableschema-go v1.1.4-0.20220401172006-6cc5f3b2411c/go.mod h1:B+DhLlwjCf6p6FqVkqpdYyAIy7L8jHCaxa2wFaqpYdc=
+github.com/frictionlessdata/tableschema-go v1.5.2 h1:zELnABwto2Q9/nB7EhEf8rDR/ju+CsDI+BhNg6Z0jU8=
+github.com/frictionlessdata/tableschema-go v1.5.2/go.mod h1:B+DhLlwjCf6p6FqVkqpdYyAIy7L8jHCaxa2wFaqpYdc=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=