From e8a2088c282230c4caabaf038048235d806ea9b1 Mon Sep 17 00:00:00 2001 From: Joellensilva Date: Tue, 16 Apr 2024 16:02:49 -0300 Subject: [PATCH] modificando o separador de colunas e decimal --- .github/workflows/dump.yaml | 8 ++++---- dump/dump.go | 7 +++++++ go.mod | 2 +- go.sum | 4 ++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dump.yaml b/.github/workflows/dump.yaml index 5912523..053bd98 100644 --- a/.github/workflows/dump.yaml +++ b/.github/workflows/dump.yaml @@ -18,16 +18,16 @@ jobs: - uses: actions/checkout@v3 - name: Criando coleta.csv... - run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id as chave_coleta, id_orgao as orgao, mes, ano, timestamp as timestampb_coleta, repositorio_coletor, versao_coletor, repositorio_parser, versao_parser FROM coletas WHERE (procinfo is null or procinfo::text = 'null') and atual=true) TO 'dump/coleta.csv' csv header;" + run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id as chave_coleta, id_orgao as orgao, mes, ano, timestamp as timestampb_coleta, repositorio_coletor, versao_coletor, repositorio_parser, versao_parser FROM coletas WHERE (procinfo is null or procinfo::text = 'null') and atual=true) TO 'dump/coleta.csv' csv header delimiter ';';" - name: Criando metadados.csv... - run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id_orgao as orgao, mes, ano, formato_aberto, acesso, extensao, estritamente_tabular, formato_consistente, tem_matricula, tem_lotacao, tem_cargo, detalhamento_receita_base, detalhamento_outras_receitas, detalhamento_descontos, indice_completude, indice_facilidade, indice_transparencia FROM coletas WHERE (procinfo is null or procinfo::text = 'null') and atual=true) TO 'dump/metadados.csv' csv header;" + run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id_orgao as orgao, mes, ano, formato_aberto, acesso, extensao, estritamente_tabular, formato_consistente, tem_matricula, tem_lotacao, tem_cargo, detalhamento_receita_base, detalhamento_outras_receitas, detalhamento_descontos, replace(indice_completude::text, '.', ',') as indice_completude, replace(indice_facilidade::text, '.', ',') as indice_facilidade, replace(indice_transparencia::text, '.', ',') as indice_transparencia FROM coletas WHERE (procinfo is null or procinfo::text = 'null') and atual=true) TO 'dump/metadados.csv' csv header delimiter ';';" - name: Criando contracheque.csv... - run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id as id_contracheque, orgao, mes, ano, nome, matricula, funcao, local_trabalho, salario, beneficios, descontos, remuneracao FROM contracheques) TO 'dump/contracheque.csv' csv header;" + run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id as id_contracheque, orgao, mes, ano, nome, matricula, funcao, local_trabalho, replace(salario::text, '.', ',') as salario, replace(beneficios::text, '.', ',') as beneficios, replace(descontos::text, '.', ',') as descontos, replace(remuneracao::text, '.', ',') as remuneracao, situacao FROM contracheques) TO 'dump/contracheque.csv' csv header delimiter ';';" - name: Criando remuneracao.csv... - run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id_contracheque, orgao, mes, ano, tipo, categoria, item, valor FROM remuneracoes) TO 'dump/remuneracao.csv' csv header;" + run: PGPASSWORD=${{ secrets.POSTGRES_PASSWORD }} psql -h "${{ secrets.POSTGRES_HOST }}" -U ${{ secrets.POSTGRES_USER }} -d ${{ secrets.POSTGRES_DB }} -c "\copy (SELECT id_contracheque, orgao, mes, ano, tipo, categoria, item, replace(valor::text, '.', ',') as valor FROM remuneracoes) TO 'dump/remuneracao.csv' csv header delimiter ';';" - name: Realizando dump... run: | diff --git a/dump/dump.go b/dump/dump.go index a795521..3b715d3 100644 --- a/dump/dump.go +++ b/dump/dump.go @@ -50,6 +50,7 @@ func main() { year, month, _ := time.Now().Date() pkgName := fmt.Sprintf("dadosjusbr-%d-%d.zip", year, month) desc, err := datapackage.DescriptorMapV2() + if err != nil { log.Fatalf("error DescriptorMapV2(): %v", err) } @@ -57,10 +58,16 @@ func main() { if err != nil { log.Fatalf("error create datapackage: %v", err) } + if err := pkg.Zip(pkgName); err != nil { log.Fatalf("error zipping datapackage: %v", err) } + // Testando o pacote + if _, err = datapackage.LoadV2(pkgName); err != nil { + log.Fatalf("error loading datapackage: %v", err) + } + // Armazenando no S3 _, err = pgS3Client.Cloud.UploadFile(pkgName, "dumps/"+pkgName) if err != nil { diff --git a/go.mod b/go.mod index 567ce13..6ec7c88 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.14 require ( github.com/aws/aws-sdk-go v1.44.127 - github.com/dadosjusbr/datapackage v0.0.0-20230904162108-6e2264aafb68 + github.com/dadosjusbr/datapackage v0.0.0-20240320203926-5f369dadd3a5 github.com/dadosjusbr/proto v0.0.0-20221212025627-91c60aa3cd12 github.com/frictionlessdata/datapackage-go v1.0.4 github.com/golang/mock v1.6.0 diff --git a/go.sum b/go.sum index 36c95d0..e72b874 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dadosjusbr/datapackage v0.0.0-20230904162108-6e2264aafb68 h1:DLea5zdEkGQkhwALQl2b+cpld7rVHZgeXKv+SXQ1/6o= github.com/dadosjusbr/datapackage v0.0.0-20230904162108-6e2264aafb68/go.mod h1:o1FghJC46mR+lgHcQWVve/q84/SAd0WVdYMcgq88BBU= +github.com/dadosjusbr/datapackage v0.0.0-20240320203926-5f369dadd3a5 h1:Oh8goNtBxzPls8mvFt1GZ0ISSOcpINyip3eRknSMkK0= +github.com/dadosjusbr/datapackage v0.0.0-20240320203926-5f369dadd3a5/go.mod h1:xDZc4jX8VDOP7SFo793x0Q8gjUjuMSOmNEUG/wqLKAw= github.com/dadosjusbr/proto v0.0.0-20221212025627-91c60aa3cd12 h1:ufl8nbCEo6g2VHUbedGy0gYk9Sgrynf9rcnzuSw4TEg= github.com/dadosjusbr/proto v0.0.0-20221212025627-91c60aa3cd12/go.mod h1:gPA7VxjEmyez/xtln4qBj+tM1NO0/zcw3ryjxTRNSco= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -44,6 +46,8 @@ github.com/frictionlessdata/datapackage-go v1.0.4 h1:B2td8AuRBezuxCtbHStFL9+Qoud github.com/frictionlessdata/datapackage-go v1.0.4/go.mod h1:bqt5SzFjnpsZNw4kXO4CFIwH9vV5LhEdQikftJ0uJ5M= github.com/frictionlessdata/tableschema-go v1.1.4-0.20220401172006-6cc5f3b2411c h1:7S5F4VDf8vkLL3egYLWobmq1FbZb7ig33IbliL7Tr/M= github.com/frictionlessdata/tableschema-go v1.1.4-0.20220401172006-6cc5f3b2411c/go.mod h1:B+DhLlwjCf6p6FqVkqpdYyAIy7L8jHCaxa2wFaqpYdc= +github.com/frictionlessdata/tableschema-go v1.5.2 h1:zELnABwto2Q9/nB7EhEf8rDR/ju+CsDI+BhNg6Z0jU8= +github.com/frictionlessdata/tableschema-go v1.5.2/go.mod h1:B+DhLlwjCf6p6FqVkqpdYyAIy7L8jHCaxa2wFaqpYdc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=