Skip to content

Commit

Permalink
Merge pull request #140 from dadosjusbr/inconsistent-column
Browse files Browse the repository at this point in the history 
Inconsistent column
  • Loading branch information
@joellensilva
joellensilva authored Dec 20, 2024
2 parents d3b675a + f6caec4 commit 04adcca
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 34 deletions.
2 changes: 2 additions & 0 deletions models/monthlyInfo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type AgencyMonthlyInfo struct {
Score *Score `json:"score,omitempty"`
Duration float64 `json:"duration,omitempty"` // Crawling duration (seconds)
ManualCollection bool `json:"coleta_manual,omitempty"` // If the data was collected manually
Inconsistent bool `json:"inconsistent"` // If the data is inconsistent
}

type Meta struct {
Expand Down Expand Up @@ -94,6 +95,7 @@ type AnnualSummary struct {
NumMonthsWithData int `json:"months_with_data,omitempty"`
Package *Backup `json:"package,omitempty"`
ItemSummary ItemSummary `json:"item_summary,omitempty"`
Inconsistent bool `json:"inconsistent,omitempty"` // If the data is inconsistent
}

type RemmunerationSummary struct {
Expand Down
3 changes: 3 additions & 0 deletions repo/database/dto/annuaISummary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ type AnnualSummaryDTO struct {
RemunerationsPerCapita float64 `gorm:"column:remuneracoes_membro"`
NumMonthsWithData int `gorm:"column:meses_com_dados"`
ItemSummary ItemSummary `gorm:"embedded"`
Inconsistent bool `gorm:"column:inconsistente;<-:false"`
}

func NewAnnualSummaryDTO(ami models.AnnualSummary) *AnnualSummaryDTO {
Expand All @@ -44,6 +45,7 @@ func NewAnnualSummaryDTO(ami models.AnnualSummary) *AnnualSummaryDTO {
HealthAllowance: ami.ItemSummary.HealthAllowance,
Others: ami.ItemSummary.Others,
},
Inconsistent: ami.Inconsistent,
}
}

Expand Down Expand Up @@ -71,5 +73,6 @@ func (ami *AnnualSummaryDTO) ConvertToModel() *models.AnnualSummary {
HealthAllowance: ami.ItemSummary.HealthAllowance,
Others: ami.ItemSummary.Others,
},
Inconsistent: ami.Inconsistent,
}
}
5 changes: 4 additions & 1 deletion repo/database/dto/monthlyInfoDTO.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ type AgencyMonthlyInfoDTO struct {
Duration float64 `gorm:"column:duracao_segundos"` // Tempo de execução da coleta em segundos
Meta
Score
ManualCollection bool `gorm:"column:manual"` // Tempo de execução da coleta em segundos
ManualCollection bool `gorm:"column:manual"` // A coleta foi realizada manualmente?
Inconsistent bool `gorm:"column:inconsistente;<-:false"`
}

func (AgencyMonthlyInfoDTO) TableName() string {
Expand Down Expand Up @@ -133,6 +134,7 @@ func (a AgencyMonthlyInfoDTO) ConvertToModel() (*models.AgencyMonthlyInfo, error
Package: &pkg,
Duration: a.Duration,
ManualCollection: a.ManualCollection,
Inconsistent: a.Inconsistent,
}, nil
}

Expand Down Expand Up @@ -212,6 +214,7 @@ func NewAgencyMonthlyInfoDTO(agmi models.AgencyMonthlyInfo) (*AgencyMonthlyInfoD
Package: pkg,
Duration: agmi.Duration,
ManualCollection: agmi.ManualCollection,
Inconsistent: agmi.Inconsistent,
}, nil
}

Expand Down
156 changes: 123 additions & 33 deletions repo/database/postgres.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,11 +283,34 @@ func (p *PostgresDB) GetMonthlyInfo(agencies []models.Agency, year int) (map[str
//Mapeando os órgãos
for _, agency := range agencies {
var dtoAgmis []dto.AgencyMonthlyInfoDTO
//Pegando as coletas do postgres, filtrando por órgão, ano e a coleta atual.
m := p.db.Model(&dto.AgencyMonthlyInfoDTO{})
m = m.Where("id_orgao = ? AND ano = ? AND atual = TRUE AND (procinfo::text = 'null' OR procinfo IS NULL) ", agency.ID, year)
m = m.Order("mes ASC")
if err := m.Find(&dtoAgmis).Error; err != nil {
// Pegando as coletas do postgres, filtrando por órgão, ano e a coleta atual.
// GORM não suporta diretamente expressões como CTEs (WITH), mas é possível integrar a query bruta.
// Estamos utilizando essa estratégia a fim de não comprometer o desempenho da query
// devido a junção com a tabela de remuneracoes
query := `
WITH remuneracoes_inconsistentes AS (
SELECT DISTINCT orgao, ano, mes
FROM remuneracoes
WHERE inconsistente = TRUE
)
SELECT c.*,
CASE
WHEN ri.orgao IS NOT NULL THEN TRUE
ELSE FALSE
END AS inconsistente
FROM coletas c
LEFT JOIN remuneracoes_inconsistentes ri
ON ri.orgao = c.id_orgao
AND ri.ano = c.ano
AND ri.mes = c.mes
WHERE c.id_orgao = ?
AND c.ano = ?
AND c.atual = TRUE
AND (c.procinfo::text = 'null' OR c.procinfo IS NULL)
ORDER BY c.mes ASC;
`

if err := p.db.Raw(query, agency.ID, year).Scan(&dtoAgmis).Error; err != nil {
return nil, fmt.Errorf("error getting monthly info: %q", err)
}
//Convertendo os DTO's para modelos
Expand All @@ -303,36 +326,95 @@ func (p *PostgresDB) GetMonthlyInfo(agencies []models.Agency, year int) (map[str
}

func (p *PostgresDB) GetAnnualSummary(agency string) ([]models.AnnualSummary, error) {
var dtoAgmi dto.AgencyMonthlyInfoDTO
var dtoAmis []dto.AnnualSummaryDTO
agency = strings.ToLower(agency)

// GORM não suporta diretamente expressões como CTEs (WITH), mas é possível integrar a query bruta.
// Estamos utilizando essa estratégia a fim de não comprometer o desempenho da query
// devido a junção com a tabela de remuneracoes
query := `
coletas.ano,
id_orgao,
TRUNC(AVG((sumario -> 'membros')::text::int)) AS media_num_membros,
SUM((sumario -> 'membros')::text::int) AS total_num_membros,
SUM(CAST(sumario -> 'remuneracao_base' ->> 'total' AS DECIMAL)) AS remuneracao_base,
SUM(CAST(sumario -> 'outras_remuneracoes' ->> 'total' AS DECIMAL)) AS outras_remuneracoes,
SUM(CAST(sumario -> 'descontos' ->> 'total' AS DECIMAL)) AS descontos,
SUM(CAST(sumario -> 'remuneracoes' ->> 'total' AS DECIMAL)) AS remuneracoes,
SUM(CAST(sumario -> 'resumo_rubricas' ->> 'auxilio_alimentacao' AS DECIMAL)) AS auxilio_alimentacao,
SUM(CAST(sumario -> 'resumo_rubricas' ->> 'licenca_premio' AS DECIMAL)) AS licenca_premio,
SUM(CAST(sumario -> 'resumo_rubricas' ->> 'indenizacao_de_ferias' AS DECIMAL)) AS indenizacao_de_ferias,
SUM(CAST(sumario -> 'resumo_rubricas' ->> 'gratificacao_natalina' AS DECIMAL)) AS gratificacao_natalina,
SUM(CAST(sumario -> 'resumo_rubricas' ->> 'licenca_compensatoria' AS DECIMAL)) AS licenca_compensatoria,
SUM(CAST(sumario -> 'resumo_rubricas' ->> 'auxilio_saude' AS DECIMAL)) AS auxilio_saude,
SUM(CAST(sumario -> 'resumo_rubricas' ->> 'outras' AS DECIMAL)) AS outras,
SUM(CAST(sumario -> 'resumo_rubricas' ->> 'ferias' AS DECIMAL)) AS ferias,
COUNT(*) AS meses_com_dados,
MAX(media_por_membro.salario) AS remuneracao_base_membro,
MAX(media_por_membro.beneficios) AS outras_remuneracoes_membro,
MAX(media_por_membro.descontos) AS descontos_membro,
MAX(media_por_membro.remuneracao) AS remuneracoes_membro`

join := `LEFT JOIN media_por_membro on coletas.ano = media_por_membro.ano and coletas.id_orgao = media_por_membro.orgao`
m := p.db.Model(&dtoAgmi).Select(query).Joins(join)
m = m.Where("id_orgao = ? AND atual = TRUE AND (procinfo::text = 'null' OR procinfo IS NULL) ", agency)
m = m.Group("coletas.ano, id_orgao").Order("coletas.ano ASC")
WITH sumario_processado AS (
SELECT
ano,
id_orgao,
(sumario ->> 'membros')::INT AS num_membros,
(sumario -> 'remuneracao_base' ->> 'total')::DECIMAL AS remuneracao_base_total,
(sumario -> 'outras_remuneracoes' ->> 'total')::DECIMAL AS outras_remuneracoes_total,
(sumario -> 'descontos' ->> 'total')::DECIMAL AS descontos_total,
(sumario -> 'remuneracoes' ->> 'total')::DECIMAL AS remuneracoes_total,
(sumario -> 'resumo_rubricas' ->> 'auxilio_alimentacao')::DECIMAL AS auxilio_alimentacao,
(sumario -> 'resumo_rubricas' ->> 'licenca_premio')::DECIMAL AS licenca_premio,
(sumario -> 'resumo_rubricas' ->> 'indenizacao_de_ferias')::DECIMAL AS indenizacao_de_ferias,
(sumario -> 'resumo_rubricas' ->> 'gratificacao_natalina')::DECIMAL AS gratificacao_natalina,
(sumario -> 'resumo_rubricas' ->> 'licenca_compensatoria')::DECIMAL AS licenca_compensatoria,
(sumario -> 'resumo_rubricas' ->> 'auxilio_saude')::DECIMAL AS auxilio_saude,
(sumario -> 'resumo_rubricas' ->> 'outras')::DECIMAL AS outras,
(sumario -> 'resumo_rubricas' ->> 'ferias')::DECIMAL AS ferias
FROM coletas
WHERE id_orgao = ? AND atual = TRUE AND (procinfo::TEXT = 'null' OR procinfo IS NULL)
),
media_membro AS (
SELECT
ano,
orgao,
MAX(salario) AS remuneracao_base_membro,
MAX(beneficios) AS outras_remuneracoes_membro,
MAX(descontos) AS descontos_membro,
MAX(remuneracao) AS remuneracoes_membro
FROM media_por_membro
WHERE orgao = ?
GROUP BY ano, orgao
),
remuneracoes_inconsistentes AS (
SELECT DISTINCT orgao, ano
FROM remuneracoes
WHERE inconsistente = TRUE AND orgao = ?
)
SELECT
sp.ano,
sp.id_orgao,
TRUNC(AVG(sp.num_membros)) AS media_num_membros,
SUM(sp.num_membros) AS total_num_membros,
SUM(sp.remuneracao_base_total) AS remuneracao_base,
SUM(sp.outras_remuneracoes_total) AS outras_remuneracoes,
SUM(sp.descontos_total) AS descontos,
SUM(sp.remuneracoes_total) AS remuneracoes,
SUM(sp.auxilio_alimentacao) AS auxilio_alimentacao,
SUM(sp.licenca_premio) AS licenca_premio,
SUM(sp.indenizacao_de_ferias) AS indenizacao_de_ferias,
SUM(sp.gratificacao_natalina) AS gratificacao_natalina,
SUM(sp.licenca_compensatoria) AS licenca_compensatoria,
SUM(sp.auxilio_saude) AS auxilio_saude,
SUM(sp.outras) AS outras,
SUM(sp.ferias) AS ferias,
COUNT(*) AS meses_com_dados,
mm.remuneracao_base_membro,
mm.outras_remuneracoes_membro,
mm.descontos_membro,
mm.remuneracoes_membro,
CASE
WHEN ri.orgao IS NOT NULL THEN TRUE
ELSE FALSE
END AS inconsistente
FROM sumario_processado sp
LEFT JOIN media_membro mm
ON sp.ano = mm.ano
AND sp.id_orgao = mm.orgao
LEFT JOIN remuneracoes_inconsistentes ri
ON sp.ano = ri.ano
AND sp.id_orgao = ri.orgao
GROUP BY
sp.ano,
sp.id_orgao,
mm.remuneracao_base_membro,
mm.outras_remuneracoes_membro,
mm.descontos_membro,
mm.remuneracoes_membro,
ri.orgao
ORDER BY sp.ano ASC;
`

m := p.db.Raw(query, agency, agency, agency)
if err := m.Scan(&dtoAmis).Error; err != nil {
return nil, fmt.Errorf("error getting annual monthly info: %q", err)
}
Expand All @@ -346,7 +428,15 @@ func (p *PostgresDB) GetAnnualSummary(agency string) ([]models.AnnualSummary, er
func (p *PostgresDB) GetOMA(month int, year int, agency string) (*models.AgencyMonthlyInfo, *models.Agency, error) {
var dtoAgmi dto.AgencyMonthlyInfoDTO
id := fmt.Sprintf("%s/%s/%d", strings.ToLower(agency), dto.AddZeroes(month), year)
m := p.db.Model(dto.AgencyMonthlyInfoDTO{}).Where("id = ? AND atual = true", id).First(&dtoAgmi)
m := p.db.Model(dto.AgencyMonthlyInfoDTO{}).Select(`coletas.*, EXISTS (
SELECT 1
FROM remuneracoes r
WHERE inconsistente = TRUE
AND r.orgao = coletas.id_orgao
AND r.ano = coletas.ano
and r.mes = coletas.mes
) AS inconsistente`)
m = m.Where("id = ? AND atual = true", id).First(&dtoAgmi)
if err := m.Error; err != nil {
if err == gorm.ErrRecordNotFound {
return nil, nil, fmt.Errorf("there is no data with this parameters")
Expand Down
5 changes: 5 additions & 0 deletions repo/database/postgres_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -788,13 +788,15 @@ func (g getMonthlyInfo) testWhenMonthlyInfoExists(t *testing.T) {
Month: 1,
CrawlingTimestamp: timestamppb.Now(),
ManualCollection: false,
Inconsistent: false,
},
{
AgencyID: "tjsp",
Year: 2020,
Month: 1,
CrawlingTimestamp: timestamppb.Now(),
ManualCollection: true,
Inconsistent: false,
},
}
if err := insertMonthlyInfos(agmis); err != nil {
Expand Down Expand Up @@ -1142,6 +1144,8 @@ func (g getAnnualSummary) testWhenMonthlyInfoExists(t *testing.T) {
assert.Equal(t, amis[1].ItemSummary.Vacation, returnedAmis[1].ItemSummary.Vacation)
assert.Equal(t, 1000.0, returnedAmis[2].BaseRemunerationPerCapita)
assert.Equal(t, 1200.0, returnedAmis[2].OtherRemunerationsPerCapita)
assert.Equal(t, amis[0].Inconsistent, returnedAmis[0].Inconsistent)
assert.Equal(t, false, returnedAmis[0].Inconsistent)
truncateTables()
}

Expand Down Expand Up @@ -1189,6 +1193,7 @@ func (g getOMA) testWhenDataExists(t *testing.T) {
assert.Equal(t, agmi.Month, returnedAgmi.Month)
assert.Equal(t, agmi.Year, returnedAgmi.Year)
assert.Equal(t, agencies[0], *agency)
assert.Equal(t, returnedAgmi.Inconsistent, false)
truncateTables()
}

Expand Down

0 comments on commit 04adcca

Please sign in to comment.