Skip to content

Commit

Permalink
Coleta cota parlamentar do Maranhão;
Browse files Browse the repository at this point in the history
Melhoria nos logs com inclusão do Serilog/Seq e uso de Scope;
Processamento paralelo para todos os estados para reduzir o tempo de processamento total;
  • Loading branch information
VanderleiDenir committed Aug 15, 2024
1 parent a902604 commit 7985d22
Show file tree
Hide file tree
Showing 51 changed files with 1,586 additions and 1,140 deletions.
6 changes: 3 additions & 3 deletions OPS.API/OPS.API.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@
</ItemGroup>

<ItemGroup>
<PackageReference Include="Marvin.Cache.Headers" Version="7.1.0" />
<PackageReference Include="Marvin.Cache.Headers" Version="7.1.0" />
<PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.20.1" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
<PackageReference Include="Microsoft.Win32.Primitives" Version="4.3.0" />
<PackageReference Include="MySqlConnector.Logging.Serilog" Version="2.0.0" />
<PackageReference Include="System.Net.NameResolution" Version="4.3.0" />
<PackageReference Include="System.Text.Json" Version="8.0.3" />
<PackageReference Include="System.Text.Json" Version="8.0.4" />
</ItemGroup>

<ItemGroup>
Expand Down
4 changes: 2 additions & 2 deletions OPS.Core.Test/OPS.Core.Test.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
<PackageReference Include="xunit" Version="2.8.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.1">
<PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
Expand Down
114 changes: 114 additions & 0 deletions OPS.Core/Entity/DeputadoFederalDespesaTemp.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
using System;
using System.Text.Json.Serialization;
using Dapper;

namespace OPS.Core.Entity
{
[Table("cf_despesa_temp", Schema = "ops_tmp")]
public class DeputadoFederalDespesaTemp
{
[JsonIgnore]
[Column("id")]
public int Id { get; set; }

[Column("idDeputado")]
public long? IdDeputado { get; set; }

[Column("nomeParlamentar")]
public string Nome { get; set; }

[Column("numeroCarteiraParlamentar")]
public int? CarteiraParlamentar { get; set; }

[Column("legislatura")]
public int? Legislatura { get; set; }

[Column("siglaUF")]
public string SiglaUF { get; set; }

[Column("siglaPartido")]
public string SiglaPartido { get; set; }

[Column("codigoLegislatura")]
public int? CodigoLegislatura { get; set; }

[Column("numeroSubCota")]
public int? NumeroSubCota { get; set; }

[Column("descricao")]
public string Descricao { get; set; }

[Column("numeroEspecificacaoSubCota")]
public int? NumeroEspecificacaoSubCota { get; set; }

[Column("descricaoEspecificacao")]
public string DescricaoEspecificacao { get; set; }

[Column("fornecedor")]
public string Fornecedor { get; set; }

[Column("cnpjCPF")]
public string CnpjCpf { get; set; }

[Column("numero")]
public string Numero { get; set; }

[Column("tipoDocumento")]
public int? TipoDocumento { get; set; }

[Column("dataEmissao")]
public DateOnly? DataEmissao { get; set; }

[Column("valorDocumento")]
public decimal ValorDocumento { get; set; }

[Column("valorGlosa")]
public decimal ValorGlosa { get; set; }

[Column("valorLiquido")]
public decimal ValorLiquido { get; set; }

[Column("mes")]
public short Mes { get; set; }

[Column("ano")]
public short Ano { get; set; }

[Column("parcela")]
public int? Parcela { get; set; }

[Column("passageiro")]
public string Passageiro { get; set; }

[Column("trecho")]
public string Trecho { get; set; }

[Column("lote")]
public int? Lote { get; set; }

[Column("ressarcimento")]
public int? NumeroRessarcimento { get; set; }

[Column("idDocumento")]
public string IdDocumento { get; set; }

[Column("restituicao")]
public decimal? ValorRestituicao { get; set; }

[Column("datPagamentoRestituicao")]
public DateOnly? DataPagamentoRestituicao { get; set; }

[Column("numeroDeputadoID")]
public int? NumeroDeputadoID { get; set; }

[Column("cpf")]
public string Cpf { get; set; }

[Column("urlDocumento")]
public string UrlDocumento { get; set; }

[JsonIgnore]
[Column("hash")]
public byte[] Hash { get; set; }
}
}
4 changes: 2 additions & 2 deletions OPS.Core/OPS.Core.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@
<PackageReference Include="Dapper" Version="2.1.35" />
<PackageReference Include="MySqlConnector" Version="2.3.7" />
<PackageReference Include="MySqlConnector.Logging.Serilog" Version="2.0.0" />
<PackageReference Include="RestSharp" Version="111.2.0" />
<PackageReference Include="RestSharp" Version="111.4.1" />
<PackageReference Include="System.Runtime.Caching" Version="8.0.0" />
<PackageReference Include="System.Text.Json" Version="8.0.3" />
<PackageReference Include="System.Text.Json" Version="8.0.4" />
</ItemGroup>

</Project>
58 changes: 30 additions & 28 deletions OPS.Core/Utilities/RestClientExtension.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,40 +9,42 @@ public static class RestClientExtension
{
public static RestResponse GetWithAutoRetry(this RestClient client, RestRequest request, int totalRetries = 5)
{
var watch = System.Diagnostics.Stopwatch.StartNew();
Log.Verbose("Chamando URL: {Url}", request.Resource);
// Realizando retry pelo polly.
return client.Get(request);
//var watch = System.Diagnostics.Stopwatch.StartNew();
//Log.Verbose("Chamando URL: {Url}", request.Resource);

try
{
var response = client.Get(request);
if (!string.IsNullOrEmpty(response.Content))
return response;
//try
//{
// var response = client.Get(request);
// if (!string.IsNullOrEmpty(response.Content))
// return response;

if (totalRetries > 0)
return ReTry(client, request, totalRetries);
// if (totalRetries > 0)
// return ReTry(client, request, totalRetries);

return response;
}
catch (Exception)
{
if (totalRetries > 0)
return ReTry(client, request, totalRetries);
// return response;
//}
//catch (Exception)
//{
// if (totalRetries > 0)
// return ReTry(client, request, totalRetries);

throw;
}
finally
{
watch.Stop();
Log.Verbose("Requisição processada em {TimeElapsed:c}", watch.Elapsed);
}
// throw;
//}
//finally
//{
// watch.Stop();
// Log.Verbose("Requisição processada em {TimeElapsed:c}", watch.Elapsed);
//}
}

private static RestResponse ReTry(RestClient client, RestRequest request, int totalRetries)
{
Log.Verbose("Tentativa {Tentativa} para a url {Url}", totalRetries, request.Resource);
//private static RestResponse ReTry(RestClient client, RestRequest request, int totalRetries)
//{
// Log.Verbose("Tentativa {Tentativa} para a url {Url}", totalRetries, request.Resource);

Thread.Sleep(TimeSpan.FromMinutes(1));
return GetWithAutoRetry(client, request, totalRetries - 1);
}
// Thread.Sleep(TimeSpan.FromMinutes(1));
// return GetWithAutoRetry(client, request, totalRetries - 1);
//}
}
}
12 changes: 7 additions & 5 deletions OPS.Core/Utilities/Utils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -311,11 +311,6 @@ public static String ReduceWhitespace(this string text)
return Regex.Replace(text, @"\s+", " ");
}

public static string RemoveSpaces(this string value)
{
return Regex.Replace(value, @"\s+", " ");
}

public static string RemoveAccents(this string text)
{
if (string.IsNullOrEmpty(text)) return text;
Expand All @@ -330,6 +325,13 @@ public static string RemoveAccents(this string text)
return sbReturn.ToString();
}

public static string ForceWindows1252ToUtf8Encoding(this string text)
{
const int WindowsCodepage1252 = 1252;
byte[] bytes = Encoding.GetEncoding(WindowsCodepage1252).GetBytes(text);
return Encoding.Latin1.GetString(bytes);
}

//public static string GetIPAddress()
//{
// System.Web.HttpContext context = System.Web.HttpContext.Current;
Expand Down
8 changes: 4 additions & 4 deletions OPS.Importador/ALE/Acre.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using OPS.Core.Enum;
using OPS.Importador.ALE.Despesa;
using OPS.Importador.ALE.Parlamentar;
using OPS.Importador.Utilities;

namespace OPS.Importador.ALE;

Expand Down Expand Up @@ -64,7 +65,6 @@ public ImportadorParlamentarAcre(IServiceProvider serviceProvider) : base(servic

public override Task Importar()
{
logger.LogWarning("Parlamentares do(a) {idEstado}:{CasaLegislativa}", config.Estado.GetHashCode(), config.Estado.ToString());
ArgumentNullException.ThrowIfNull(config, nameof(config));

var legislatura = 17; // 2023-2027
Expand All @@ -86,7 +86,7 @@ public override Task Importar()
InsertOrUpdate(deputado);
}

logger.LogWarning("Parlamentares Inseridos: {Inseridos}; Atualizados {Atualizados};", base.registrosInseridos, base.registrosAtualizados);
logger.LogInformation("Parlamentares Inseridos: {Inseridos}; Atualizados {Atualizados};", base.registrosInseridos, base.registrosAtualizados);
return Task.CompletedTask;
}

Expand All @@ -95,7 +95,7 @@ private async Task ObterDetalhesDoPerfil(DeputadoEstadual deputado)
var angleSharpConfig = Configuration.Default.WithDefaultLoader();
var context = BrowsingContext.New(angleSharpConfig);

var document = await context.OpenAsync(deputado.UrlPerfil);
var document = await context.OpenAsyncAutoRetry(deputado.UrlPerfil);
if (document.StatusCode != HttpStatusCode.OK)
{
Console.WriteLine($"{config.BaseAddress} {document.StatusCode}");
Expand All @@ -113,7 +113,7 @@ private async Task ObterDetalhesDoPerfil(DeputadoEstadual deputado)
}
else
{
logger.LogWarning($"Verificar possivel mudança no perfil do deputado: {deputado.UrlPerfil}");
logger.LogWarning("Verificar possivel mudança no perfil do parlamentar: {UrlPerfil}", deputado.UrlPerfil);
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions OPS.Importador/ALE/Amapa.cs
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public override void ImportarDespesas(IBrowsingContext context, int ano, int mes
var deputado = deputados.Find(x => gabinete.Value.Contains(x.Gabinete.ToString()));
if (deputado == null)
{
logger.LogError($"Deputado {gabinete.Value}: {gabinete.Text} não existe ou não possui gabinete relacionado!");
logger.LogError("Parlamentar {Gabinete}: {Parlamentar} não existe ou não possui gabinete relacionado!", gabinete.Value, gabinete.Text);
}

address = $"{config.BaseAddress}transparencia/pagina.php?pg=ceap&acao=buscar&ano_verbaB={ano}&mes_verbaB={mes:00}&idgabineteB={gabinete.Value}";
Expand Down Expand Up @@ -129,7 +129,7 @@ public ImportadorParlamentarAmapa(IServiceProvider serviceProvider) : base(servi
// var angleSharpConfig = Configuration.Default.WithDefaultLoader();
// var context = BrowsingContext.New(angleSharpConfig);

// var document = await context.OpenAsync(config.BaseAddress);
// var document = await context.OpenAsyncAutoRetry(config.BaseAddress);
// if (document.StatusCode != HttpStatusCode.OK)
// {
// Console.WriteLine($"{config.BaseAddress} {document.StatusCode}");
Expand All @@ -145,10 +145,10 @@ public ImportadorParlamentarAmapa(IServiceProvider serviceProvider) : base(servi
// UrlPerfil = $"http://www.al.ap.gov.br/pagina.php?pg=exibir_parlamentar&iddeputado={matricula}"
// };

// var subDocument = await context.OpenAsync(deputado.UrlPerfil);
// var subDocument = await context.OpenAsyncAutoRetry(deputado.UrlPerfil);
// if (document.StatusCode != HttpStatusCode.OK)
// {
// logger.LogError("Erro ao consultar deputado: {NomeDeputado} {StatusCode}", deputado.UrlPerfil, subDocument.StatusCode);
// logger.LogError("Erro ao consultar parlamentar: {NomeDeputado} {StatusCode}", deputado.UrlPerfil, subDocument.StatusCode);
// continue;
// };

Expand Down
Loading

0 comments on commit 7985d22

Please sign in to comment.