[Chore] Add Prometheus support for insights (#96)

* Install prometheus exporter

* Add prometheus service provider

* Add census metrics

* Add setup agent script

* Ignore agent

* Add production dockerfile

* Add agent config

* Add install agent composer task

* Add agent worker

* Move setup agent to post install

* Add prometheus service to compose

* Run linter

* Implement collectors

* Fix prometheus network

* Add agent doc

* Add laravel scrape config

* Run lint fix

* Add gauge helpers

* Convert site insights to gauge

* Override screen handle to fix save method params

Used the workaround proposed in this [PR](orchidsoftware/platform#2299)

* Add seed allocation gauge

* Add model typehint

* Convert batch insights to gauge

* Remove redundant crop metric

* Convert crop insights to gauge

* Add Grafana dashboard export jsons

* Change values to float

* Remove unnecessary cast

* Add crop profit dashboard

* Convert farmland insights to gauge

* Run lint fix

* Add farmland dashboard

* Add estimated yield amount metric

* Convert farmer report insights to gauge

* Add farmer report dashboard

* Add site dashboard

.dockerignore

@@ -0,0 +1,27 @@
+.git
+.gitignore
+.github
+/node_modules
+/public/hot
+/public/storage
+/storage/*.key
+/storage/framework/cache
+/storage/framework/views
+/storage/app
+/docker
+/tests
+/vendor
+.env
+.env.ci
+.devcontainer
+.vscode
+.editorconfig
+.styleci.yml
+docker-compose.yml
+pint.json
+phpcs.xml
+phpunit.xml
+.phpunit.result.cache
+.phpstorm.meta.php
+.editorconfig
+*.log

.gitignore

@@ -15,3 +15,4 @@ database.env
 .php-cs-fixer.cache
 .vscode
 influxdb.env
+agent-linux-amd64

Dockerfile

@@ -0,0 +1,20 @@
+FROM richarvey/nginx-php-fpm:1.7.2
+
+COPY . .
+
+# Image config
+ENV SKIP_COMPOSER 1
+ENV WEBROOT /var/www/html/public
+ENV PHP_ERRORS_STDERR 1
+ENV RUN_SCRIPTS 1
+ENV REAL_IP_HEADER 1
+
+# Laravel config
+ENV APP_ENV production
+ENV APP_DEBUG false
+ENV LOG_CHANNEL stderr
+
+# Allow composer to run as root
+ENV COMPOSER_ALLOW_SUPERUSER 1
+
+CMD ["/start.sh"]

Procfile

@@ -1,2 +1,3 @@
 web: vendor/bin/heroku-php-apache2 public/
 worker: php artisan queue:restart && php artisan queue:work --tries=3
+agent: ./agent-linux-amd64 -config.file=agent-config.yml -config.expand-env

README.md

@@ -50,3 +50,13 @@ The exact PHP and MariaDB versions can be found in [XAMPP 8.1.6](https://www.apa
 -   Open `anikultura-backend`, which is in Ubuntu again, using VS Code.
 -   A more detailed article is available [here](https://laravel.com/docs/9.x/sail).
 -   Done!
+
+## Metrics
+
+### Grafana Cloud Agent
+
+Send metrics via the agent.
+
+```bash
+PROMETHEUS_PUSH_URL=http://localhost:9090/api/v1/write PROMETHEUS_TARGET_URL=localhost ./agent-linux-amd64 -config.file=agent-config.yml -config.expand-env
+```

agent-config.yml

@@ -0,0 +1,15 @@
+metrics:
+  configs:
+    - name: integrations
+      remote_write:
+        - basic_auth:
+            password: ${PROMETHEUS_PASSWORD}
+            username: ${PROMETHEUS_USERNAME}
+          url: ${PROMETHEUS_PUSH_URL}
+      scrape_configs:
+        - job_name: app
+          static_configs:
+            - targets: ["${PROMETHEUS_TARGET_URL}"]
+  global:
+    scrape_interval: 60s
+  wal_directory: /tmp/grafana-agent-wal

app/Collectors/BatchCollector.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace App\Collectors;
+
+use Arquivei\LaravelPrometheusExporter\CollectorInterface;
+use Arquivei\LaravelPrometheusExporter\PrometheusExporter;
+
+class BatchCollector implements CollectorInterface
+{
+    public function getName(): string
+    {
+        return 'batches';
+    }
+
+    public function registerMetrics(PrometheusExporter $exporter): void
+    {
+        $exporter->registerGauge(
+            'batch_total',
+            'The total number of batches.',
+            ['region', 'province', 'municity']
+        );
+
+        $exporter->registerGauge(
+            'batch_seed_allocation_total',
+            'The total number of batch seed allocations.',
+            ['crop', 'region', 'province', 'municity']
+        );
+    }
+
+    public function collect(): void
+    {
+    }
+}

app/Collectors/CropCollector.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace App\Collectors;
+
+use Arquivei\LaravelPrometheusExporter\CollectorInterface;
+use Arquivei\LaravelPrometheusExporter\PrometheusExporter;
+
+class CropCollector implements CollectorInterface
+{
+    public function getName(): string
+    {
+        return 'crops';
+    }
+
+    public function registerMetrics(PrometheusExporter $exporter): void
+    {
+        $exporter->registerGauge(
+            'crop_profit_per_kg_pesos',
+            'The total profit per kilogram of crops.',
+            ['crop']
+        );
+
+        $exporter->registerGauge(
+            'crop_net_profit_cost_ratio',
+            'The total net profit cost ratio of crops.',
+            ['crop']
+        );
+    }
+
+    public function collect(): void
+    {
+    }
+}

app/Collectors/FarmerCollector.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace App\Collectors;
+
+use Arquivei\LaravelPrometheusExporter\CollectorInterface;
+use Arquivei\LaravelPrometheusExporter\PrometheusExporter;
+
+class FarmerCollector implements CollectorInterface
+{
+    public function getName(): string
+    {
+        return 'farmers';
+    }
+
+    public function registerMetrics(PrometheusExporter $exporter): void
+    {
+        $exporter->registerGauge(
+            'farmer_total',
+            'The total number of farmers.',
+            ['region', 'province', 'municity']
+        );
+
+        $exporter->registerGauge(
+            'farmer_report_total',
+            'The total number of farmer reports.',
+            ['crop', 'seed_stage', 'region', 'province', 'municity']
+        );
+
+        $exporter->registerGauge(
+            'farmer_report_estimated_yield_grams',
+            'The estimated yield in grams from farmer reports.',
+            ['crop', 'region', 'province', 'municity', 'yield_date_earliest', 'yield_date_latest']
+        );
+    }
+
+    public function collect(): void
+    {
+    }
+}

app/Collectors/FarmlandCollector.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace App\Collectors;
+
+use Arquivei\LaravelPrometheusExporter\CollectorInterface;
+use Arquivei\LaravelPrometheusExporter\PrometheusExporter;
+
+class FarmlandCollector implements CollectorInterface
+{
+    public function getName(): string
+    {
+        return 'farmlands';
+    }
+
+    public function registerMetrics(PrometheusExporter $exporter): void
+    {
+        $exporter->registerGauge(
+            'farmland_total',
+            'The total number of farmlands.',
+            ['type', 'status', 'region', 'province', 'municity']
+        );
+
+        $exporter->registerGauge(
+            'farmland_hectares',
+            'The total hectares of farmlands.',
+            ['type', 'status', 'region', 'province', 'municity']
+        );
+    }
+
+    public function collect(): void
+    {
+    }
+}

app/Collectors/SiteCollector.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Collectors;
+
+use Arquivei\LaravelPrometheusExporter\CollectorInterface;
+use Arquivei\LaravelPrometheusExporter\PrometheusExporter;
+
+class SiteCollector implements CollectorInterface
+{
+    public function getName(): string
+    {
+        return 'sites';
+    }
+
+    public function registerMetrics(PrometheusExporter $exporter): void
+    {
+        $exporter->registerGauge(
+            'municipality_city_total',
+            'The total number of site municipalities and cities.',
+            ['region', 'province']
+        );
+
+        $exporter->registerGauge(
+            'province_total',
+            'The total number of site provinces.',
+            ['region']
+        );
+
+        $exporter->registerGauge(
+            'region_total',
+            'The total number of site regions.'
+        );
+    }
+
+    public function collect(): void
+    {
+    }
+}

app/Helpers/InsightsHelper.php

@@ -4,6 +4,16 @@
 
 class InsightsHelper
 {
+    public static function incrementGauge(string $name, array $labels = [], float $increment = 1): void
+    {
+        app('prometheus')->getGauge($name)->incBy($increment, $labels);
+    }
+
+    public static function decrementGauge(string $name, array $labels = [], float $decrement = 1): void
+    {
+        app('prometheus')->getGauge($name)->decBy($decrement, $labels);
+    }
+
     public static function isInsightsEnabled(): bool
     {
         return config('influxdb.enabled');

app/Observers/Batch/BatchObserver.php

@@ -2,46 +2,63 @@
 
 namespace App\Observers\Batch;
 
-use App\Actions\Insights\CreateCensusMetric;
-use App\Actions\Insights\Farmer\CreateFarmerEnrollmentMetric;
+use App\Helpers\InsightsHelper;
 use App\Models\Batch\Batch;
-use App\Models\Farmer\Farmer;
 use App\Traits\AsInsightSender;
+use Illuminate\Database\Eloquent\Model;
 
 class BatchObserver
 {
     use AsInsightSender;
 
-    private function sendInsights($model, bool $shouldIncrement)
+    private function sendInsights(Model $model, bool $shouldIncrement)
     {
-        CreateCensusMetric::dispatch(
-            [
-                'model' => [
-                    'id' => $model->id,
-                    'class' => Batch::class,
-                ],
-                'point' => [
-                    'increment' => $shouldIncrement,
-                    'measurement' => 'census-batch',
-                    'tags' => [
-                        'region' => 'id',
-                        'province' => 'id',
-                        'municity' => 'id',
-                    ],
-                ],
-            ]
-        );
+        $labels = [
+            'region' => $model->region->slug,
+            'province' => $model->province->slug,
+            'municity' => $model->municity->slug,
+        ];
+
+        if ($shouldIncrement) {
+            InsightsHelper::incrementGauge('batch_total', $labels);
+        } else {
+            InsightsHelper::decrementGauge('batch_total', $labels);
+        }
     }
 
     /**
-     * Farmer assigned to batch event.
+     * Handles the farmer assigned to batch event.
      */
     public function belongsToManyAttached(string $relation, Batch $batch, array $farmerIds)
     {
         if ($relation != 'farmers') {
             return;
         }
 
-        CreateFarmerEnrollmentMetric::dispatch($batch, $farmerIds);
+        $farmerAssignedCount = count($farmerIds);
+
+        InsightsHelper::incrementGauge('farmer_total', [
+            'region' => $batch->region->slug,
+            'province' => $batch->province->slug,
+            'municity' => $batch->municity->slug,
+        ], $farmerAssignedCount);
+    }
+
+    /**
+     * Handles the farmer unassigned to batch event.
+     */
+    public function belongsToManyDetached(string $relation, Batch $batch, array $farmerIds)
+    {
+        if ($relation != 'farmers') {
+            return;
+        }
+
+        $farmerAssignedCount = count($farmerIds);
+
+        InsightsHelper::decrementGauge('farmer_total', [
+            'region' => $batch->region->slug,
+            'province' => $batch->province->slug,
+            'municity' => $batch->municity->slug,
+        ], $farmerAssignedCount);
     }
 }