From 3982d94b1f93c024bf45fbc25119ad48e85ae401 Mon Sep 17 00:00:00 2001 From: Michal Obirek Date: Tue, 12 Dec 2023 15:28:48 +0100 Subject: [PATCH 1/5] feat: add fluentbit monitoring --- lib/facade/MonitoringFacade.ts | 8 + .../fluentbit/FluentBitConstants.ts | 29 ++ .../fluentbit/FluentBitMetricFactory.ts | 105 ++++++ .../fluentbit/FluentBitMonitoring.ts | 108 ++++++ lib/monitoring/fluentbit/index.ts | 3 + lib/monitoring/index.ts | 1 + .../fluentbit/FluentBitMonitoring.test.ts | 19 + .../FluentBitMonitoring.test.ts.snap | 347 ++++++++++++++++++ 8 files changed, 620 insertions(+) create mode 100644 lib/monitoring/fluentbit/FluentBitConstants.ts create mode 100644 lib/monitoring/fluentbit/FluentBitMetricFactory.ts create mode 100644 lib/monitoring/fluentbit/FluentBitMonitoring.ts create mode 100644 lib/monitoring/fluentbit/index.ts create mode 100644 test/monitoring/fluentbit/FluentBitMonitoring.test.ts create mode 100644 test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap diff --git a/lib/facade/MonitoringFacade.ts b/lib/facade/MonitoringFacade.ts index 17c77690..6a487b78 100644 --- a/lib/facade/MonitoringFacade.ts +++ b/lib/facade/MonitoringFacade.ts @@ -70,6 +70,8 @@ import { FargateNetworkLoadBalancerMonitoringProps, FargateServiceMonitoring, FargateServiceMonitoringProps, + FluentBitMonitoring, + FluentBitMonitoringProps, getQueueProcessingEc2ServiceMonitoring, getQueueProcessingFargateServiceMonitoring, GlueJobMonitoring, @@ -753,4 +755,10 @@ export class MonitoringFacade extends MonitoringScope { this.addSegment(segment, props); return this; } + + monitorFluentBit(props: FluentBitMonitoringProps) { + const segment = new FluentBitMonitoring(this, props); + this.addSegment(segment, props); + return this; + } } diff --git a/lib/monitoring/fluentbit/FluentBitConstants.ts b/lib/monitoring/fluentbit/FluentBitConstants.ts new file mode 100644 index 00000000..9f20055d --- /dev/null +++ b/lib/monitoring/fluentbit/FluentBitConstants.ts @@ -0,0 +1,29 @@ +export enum FluentBitStorageMetricTag { + TOTAL_CHUNKS = "total_chunks", + MEM_CHUNKS = "mem_chunks", + FS_CHUNKS = "fs_chunks", + FS_CHUNKS_UP = "fs_chunks_up", + FS_CHUNKS_DOWN = "fs_chunks_down", +} + +export enum FluentBitOutputMetricTag { + OUTPUT_RETRIES = "fluentbit_output_retries_total", + OUTPUT_RETRIES_FAILED = "fluentbit_output_retries_failed_total", + OUTPUT_ERRORS = "fluentbit_output_errors_total", + OUTPUT_DROPPED_RECORDS = "fluentbit_output_dropped_records_total", +} + +export enum FluentBitInputMetricTag { + INPUT_RECORDS = "fluentbit_input_records_total", +} +export enum FluentBitFilterMetricTag { + FILTER_EMIT_RECORDS = "fluentbit_filter_emit_records_total", + FILTER_DROP_RECORDS = "fluentbit_filter_drop_records_total", + FILTER_ADD_RECORDS = "fluentbit_filter_add_records_total", +} + +export enum FluentBitMetricsWithoutWidget { + INPUT_BYTES = "fluentbit_input_bytes_total", + OUTPUT_PROC_RECORDS = "fluentbit_output_proc_records_total", + OUTPUT_PROC_BYTES = "fluentbit_output_proc_bytes_total", +} diff --git a/lib/monitoring/fluentbit/FluentBitMetricFactory.ts b/lib/monitoring/fluentbit/FluentBitMetricFactory.ts new file mode 100644 index 00000000..e25d8864 --- /dev/null +++ b/lib/monitoring/fluentbit/FluentBitMetricFactory.ts @@ -0,0 +1,105 @@ +import { Metric } from "aws-cdk-lib/aws-cloudwatch"; +import { FilterPattern, ILogGroup, MetricFilter } from "aws-cdk-lib/aws-logs"; +import { + FluentBitFilterMetricTag, + FluentBitInputMetricTag, + FluentBitMetricsWithoutWidget, + FluentBitOutputMetricTag, + FluentBitStorageMetricTag, +} from "./FluentBitConstants"; +import { MetricFactory, MetricStatistic, MonitoringScope } from "../../common"; + +export interface FluentBitMetricFactoryProps { + /** + * Namespace that metrics will be emitted to. + * @default metric factory default + */ + readonly namespace?: string; +} + +export class FluentBitMetricFactory { + protected readonly metricFactory: MetricFactory; + protected readonly namespace: string; + protected readonly scope: MonitoringScope; + + constructor(scope: MonitoringScope, props: FluentBitMetricFactoryProps) { + this.scope = scope; + this.metricFactory = scope.createMetricFactory(); + this.namespace = + props.namespace ?? + this.metricFactory.getNamespaceWithFallback(props.namespace); + } + + fluentBitFilterMetrics(logGroup: ILogGroup) { + const filterMetrics: Metric[] = []; + Object.values(FluentBitFilterMetricTag).forEach((metricName) => { + const metric = this.pluginMetric(logGroup, metricName); + filterMetrics.push(metric); + }); + return filterMetrics; + } + + fluentBitOutputMetrics(logGroup: ILogGroup) { + const outputMetrics: Metric[] = []; + Object.values(FluentBitOutputMetricTag).forEach((metricName) => { + const metric = this.pluginMetric(logGroup, metricName); + outputMetrics.push(metric); + }); + return outputMetrics; + } + + fluentBitInputMetrics(logGroup: ILogGroup) { + const inputMetrics: Metric[] = []; + Object.values(FluentBitInputMetricTag).forEach((metricName) => { + const metric = this.pluginMetric(logGroup, metricName); + inputMetrics.push(metric); + }); + return inputMetrics; + } + + private pluginMetric(logGroup: ILogGroup, metricName: string) { + const metricFilter = new MetricFilter( + this.scope, + `FluentBit-${metricName}-${logGroup}-MetricFilter`, + { + logGroup: logGroup, + filterPattern: FilterPattern.literal(`{ $.metric = "${metricName}" }`), + metricNamespace: this.namespace, + metricName, + metricValue: "$.value", + } + ); + return metricFilter.metric({ + statistic: MetricStatistic.MAX, + }); + } + + fluentBitStorageMetrics(logGroup: ILogGroup) { + const storageMetrics: Metric[] = []; + Object.values(FluentBitStorageMetricTag).forEach((metricName) => { + const valueString = `$.storage_layer.chunks.${metricName}`; + const metricFilter = new MetricFilter( + this.scope, + `FluentBit-${metricName}-${logGroup}-MetricFilter`, + { + logGroup: logGroup, + filterPattern: FilterPattern.literal(`{ ${valueString} = * }`), + metricNamespace: this.namespace, + metricName, + metricValue: `${valueString}`, + } + ); + const metric = metricFilter.metric({ + statistic: MetricStatistic.MAX, + }); + storageMetrics.push(metric); + }); + return storageMetrics; + } + + fluentBitMetricsWithoutWidgets(logGroup: ILogGroup) { + Object.values(FluentBitMetricsWithoutWidget).forEach((metricName) => + this.pluginMetric(logGroup, metricName) + ); + } +} diff --git a/lib/monitoring/fluentbit/FluentBitMonitoring.ts b/lib/monitoring/fluentbit/FluentBitMonitoring.ts new file mode 100644 index 00000000..96c96c02 --- /dev/null +++ b/lib/monitoring/fluentbit/FluentBitMonitoring.ts @@ -0,0 +1,108 @@ +import { GraphWidget, IWidget, Metric } from "aws-cdk-lib/aws-cloudwatch"; +import { ILogGroup } from "aws-cdk-lib/aws-logs"; +import { + FluentBitMetricFactory, + FluentBitMetricFactoryProps, +} from "./FluentBitMetricFactory"; +import { + BaseMonitoringProps, + CountAxisFromZero, + DefaultGraphWidgetHeight, + HalfWidth, + MetricWithAlarmSupport, + Monitoring, + MonitoringScope, +} from "../../common"; +import { MonitoringHeaderWidget } from "../../dashboard"; + +export interface FluentBitMonitoringProps + extends FluentBitMetricFactoryProps, + BaseMonitoringProps { + /** + +Log group to which FluentBit emits metrics logs + */ + readonly logGroup: ILogGroup; +} + +export class FluentBitMonitoring extends Monitoring { + protected readonly logGroupName: string; + protected readonly metricFactory: FluentBitMetricFactory; + protected readonly fluentBitStorageMetrics: Metric[]; + protected readonly fluentBitInputMetrics: Metric[]; + protected readonly fluentBitOutputMetrics: Metric[]; + protected readonly fluentBitFilterMetrics: Metric[]; + + constructor(scope: MonitoringScope, props: FluentBitMonitoringProps) { + super(scope, props); + this.logGroupName = props.logGroup.logGroupName; + this.metricFactory = new FluentBitMetricFactory(scope, props); + + this.fluentBitStorageMetrics = this.metricFactory.fluentBitStorageMetrics( + props.logGroup + ); + this.fluentBitInputMetrics = this.metricFactory.fluentBitInputMetrics( + props.logGroup + ); + this.fluentBitOutputMetrics = this.metricFactory.fluentBitOutputMetrics( + props.logGroup + ); + this.fluentBitFilterMetrics = this.metricFactory.fluentBitFilterMetrics( + props.logGroup + ); + this.metricFactory.fluentBitMetricsWithoutWidgets(props.logGroup); + } + + widgets(): IWidget[] { + return [ + new MonitoringHeaderWidget({ + title: "FluentBit", + }), + this.createFluentBitInputWidget(), + this.createFluentBitOutputWidget(), + this.createFluentBitFilterWidget(), + this.createFluentBitStorageWidget(), + ]; + } + + private createFluentBitInputWidget() { + return this.createFluentBitMetricWidget( + [...Object.values(this.fluentBitInputMetrics)], + "Input Metrics" + ); + } + private createFluentBitOutputWidget() { + return this.createFluentBitMetricWidget( + [...Object.values(this.fluentBitOutputMetrics)], + "Output Metrics" + ); + } + + private createFluentBitFilterWidget() { + return this.createFluentBitMetricWidget( + [...Object.values(this.fluentBitFilterMetrics)], + "Filter Metrics" + ); + } + + private createFluentBitStorageWidget() { + return this.createFluentBitMetricWidget( + [...Object.values(this.fluentBitStorageMetrics)], + "Storage Metrics" + ); + } + + private createFluentBitMetricWidget( + metrics: MetricWithAlarmSupport[], + title: string + ): GraphWidget { + return new GraphWidget({ + width: HalfWidth, + height: DefaultGraphWidgetHeight, + title: `${title}`, + left: metrics, + leftAnnotations: undefined, + leftYAxis: CountAxisFromZero, + }); + } +} diff --git a/lib/monitoring/fluentbit/index.ts b/lib/monitoring/fluentbit/index.ts new file mode 100644 index 00000000..49a45d5c --- /dev/null +++ b/lib/monitoring/fluentbit/index.ts @@ -0,0 +1,3 @@ +export * from "./FluentBitConstants"; +export * from "./FluentBitMetricFactory"; +export * from "./FluentBitMonitoring"; diff --git a/lib/monitoring/index.ts b/lib/monitoring/index.ts index 3fb5df30..9dec5dc9 100644 --- a/lib/monitoring/index.ts +++ b/lib/monitoring/index.ts @@ -27,3 +27,4 @@ export * from "./aws-step-functions"; export * from "./aws-synthetics"; export * from "./aws-wafv2"; export * from "./custom"; +export * from "./fluentbit"; diff --git a/test/monitoring/fluentbit/FluentBitMonitoring.test.ts b/test/monitoring/fluentbit/FluentBitMonitoring.test.ts new file mode 100644 index 00000000..2593f961 --- /dev/null +++ b/test/monitoring/fluentbit/FluentBitMonitoring.test.ts @@ -0,0 +1,19 @@ +import { Stack } from "aws-cdk-lib"; +import { Template } from "aws-cdk-lib/assertions"; +import { LogGroup } from "aws-cdk-lib/aws-logs"; +import { FluentBitMonitoring } from "../../../lib"; +import { addMonitoringDashboardsToStack } from "../../utils/SnapshotUtil"; +import { TestMonitoringScope } from "../TestMonitoringScope"; + +test("snapshot test", () => { + const stack = new Stack(); + const scope = new TestMonitoringScope(stack, "Scope"); + const logGroup = new LogGroup(stack, "DummyLogGroup"); + const monitoring = new FluentBitMonitoring(scope, { + logGroup, + namespace: "DummyNamespace", + }); + + addMonitoringDashboardsToStack(stack, monitoring); + expect(Template.fromStack(stack)).toMatchSnapshot(); +}); diff --git a/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap b/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap new file mode 100644 index 00000000..fe69bf3d --- /dev/null +++ b/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap @@ -0,0 +1,347 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`snapshot test 1`] = ` +Object { + "Parameters": Object { + "BootstrapVersion": Object { + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]", + "Type": "AWS::SSM::Parameter::Value", + }, + }, + "Resources": Object { + "Alarm7103F465": Object { + "Properties": Object { + "DashboardBody": "{\\"widgets\\":[]}", + }, + "Type": "AWS::CloudWatch::Dashboard", + }, + "DummyLogGroup09717585": Object { + "DeletionPolicy": "Retain", + "Properties": Object { + "RetentionInDays": 731, + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "Resource": Object { + "Properties": Object { + "DashboardBody": Object { + "Fn::Join": Array [ + "", + Array [ + "{\\"widgets\\":[{\\"type\\":\\"text\\",\\"width\\":24,\\"height\\":1,\\"x\\":0,\\"y\\":0,\\"properties\\":{\\"markdown\\":\\"### FluentBit\\"}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":0,\\"y\\":1,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Input Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"fluentbit_input_records_total\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":12,\\"y\\":1,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Output Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"fluentbit_output_retries_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_retries_failed_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_errors_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_dropped_records_total\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":0,\\"y\\":6,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Filter Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"fluentbit_filter_emit_records_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_filter_drop_records_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_filter_add_records_total\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":12,\\"y\\":6,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Storage Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"total_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"mem_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks_up\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks_down\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}}]}", + ], + ], + }, + }, + "Type": "AWS::CloudWatch::Dashboard", + }, + "ScopeFluentBitfluentbitfilteraddrecordstotalDefaultDummyLogGroupMetricFilter63BA010D": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_filter_add_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_filter_add_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitfilterdroprecordstotalDefaultDummyLogGroupMetricFilter278C811C": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_filter_drop_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_filter_drop_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitfilteremitrecordstotalDefaultDummyLogGroupMetricFilter2054D228": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_filter_emit_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_filter_emit_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitinputbytestotalDefaultDummyLogGroupMetricFilter69312BC8": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_input_bytes_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_input_bytes_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitinputrecordstotalDefaultDummyLogGroupMetricFilter7383F883": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_input_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_input_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputdroppedrecordstotalDefaultDummyLogGroupMetricFilter10A7E033": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_dropped_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_dropped_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputerrorstotalDefaultDummyLogGroupMetricFilter26B2575E": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_errors_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_errors_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputprocbytestotalDefaultDummyLogGroupMetricFilter58311F26": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_proc_bytes_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_proc_bytes_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputprocrecordstotalDefaultDummyLogGroupMetricFilterC48AE6B4": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_proc_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_proc_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputretriesfailedtotalDefaultDummyLogGroupMetricFilter774C1B5E": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_retries_failed_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_retries_failed_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputretriestotalDefaultDummyLogGroupMetricFilter1ABA9ECA": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_retries_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_retries_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfschunksDefaultDummyLogGroupMetricFilterC8FA94CD": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.fs_chunks = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fs_chunks", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.fs_chunks", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfschunksdownDefaultDummyLogGroupMetricFilter631D8E97": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.fs_chunks_down = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fs_chunks_down", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.fs_chunks_down", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfschunksupDefaultDummyLogGroupMetricFilter4D35B33A": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.fs_chunks_up = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fs_chunks_up", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.fs_chunks_up", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitmemchunksDefaultDummyLogGroupMetricFilter8595E036": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.mem_chunks = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "mem_chunks", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.mem_chunks", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBittotalchunksDefaultDummyLogGroupMetricFilter972137A0": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.total_chunks = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "total_chunks", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.total_chunks", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "Summary68521F81": Object { + "Properties": Object { + "DashboardBody": "{\\"widgets\\":[]}", + }, + "Type": "AWS::CloudWatch::Dashboard", + }, + }, + "Rules": Object { + "CheckBootstrapVersion": Object { + "Assertions": Array [ + Object { + "Assert": Object { + "Fn::Not": Array [ + Object { + "Fn::Contains": Array [ + Array [ + "1", + "2", + "3", + "4", + "5", + ], + Object { + "Ref": "BootstrapVersion", + }, + ], + }, + ], + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI.", + }, + ], + }, + }, +} +`; From c712c9051eb376458ade7cc9eee2a66901cd2a57 Mon Sep 17 00:00:00 2001 From: Michal Obirek Date: Tue, 12 Dec 2023 16:34:55 +0100 Subject: [PATCH 2/5] fix: commit API.md changes --- .gitignore | 18 +- .vscode/settings.json | 2 +- API.md | 588 +++++++++++++++++++++++++++++++++++++++++- README.md | 1 + package.json | 4 +- 5 files changed, 599 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index fb5a8558..53205a86 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ # ~~ Generated by projen. To modify, edit .projenrc.ts and run "npx projen". !/.gitattributes -!/.projen/tasks.json -!/.projen/deps.json -!/.projen/files.json -!/.github/workflows/pull-request-lint.yml -!/.github/workflows/auto-approve.yml +!/.projen\tasks.json +!/.projen\deps.json +!/.projen\files.json +!/.github\workflows\pull-request-lint.yml +!/.github\workflows\auto-approve.yml !/package.json !/LICENSE !/.npmignore @@ -35,13 +35,13 @@ jspm_packages/ /test-reports/ junit.xml /coverage/ -!/.github/workflows/build.yml +!/.github\workflows\build.yml /dist/changelog.md /dist/version.txt -!/.github/workflows/release.yml +!/.github\workflows\release.yml !/.mergify.yml -!/.github/workflows/upgrade-main.yml -!/.github/pull_request_template.md +!/.github\workflows\upgrade-main.yml +!/.github\pull_request_template.md !/.prettierignore !/.prettierrc.json !/test/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 26852ec6..71f26d92 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { "editor.codeActionsOnSave": { - "source.fixAll.eslint": true + "source.fixAll.eslint": "explicit" }, "eslint.validate": ["javascript", "typescript"] } diff --git a/API.md b/API.md index 8665bbde..38580e45 100644 --- a/API.md +++ b/API.md @@ -1085,6 +1085,7 @@ new MonitoringFacade(scope: Construct, id: string, props?: MonitoringFacadeProps | monitorFargateApplicationLoadBalancer | *No description.* | | monitorFargateNetworkLoadBalancer | *No description.* | | monitorFargateService | *No description.* | +| monitorFluentBit | *No description.* | | monitorGlueJob | *No description.* | | monitorKinesisDataAnalytics | *No description.* | | monitorKinesisDataStream | *No description.* | @@ -1683,6 +1684,18 @@ public monitorFargateService(props: FargateServiceMonitoringProps): MonitoringFa --- +##### `monitorFluentBit` + +```typescript +public monitorFluentBit(props: FluentBitMonitoringProps): MonitoringFacade +``` + +###### `props`Required + +- *Type:* FluentBitMonitoringProps + +--- + ##### `monitorGlueJob` ```typescript @@ -19358,6 +19371,186 @@ Threshold value between [0.0, 1.0) for when the alarm should be triggered. --- +### FluentBitMetricFactoryProps + +#### Initializer + +```typescript +import { FluentBitMetricFactoryProps } from 'cdk-monitoring-constructs' + +const fluentBitMetricFactoryProps: FluentBitMetricFactoryProps = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| namespace | string | Namespace that metrics will be emitted to. | + +--- + +##### `namespace`Optional + +```typescript +public readonly namespace: string; +``` + +- *Type:* string +- *Default:* metric factory default + +Namespace that metrics will be emitted to. + +--- + +### FluentBitMonitoringProps + +#### Initializer + +```typescript +import { FluentBitMonitoringProps } from 'cdk-monitoring-constructs' + +const fluentBitMonitoringProps: FluentBitMonitoringProps = { ... } +``` + +#### Properties + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| namespace | string | Namespace that metrics will be emitted to. | +| alarmFriendlyName | string | Plain name, used in naming alarms. | +| humanReadableName | string | Human-readable name is a freeform string, used as a caption or description. | +| localAlarmNamePrefixOverride | string | If this is defined, the local alarm name prefix used in naming alarms for the construct will be set to this value. | +| addToAlarmDashboard | boolean | Flag indicating if the widgets should be added to alarm dashboard. | +| addToDetailDashboard | boolean | Flag indicating if the widgets should be added to detailed dashboard. | +| addToSummaryDashboard | boolean | Flag indicating if the widgets should be added to summary dashboard. | +| useCreatedAlarms | IAlarmConsumer | Calls provided function to process all alarms created. | +| logGroup | aws-cdk-lib.aws_logs.ILogGroup | Log group to which FluentBit emits metrics logs. | + +--- + +##### `namespace`Optional + +```typescript +public readonly namespace: string; +``` + +- *Type:* string +- *Default:* metric factory default + +Namespace that metrics will be emitted to. + +--- + +##### `alarmFriendlyName`Optional + +```typescript +public readonly alarmFriendlyName: string; +``` + +- *Type:* string +- *Default:* derives name from the construct itself + +Plain name, used in naming alarms. + +This unique among other resources, and respect the AWS CDK restriction posed on alarm names. +The length must be 1 - 255 characters and although the validation rules are undocumented, we recommend using ASCII and hyphens. + +--- + +##### `humanReadableName`Optional + +```typescript +public readonly humanReadableName: string; +``` + +- *Type:* string +- *Default:* use alarmFriendlyName + +Human-readable name is a freeform string, used as a caption or description. + +There are no limitations on what it can be. + +--- + +##### `localAlarmNamePrefixOverride`Optional + +```typescript +public readonly localAlarmNamePrefixOverride: string; +``` + +- *Type:* string + +If this is defined, the local alarm name prefix used in naming alarms for the construct will be set to this value. + +The length must be 1 - 255 characters and although the validation rules are undocumented, we recommend using ASCII and hyphens. + +> [AlarmNamingStrategy for more details on alarm name prefixes](AlarmNamingStrategy for more details on alarm name prefixes) + +--- + +##### `addToAlarmDashboard`Optional + +```typescript +public readonly addToAlarmDashboard: boolean; +``` + +- *Type:* boolean +- *Default:* true + +Flag indicating if the widgets should be added to alarm dashboard. + +--- + +##### `addToDetailDashboard`Optional + +```typescript +public readonly addToDetailDashboard: boolean; +``` + +- *Type:* boolean +- *Default:* true + +Flag indicating if the widgets should be added to detailed dashboard. + +--- + +##### `addToSummaryDashboard`Optional + +```typescript +public readonly addToSummaryDashboard: boolean; +``` + +- *Type:* boolean +- *Default:* true + +Flag indicating if the widgets should be added to summary dashboard. + +--- + +##### `useCreatedAlarms`Optional + +```typescript +public readonly useCreatedAlarms: IAlarmConsumer; +``` + +- *Type:* IAlarmConsumer + +Calls provided function to process all alarms created. + +--- + +##### `logGroup`Required + +```typescript +public readonly logGroup: ILogGroup; +``` + +- *Type:* aws-cdk-lib.aws_logs.ILogGroup + +Log group to which FluentBit emits metrics logs. + +--- + ### FullRestartCountThreshold #### Initializer @@ -55653,6 +55846,256 @@ Creates annotation based on the metric and alarm properties. +### FluentBitMetricFactory + +#### Initializers + +```typescript +import { FluentBitMetricFactory } from 'cdk-monitoring-constructs' + +new FluentBitMetricFactory(scope: MonitoringScope, props: FluentBitMetricFactoryProps) +``` + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| scope | MonitoringScope | *No description.* | +| props | FluentBitMetricFactoryProps | *No description.* | + +--- + +##### `scope`Required + +- *Type:* MonitoringScope + +--- + +##### `props`Required + +- *Type:* FluentBitMetricFactoryProps + +--- + +#### Methods + +| **Name** | **Description** | +| --- | --- | +| fluentBitFilterMetrics | *No description.* | +| fluentBitInputMetrics | *No description.* | +| fluentBitMetricsWithoutWidgets | *No description.* | +| fluentBitOutputMetrics | *No description.* | +| fluentBitStorageMetrics | *No description.* | + +--- + +##### `fluentBitFilterMetrics` + +```typescript +public fluentBitFilterMetrics(logGroup: ILogGroup): Metric[] +``` + +###### `logGroup`Required + +- *Type:* aws-cdk-lib.aws_logs.ILogGroup + +--- + +##### `fluentBitInputMetrics` + +```typescript +public fluentBitInputMetrics(logGroup: ILogGroup): Metric[] +``` + +###### `logGroup`Required + +- *Type:* aws-cdk-lib.aws_logs.ILogGroup + +--- + +##### `fluentBitMetricsWithoutWidgets` + +```typescript +public fluentBitMetricsWithoutWidgets(logGroup: ILogGroup): void +``` + +###### `logGroup`Required + +- *Type:* aws-cdk-lib.aws_logs.ILogGroup + +--- + +##### `fluentBitOutputMetrics` + +```typescript +public fluentBitOutputMetrics(logGroup: ILogGroup): Metric[] +``` + +###### `logGroup`Required + +- *Type:* aws-cdk-lib.aws_logs.ILogGroup + +--- + +##### `fluentBitStorageMetrics` + +```typescript +public fluentBitStorageMetrics(logGroup: ILogGroup): Metric[] +``` + +###### `logGroup`Required + +- *Type:* aws-cdk-lib.aws_logs.ILogGroup + +--- + + + + +### FluentBitMonitoring + +#### Initializers + +```typescript +import { FluentBitMonitoring } from 'cdk-monitoring-constructs' + +new FluentBitMonitoring(scope: MonitoringScope, props: FluentBitMonitoringProps) +``` + +| **Name** | **Type** | **Description** | +| --- | --- | --- | +| scope | MonitoringScope | *No description.* | +| props | FluentBitMonitoringProps | *No description.* | + +--- + +##### `scope`Required + +- *Type:* MonitoringScope + +--- + +##### `props`Required + +- *Type:* FluentBitMonitoringProps + +--- + +#### Methods + +| **Name** | **Description** | +| --- | --- | +| addAlarm | Adds an alarm. | +| alarmWidgets | Returns widgets for all alarms. | +| createAlarmFactory | Creates a new alarm factory. | +| createdAlarms | Returns all the alarms created. | +| createMetricFactory | Creates a new metric factory. | +| createWidgetFactory | Creates a new widget factory. | +| summaryWidgets | Returns widgets to be placed on the summary dashboard. | +| widgets | Returns widgets to be placed on the main dashboard. | +| widgetsForDashboard | Returns widgets for the requested dashboard type. | + +--- + +##### `addAlarm` + +```typescript +public addAlarm(alarm: AlarmWithAnnotation): void +``` + +Adds an alarm. + +###### `alarm`Required + +- *Type:* AlarmWithAnnotation + +alarm to add. + +--- + +##### `alarmWidgets` + +```typescript +public alarmWidgets(): IWidget[] +``` + +Returns widgets for all alarms. + +These can go to runbook or to service dashboard. + +##### `createAlarmFactory` + +```typescript +public createAlarmFactory(alarmNamePrefix: string): AlarmFactory +``` + +Creates a new alarm factory. + +Alarms created will be named with the given prefix, unless a local name override is present. + +###### `alarmNamePrefix`Required + +- *Type:* string + +alarm name prefix. + +--- + +##### `createdAlarms` + +```typescript +public createdAlarms(): AlarmWithAnnotation[] +``` + +Returns all the alarms created. + +##### `createMetricFactory` + +```typescript +public createMetricFactory(): MetricFactory +``` + +Creates a new metric factory. + +##### `createWidgetFactory` + +```typescript +public createWidgetFactory(): IWidgetFactory +``` + +Creates a new widget factory. + +##### `summaryWidgets` + +```typescript +public summaryWidgets(): IWidget[] +``` + +Returns widgets to be placed on the summary dashboard. + +##### `widgets` + +```typescript +public widgets(): IWidget[] +``` + +Returns widgets to be placed on the main dashboard. + +##### `widgetsForDashboard` + +```typescript +public widgetsForDashboard(name: string): IWidget[] +``` + +Returns widgets for the requested dashboard type. + +###### `name`Required + +- *Type:* string + +--- + + + + ### GlueJobMetricFactory #### Initializers @@ -70488,7 +70931,7 @@ Dashboard placement override props. ### IDashboardSegment -- *Implemented By:* ApiGatewayMonitoring, ApiGatewayV2HttpApiMonitoring, AppSyncMonitoring, AuroraClusterMonitoring, AutoScalingGroupMonitoring, BillingMonitoring, CertificateManagerMonitoring, CloudFrontDistributionMonitoring, CodeBuildProjectMonitoring, CustomMonitoring, DocumentDbMonitoring, DynamoTableGlobalSecondaryIndexMonitoring, DynamoTableMonitoring, EC2Monitoring, Ec2ServiceMonitoring, ElastiCacheClusterMonitoring, FargateServiceMonitoring, GlueJobMonitoring, KinesisDataAnalyticsMonitoring, KinesisDataStreamMonitoring, KinesisFirehoseMonitoring, LambdaFunctionMonitoring, LogMonitoring, Monitoring, NetworkLoadBalancerMonitoring, OpenSearchClusterMonitoring, RdsClusterMonitoring, RedshiftClusterMonitoring, S3BucketMonitoring, SecretsManagerMonitoring, SecretsManagerSecretMonitoring, SingleWidgetDashboardSegment, SnsTopicMonitoring, SqsQueueMonitoring, SqsQueueMonitoringWithDlq, StepFunctionActivityMonitoring, StepFunctionLambdaIntegrationMonitoring, StepFunctionMonitoring, StepFunctionServiceIntegrationMonitoring, SyntheticsCanaryMonitoring, WafV2Monitoring, IDashboardSegment +- *Implemented By:* ApiGatewayMonitoring, ApiGatewayV2HttpApiMonitoring, AppSyncMonitoring, AuroraClusterMonitoring, AutoScalingGroupMonitoring, BillingMonitoring, CertificateManagerMonitoring, CloudFrontDistributionMonitoring, CodeBuildProjectMonitoring, CustomMonitoring, DocumentDbMonitoring, DynamoTableGlobalSecondaryIndexMonitoring, DynamoTableMonitoring, EC2Monitoring, Ec2ServiceMonitoring, ElastiCacheClusterMonitoring, FargateServiceMonitoring, FluentBitMonitoring, GlueJobMonitoring, KinesisDataAnalyticsMonitoring, KinesisDataStreamMonitoring, KinesisFirehoseMonitoring, LambdaFunctionMonitoring, LogMonitoring, Monitoring, NetworkLoadBalancerMonitoring, OpenSearchClusterMonitoring, RdsClusterMonitoring, RedshiftClusterMonitoring, S3BucketMonitoring, SecretsManagerMonitoring, SecretsManagerSecretMonitoring, SingleWidgetDashboardSegment, SnsTopicMonitoring, SqsQueueMonitoring, SqsQueueMonitoringWithDlq, StepFunctionActivityMonitoring, StepFunctionLambdaIntegrationMonitoring, StepFunctionMonitoring, StepFunctionServiceIntegrationMonitoring, SyntheticsCanaryMonitoring, WafV2Monitoring, IDashboardSegment #### Methods @@ -70579,7 +71022,7 @@ Gets the dashboard for the requested dashboard type. ### IDynamicDashboardSegment -- *Implemented By:* ApiGatewayMonitoring, ApiGatewayV2HttpApiMonitoring, AppSyncMonitoring, AuroraClusterMonitoring, AutoScalingGroupMonitoring, BillingMonitoring, CertificateManagerMonitoring, CloudFrontDistributionMonitoring, CodeBuildProjectMonitoring, CustomMonitoring, DocumentDbMonitoring, DynamoTableGlobalSecondaryIndexMonitoring, DynamoTableMonitoring, EC2Monitoring, Ec2ServiceMonitoring, ElastiCacheClusterMonitoring, FargateServiceMonitoring, GlueJobMonitoring, KinesisDataAnalyticsMonitoring, KinesisDataStreamMonitoring, KinesisFirehoseMonitoring, LambdaFunctionMonitoring, LogMonitoring, Monitoring, NetworkLoadBalancerMonitoring, OpenSearchClusterMonitoring, RdsClusterMonitoring, RedshiftClusterMonitoring, S3BucketMonitoring, SecretsManagerMonitoring, SecretsManagerSecretMonitoring, SingleWidgetDashboardSegment, SnsTopicMonitoring, SqsQueueMonitoring, SqsQueueMonitoringWithDlq, StaticSegmentDynamicAdapter, StepFunctionActivityMonitoring, StepFunctionLambdaIntegrationMonitoring, StepFunctionMonitoring, StepFunctionServiceIntegrationMonitoring, SyntheticsCanaryMonitoring, WafV2Monitoring, IDynamicDashboardSegment +- *Implemented By:* ApiGatewayMonitoring, ApiGatewayV2HttpApiMonitoring, AppSyncMonitoring, AuroraClusterMonitoring, AutoScalingGroupMonitoring, BillingMonitoring, CertificateManagerMonitoring, CloudFrontDistributionMonitoring, CodeBuildProjectMonitoring, CustomMonitoring, DocumentDbMonitoring, DynamoTableGlobalSecondaryIndexMonitoring, DynamoTableMonitoring, EC2Monitoring, Ec2ServiceMonitoring, ElastiCacheClusterMonitoring, FargateServiceMonitoring, FluentBitMonitoring, GlueJobMonitoring, KinesisDataAnalyticsMonitoring, KinesisDataStreamMonitoring, KinesisFirehoseMonitoring, LambdaFunctionMonitoring, LogMonitoring, Monitoring, NetworkLoadBalancerMonitoring, OpenSearchClusterMonitoring, RdsClusterMonitoring, RedshiftClusterMonitoring, S3BucketMonitoring, SecretsManagerMonitoring, SecretsManagerSecretMonitoring, SingleWidgetDashboardSegment, SnsTopicMonitoring, SqsQueueMonitoring, SqsQueueMonitoringWithDlq, StaticSegmentDynamicAdapter, StepFunctionActivityMonitoring, StepFunctionLambdaIntegrationMonitoring, StepFunctionMonitoring, StepFunctionServiceIntegrationMonitoring, SyntheticsCanaryMonitoring, WafV2Monitoring, IDynamicDashboardSegment #### Methods @@ -71062,6 +71505,147 @@ Create a two sets of dashboards: standard set (interactive) and a copy (bitmap). --- +### FluentBitFilterMetricTag + +#### Members + +| **Name** | **Description** | +| --- | --- | +| FILTER_EMIT_RECORDS | *No description.* | +| FILTER_DROP_RECORDS | *No description.* | +| FILTER_ADD_RECORDS | *No description.* | + +--- + +##### `FILTER_EMIT_RECORDS` + +--- + + +##### `FILTER_DROP_RECORDS` + +--- + + +##### `FILTER_ADD_RECORDS` + +--- + + +### FluentBitInputMetricTag + +#### Members + +| **Name** | **Description** | +| --- | --- | +| INPUT_RECORDS | *No description.* | + +--- + +##### `INPUT_RECORDS` + +--- + + +### FluentBitMetricsWithoutWidget + +#### Members + +| **Name** | **Description** | +| --- | --- | +| INPUT_BYTES | *No description.* | +| OUTPUT_PROC_RECORDS | *No description.* | +| OUTPUT_PROC_BYTES | *No description.* | + +--- + +##### `INPUT_BYTES` + +--- + + +##### `OUTPUT_PROC_RECORDS` + +--- + + +##### `OUTPUT_PROC_BYTES` + +--- + + +### FluentBitOutputMetricTag + +#### Members + +| **Name** | **Description** | +| --- | --- | +| OUTPUT_RETRIES | *No description.* | +| OUTPUT_RETRIES_FAILED | *No description.* | +| OUTPUT_ERRORS | *No description.* | +| OUTPUT_DROPPED_RECORDS | *No description.* | + +--- + +##### `OUTPUT_RETRIES` + +--- + + +##### `OUTPUT_RETRIES_FAILED` + +--- + + +##### `OUTPUT_ERRORS` + +--- + + +##### `OUTPUT_DROPPED_RECORDS` + +--- + + +### FluentBitStorageMetricTag + +#### Members + +| **Name** | **Description** | +| --- | --- | +| TOTAL_CHUNKS | *No description.* | +| MEM_CHUNKS | *No description.* | +| FS_CHUNKS | *No description.* | +| FS_CHUNKS_UP | *No description.* | +| FS_CHUNKS_DOWN | *No description.* | + +--- + +##### `TOTAL_CHUNKS` + +--- + + +##### `MEM_CHUNKS` + +--- + + +##### `FS_CHUNKS` + +--- + + +##### `FS_CHUNKS_UP` + +--- + + +##### `FS_CHUNKS_DOWN` + +--- + + ### GraphWidgetType #### Members diff --git a/README.md b/README.md index dfbfe2ad..d5b5623a 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ You can browse the documentation at https://constructs.dev/packages/cdk-monitori | AWS EC2 (`.monitorEC2Instances()`) | CPU, disk operations, network | | | | AWS EC2 Auto Scaling Groups (`.monitorAutoScalingGroup()`) | Group size, instance status | | | | AWS ECS (`.monitorFargateService()`, `.monitorEc2Service()`, `.monitorSimpleFargateService()`, `monitorSimpleEc2Service()`, `.monitorQueueProcessingFargateService()`, `.monitorQueueProcessingEc2Service()`) | System resources and task health | Unhealthy task count, running tasks count, CPU/memory usage, and bytes processed by load balancer (if any) | Use for ecs-patterns load balanced ec2/fargate constructs (NetworkLoadBalancedEc2Service, NetworkLoadBalancedFargateService, ApplicationLoadBalancedEc2Service, ApplicationLoadBalancedFargateService) | +| FluentBit (`.monitorFluentBit()`) | Num of input records, Output failures & retries, Filter metrics, Storage metrics | | FluentBit needs proper configuration with metrics enabled: [Official sample configuration](https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/send-fb-internal-metrics-to-cw). This function creates MetricFilters to publish all FluentBit metrics. | | AWS ElastiCache (`.monitorElastiCacheCluster()`) | CPU/memory usage, evictions and connections | CPU, memory, items count | | | AWS Glue (`.monitorGlueJob()`) | Traffic, job status, memory/CPU usage | Failed/killed task count/rate | | | AWS Kinesis Data Analytics (`.monitorKinesisDataAnalytics`) | Up/Downtime, CPU/memory usage, KPU usage, checkpoint metrics, and garbage collection metrics | Downtime, full restart count | | diff --git a/package.json b/package.json index 21d07c67..07386998 100644 --- a/package.json +++ b/package.json @@ -83,8 +83,8 @@ "version": "0.0.0", "jest": { "testMatch": [ - "/lib/**/__tests__/**/*.ts?(x)", - "/(test|lib)/**/*(*.)@(spec|test).ts?(x)" + "**/lib/**/__tests__/**/*.ts?(x)", + "**/(test|lib)/**/*(*.)@(spec|test).ts?(x)" ], "clearMocks": true, "collectCoverage": true, From 2ad03ee56d5d714e8a08bf805659c36753f022fa Mon Sep 17 00:00:00 2001 From: Michal Obirek Date: Tue, 12 Dec 2023 16:54:01 +0100 Subject: [PATCH 3/5] fix: revert files changed by running on Windows --- .gitignore | 18 +++++++++--------- .vscode/settings.json | 2 +- package.json | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 53205a86..fb5a8558 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ # ~~ Generated by projen. To modify, edit .projenrc.ts and run "npx projen". !/.gitattributes -!/.projen\tasks.json -!/.projen\deps.json -!/.projen\files.json -!/.github\workflows\pull-request-lint.yml -!/.github\workflows\auto-approve.yml +!/.projen/tasks.json +!/.projen/deps.json +!/.projen/files.json +!/.github/workflows/pull-request-lint.yml +!/.github/workflows/auto-approve.yml !/package.json !/LICENSE !/.npmignore @@ -35,13 +35,13 @@ jspm_packages/ /test-reports/ junit.xml /coverage/ -!/.github\workflows\build.yml +!/.github/workflows/build.yml /dist/changelog.md /dist/version.txt -!/.github\workflows\release.yml +!/.github/workflows/release.yml !/.mergify.yml -!/.github\workflows\upgrade-main.yml -!/.github\pull_request_template.md +!/.github/workflows/upgrade-main.yml +!/.github/pull_request_template.md !/.prettierignore !/.prettierrc.json !/test/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 71f26d92..26852ec6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { "editor.codeActionsOnSave": { - "source.fixAll.eslint": "explicit" + "source.fixAll.eslint": true }, "eslint.validate": ["javascript", "typescript"] } diff --git a/package.json b/package.json index 07386998..21d07c67 100644 --- a/package.json +++ b/package.json @@ -83,8 +83,8 @@ "version": "0.0.0", "jest": { "testMatch": [ - "**/lib/**/__tests__/**/*.ts?(x)", - "**/(test|lib)/**/*(*.)@(spec|test).ts?(x)" + "/lib/**/__tests__/**/*.ts?(x)", + "/(test|lib)/**/*(*.)@(spec|test).ts?(x)" ], "clearMocks": true, "collectCoverage": true, From 10d726064083999d54eed3fa628c3dcb039efc78 Mon Sep 17 00:00:00 2001 From: Michal Obirek Date: Tue, 12 Dec 2023 17:47:41 +0100 Subject: [PATCH 4/5] fix: introduce changes suggested by @echeung --- API.md | 44 +++++----- README.md | 2 +- .../fluentbit/FluentBitMetricFactory.ts | 38 +++------ .../fluentbit/FluentBitMonitoring.ts | 81 ++++++++++--------- .../FluentBitMonitoring.test.ts.snap | 17 +++- 5 files changed, 94 insertions(+), 88 deletions(-) diff --git a/API.md b/API.md index 38580e45..7aa58b50 100644 --- a/API.md +++ b/API.md @@ -19424,7 +19424,7 @@ const fluentBitMonitoringProps: FluentBitMonitoringProps = { ... } | addToDetailDashboard | boolean | Flag indicating if the widgets should be added to detailed dashboard. | | addToSummaryDashboard | boolean | Flag indicating if the widgets should be added to summary dashboard. | | useCreatedAlarms | IAlarmConsumer | Calls provided function to process all alarms created. | -| logGroup | aws-cdk-lib.aws_logs.ILogGroup | Log group to which FluentBit emits metrics logs. | +| logGroup | aws-cdk-lib.aws_logs.ILogGroup | Log group that contains FluentBit metric logs. | --- @@ -19547,7 +19547,7 @@ public readonly logGroup: ILogGroup; - *Type:* aws-cdk-lib.aws_logs.ILogGroup -Log group to which FluentBit emits metrics logs. +Log group that contains FluentBit metric logs. --- @@ -55879,69 +55879,69 @@ new FluentBitMetricFactory(scope: MonitoringScope, props: FluentBitMetricFactory | **Name** | **Description** | | --- | --- | -| fluentBitFilterMetrics | *No description.* | -| fluentBitInputMetrics | *No description.* | -| fluentBitMetricsWithoutWidgets | *No description.* | -| fluentBitOutputMetrics | *No description.* | -| fluentBitStorageMetrics | *No description.* | +| filterMetrics | *No description.* | +| inputMetrics | *No description.* | +| metricsWithoutWidgets | *No description.* | +| outputMetrics | *No description.* | +| storageMetrics | *No description.* | --- -##### `fluentBitFilterMetrics` +##### `filterMetrics` ```typescript -public fluentBitFilterMetrics(logGroup: ILogGroup): Metric[] +public filterMetrics(logGroup: ILogGroup): Metric[] ``` -###### `logGroup`Required +###### `logGroup`Required - *Type:* aws-cdk-lib.aws_logs.ILogGroup --- -##### `fluentBitInputMetrics` +##### `inputMetrics` ```typescript -public fluentBitInputMetrics(logGroup: ILogGroup): Metric[] +public inputMetrics(logGroup: ILogGroup): Metric[] ``` -###### `logGroup`Required +###### `logGroup`Required - *Type:* aws-cdk-lib.aws_logs.ILogGroup --- -##### `fluentBitMetricsWithoutWidgets` +##### `metricsWithoutWidgets` ```typescript -public fluentBitMetricsWithoutWidgets(logGroup: ILogGroup): void +public metricsWithoutWidgets(logGroup: ILogGroup): void ``` -###### `logGroup`Required +###### `logGroup`Required - *Type:* aws-cdk-lib.aws_logs.ILogGroup --- -##### `fluentBitOutputMetrics` +##### `outputMetrics` ```typescript -public fluentBitOutputMetrics(logGroup: ILogGroup): Metric[] +public outputMetrics(logGroup: ILogGroup): Metric[] ``` -###### `logGroup`Required +###### `logGroup`Required - *Type:* aws-cdk-lib.aws_logs.ILogGroup --- -##### `fluentBitStorageMetrics` +##### `storageMetrics` ```typescript -public fluentBitStorageMetrics(logGroup: ILogGroup): Metric[] +public storageMetrics(logGroup: ILogGroup): Metric[] ``` -###### `logGroup`Required +###### `logGroup`Required - *Type:* aws-cdk-lib.aws_logs.ILogGroup diff --git a/README.md b/README.md index d5b5623a..15cbd6b9 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,6 @@ You can browse the documentation at https://constructs.dev/packages/cdk-monitori | AWS EC2 (`.monitorEC2Instances()`) | CPU, disk operations, network | | | | AWS EC2 Auto Scaling Groups (`.monitorAutoScalingGroup()`) | Group size, instance status | | | | AWS ECS (`.monitorFargateService()`, `.monitorEc2Service()`, `.monitorSimpleFargateService()`, `monitorSimpleEc2Service()`, `.monitorQueueProcessingFargateService()`, `.monitorQueueProcessingEc2Service()`) | System resources and task health | Unhealthy task count, running tasks count, CPU/memory usage, and bytes processed by load balancer (if any) | Use for ecs-patterns load balanced ec2/fargate constructs (NetworkLoadBalancedEc2Service, NetworkLoadBalancedFargateService, ApplicationLoadBalancedEc2Service, ApplicationLoadBalancedFargateService) | -| FluentBit (`.monitorFluentBit()`) | Num of input records, Output failures & retries, Filter metrics, Storage metrics | | FluentBit needs proper configuration with metrics enabled: [Official sample configuration](https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/send-fb-internal-metrics-to-cw). This function creates MetricFilters to publish all FluentBit metrics. | | AWS ElastiCache (`.monitorElastiCacheCluster()`) | CPU/memory usage, evictions and connections | CPU, memory, items count | | | AWS Glue (`.monitorGlueJob()`) | Traffic, job status, memory/CPU usage | Failed/killed task count/rate | | | AWS Kinesis Data Analytics (`.monitorKinesisDataAnalytics`) | Up/Downtime, CPU/memory usage, KPU usage, checkpoint metrics, and garbage collection metrics | Downtime, full restart count | | @@ -95,6 +94,7 @@ You can browse the documentation at https://constructs.dev/packages/cdk-monitori | AWS SQS Queue (`.monitorSqsQueue()`, `.monitorSqsQueueWithDlq()`) | Message count, age, size | Message count, age, DLQ incoming messages | | | AWS Step Functions (`.monitorStepFunction()`, `.monitorStepFunctionActivity()`, `monitorStepFunctionLambdaIntegration()`, `.monitorStepFunctionServiceIntegration()`) | Execution count and breakdown per state | Duration, failed, failed rate, aborted, throttled, timed out executions | | | AWS Web Application Firewall (`.monitorWebApplicationFirewallAclV2()`) | Allowed/blocked requests | Blocked requests count/rate | | +| FluentBit (`.monitorFluentBit()`) | Num of input records, Output failures & retries, Filter metrics, Storage metrics | | FluentBit needs proper configuration with metrics enabled: [Official sample configuration](https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/send-fb-internal-metrics-to-cw). This function creates MetricFilters to publish all FluentBit metrics. | | Custom metrics (`.monitorCustom()`) | Addition of custom metrics into the dashboard (each group is a widget) | | Supports anomaly detection | diff --git a/lib/monitoring/fluentbit/FluentBitMetricFactory.ts b/lib/monitoring/fluentbit/FluentBitMetricFactory.ts index e25d8864..03afb76a 100644 --- a/lib/monitoring/fluentbit/FluentBitMetricFactory.ts +++ b/lib/monitoring/fluentbit/FluentBitMetricFactory.ts @@ -1,4 +1,3 @@ -import { Metric } from "aws-cdk-lib/aws-cloudwatch"; import { FilterPattern, ILogGroup, MetricFilter } from "aws-cdk-lib/aws-logs"; import { FluentBitFilterMetricTag, @@ -30,31 +29,22 @@ export class FluentBitMetricFactory { this.metricFactory.getNamespaceWithFallback(props.namespace); } - fluentBitFilterMetrics(logGroup: ILogGroup) { - const filterMetrics: Metric[] = []; - Object.values(FluentBitFilterMetricTag).forEach((metricName) => { - const metric = this.pluginMetric(logGroup, metricName); - filterMetrics.push(metric); + filterMetrics(logGroup: ILogGroup) { + return Object.values(FluentBitFilterMetricTag).map((metricName) => { + return this.pluginMetric(logGroup, metricName); }); - return filterMetrics; } - fluentBitOutputMetrics(logGroup: ILogGroup) { - const outputMetrics: Metric[] = []; - Object.values(FluentBitOutputMetricTag).forEach((metricName) => { - const metric = this.pluginMetric(logGroup, metricName); - outputMetrics.push(metric); + outputMetrics(logGroup: ILogGroup) { + return Object.values(FluentBitOutputMetricTag).map((metricName) => { + return this.pluginMetric(logGroup, metricName); }); - return outputMetrics; } - fluentBitInputMetrics(logGroup: ILogGroup) { - const inputMetrics: Metric[] = []; - Object.values(FluentBitInputMetricTag).forEach((metricName) => { - const metric = this.pluginMetric(logGroup, metricName); - inputMetrics.push(metric); + inputMetrics(logGroup: ILogGroup) { + return Object.values(FluentBitInputMetricTag).map((metricName) => { + return this.pluginMetric(logGroup, metricName); }); - return inputMetrics; } private pluginMetric(logGroup: ILogGroup, metricName: string) { @@ -74,9 +64,8 @@ export class FluentBitMetricFactory { }); } - fluentBitStorageMetrics(logGroup: ILogGroup) { - const storageMetrics: Metric[] = []; - Object.values(FluentBitStorageMetricTag).forEach((metricName) => { + storageMetrics(logGroup: ILogGroup) { + return Object.values(FluentBitStorageMetricTag).map((metricName) => { const valueString = `$.storage_layer.chunks.${metricName}`; const metricFilter = new MetricFilter( this.scope, @@ -92,12 +81,11 @@ export class FluentBitMetricFactory { const metric = metricFilter.metric({ statistic: MetricStatistic.MAX, }); - storageMetrics.push(metric); + return metric; }); - return storageMetrics; } - fluentBitMetricsWithoutWidgets(logGroup: ILogGroup) { + metricsWithoutWidgets(logGroup: ILogGroup) { Object.values(FluentBitMetricsWithoutWidget).forEach((metricName) => this.pluginMetric(logGroup, metricName) ); diff --git a/lib/monitoring/fluentbit/FluentBitMonitoring.ts b/lib/monitoring/fluentbit/FluentBitMonitoring.ts index 96c96c02..0bf1fc93 100644 --- a/lib/monitoring/fluentbit/FluentBitMonitoring.ts +++ b/lib/monitoring/fluentbit/FluentBitMonitoring.ts @@ -20,7 +20,7 @@ export interface FluentBitMonitoringProps BaseMonitoringProps { /** -Log group to which FluentBit emits metrics logs +Log group that contains FluentBit metric logs */ readonly logGroup: ILogGroup; } @@ -28,78 +28,81 @@ Log group to which FluentBit emits metrics logs export class FluentBitMonitoring extends Monitoring { protected readonly logGroupName: string; protected readonly metricFactory: FluentBitMetricFactory; - protected readonly fluentBitStorageMetrics: Metric[]; - protected readonly fluentBitInputMetrics: Metric[]; - protected readonly fluentBitOutputMetrics: Metric[]; - protected readonly fluentBitFilterMetrics: Metric[]; + protected readonly storageMetrics: Metric[]; + protected readonly inputMetrics: Metric[]; + protected readonly outputMetrics: Metric[]; + protected readonly filterMetrics: Metric[]; constructor(scope: MonitoringScope, props: FluentBitMonitoringProps) { super(scope, props); this.logGroupName = props.logGroup.logGroupName; this.metricFactory = new FluentBitMetricFactory(scope, props); - this.fluentBitStorageMetrics = this.metricFactory.fluentBitStorageMetrics( - props.logGroup - ); - this.fluentBitInputMetrics = this.metricFactory.fluentBitInputMetrics( - props.logGroup - ); - this.fluentBitOutputMetrics = this.metricFactory.fluentBitOutputMetrics( - props.logGroup - ); - this.fluentBitFilterMetrics = this.metricFactory.fluentBitFilterMetrics( - props.logGroup - ); - this.metricFactory.fluentBitMetricsWithoutWidgets(props.logGroup); + this.storageMetrics = this.metricFactory.storageMetrics(props.logGroup); + this.inputMetrics = this.metricFactory.inputMetrics(props.logGroup); + this.outputMetrics = this.metricFactory.outputMetrics(props.logGroup); + this.filterMetrics = this.metricFactory.filterMetrics(props.logGroup); + this.metricFactory.metricsWithoutWidgets(props.logGroup); } widgets(): IWidget[] { return [ - new MonitoringHeaderWidget({ - title: "FluentBit", - }), - this.createFluentBitInputWidget(), - this.createFluentBitOutputWidget(), - this.createFluentBitFilterWidget(), - this.createFluentBitStorageWidget(), + this.createTitleWidget(), + this.inputMetricsWidget(), + this.outputMetricsWidget(), + this.filterMetricsWidget(), + this.storageMetricsWidget(), + ]; + } + + summaryWidgets(): IWidget[] { + return [ + this.createTitleWidget(), + this.outputMetricsWidget(), + this.storageMetricsWidget(), ]; } - private createFluentBitInputWidget() { - return this.createFluentBitMetricWidget( - [...Object.values(this.fluentBitInputMetrics)], + private createTitleWidget() { + return new MonitoringHeaderWidget({ + title: "FluentBit", + }); + } + private inputMetricsWidget() { + return this.createMetricWidget( + [...Object.values(this.inputMetrics)], "Input Metrics" ); } - private createFluentBitOutputWidget() { - return this.createFluentBitMetricWidget( - [...Object.values(this.fluentBitOutputMetrics)], + private outputMetricsWidget() { + return this.createMetricWidget( + [...Object.values(this.outputMetrics)], "Output Metrics" ); } - private createFluentBitFilterWidget() { - return this.createFluentBitMetricWidget( - [...Object.values(this.fluentBitFilterMetrics)], + private filterMetricsWidget() { + return this.createMetricWidget( + [...Object.values(this.filterMetrics)], "Filter Metrics" ); } - private createFluentBitStorageWidget() { - return this.createFluentBitMetricWidget( - [...Object.values(this.fluentBitStorageMetrics)], + private storageMetricsWidget() { + return this.createMetricWidget( + [...Object.values(this.storageMetrics)], "Storage Metrics" ); } - private createFluentBitMetricWidget( + private createMetricWidget( metrics: MetricWithAlarmSupport[], title: string ): GraphWidget { return new GraphWidget({ width: HalfWidth, height: DefaultGraphWidgetHeight, - title: `${title}`, + title, left: metrics, leftAnnotations: undefined, leftYAxis: CountAxisFromZero, diff --git a/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap b/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap index fe69bf3d..f004ff56 100644 --- a/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap +++ b/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap @@ -311,7 +311,22 @@ Object { }, "Summary68521F81": Object { "Properties": Object { - "DashboardBody": "{\\"widgets\\":[]}", + "DashboardBody": Object { + "Fn::Join": Array [ + "", + Array [ + "{\\"widgets\\":[{\\"type\\":\\"text\\",\\"width\\":24,\\"height\\":1,\\"x\\":0,\\"y\\":0,\\"properties\\":{\\"markdown\\":\\"### FluentBit\\"}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":0,\\"y\\":1,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Output Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"fluentbit_output_retries_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_retries_failed_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_errors_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_dropped_records_total\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":12,\\"y\\":1,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Storage Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"total_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"mem_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks_up\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks_down\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}}]}", + ], + ], + }, }, "Type": "AWS::CloudWatch::Dashboard", }, From 5611b95911a40ecd9aaf1755a00bdd1854ccbaee Mon Sep 17 00:00:00 2001 From: Michal Obirek Date: Tue, 12 Dec 2023 18:23:57 +0100 Subject: [PATCH 5/5] feat (fluentbit): add createOptionalMetricFilters flag --- API.md | 16 + .../fluentbit/FluentBitMonitoring.ts | 16 +- .../fluentbit/FluentBitMonitoring.test.ts | 16 +- .../FluentBitMonitoring.test.ts.snap | 315 +++++++++++++++++- 4 files changed, 357 insertions(+), 6 deletions(-) diff --git a/API.md b/API.md index 7aa58b50..26b8af33 100644 --- a/API.md +++ b/API.md @@ -19425,6 +19425,7 @@ const fluentBitMonitoringProps: FluentBitMonitoringProps = { ... } | addToSummaryDashboard | boolean | Flag indicating if the widgets should be added to summary dashboard. | | useCreatedAlarms | IAlarmConsumer | Calls provided function to process all alarms created. | | logGroup | aws-cdk-lib.aws_logs.ILogGroup | Log group that contains FluentBit metric logs. | +| createOptionalMetricFilters | boolean | Metrics for input bytes total, output bytes total and output records total are not shown on default dashboard. | --- @@ -19551,6 +19552,21 @@ Log group that contains FluentBit metric logs. --- +##### `createOptionalMetricFilters`Optional + +```typescript +public readonly createOptionalMetricFilters: boolean; +``` + +- *Type:* boolean +- *Default:* false + +Metrics for input bytes total, output bytes total and output records total are not shown on default dashboard. + +If you want to get MetricFilters created to have those metrics present in CloudWatch set this flag to true + +--- + ### FullRestartCountThreshold #### Initializer diff --git a/lib/monitoring/fluentbit/FluentBitMonitoring.ts b/lib/monitoring/fluentbit/FluentBitMonitoring.ts index 0bf1fc93..e24aed2d 100644 --- a/lib/monitoring/fluentbit/FluentBitMonitoring.ts +++ b/lib/monitoring/fluentbit/FluentBitMonitoring.ts @@ -19,10 +19,16 @@ export interface FluentBitMonitoringProps extends FluentBitMetricFactoryProps, BaseMonitoringProps { /** - -Log group that contains FluentBit metric logs - */ + * Log group that contains FluentBit metric logs + */ readonly logGroup: ILogGroup; + + /** + * Metrics for input bytes total, output bytes total and output records total are not shown on default dashboard. + * If you want to get MetricFilters created to have those metrics present in CloudWatch set this flag to true + * @default false + */ + readonly createOptionalMetricFilters?: boolean; } export class FluentBitMonitoring extends Monitoring { @@ -42,7 +48,9 @@ export class FluentBitMonitoring extends Monitoring { this.inputMetrics = this.metricFactory.inputMetrics(props.logGroup); this.outputMetrics = this.metricFactory.outputMetrics(props.logGroup); this.filterMetrics = this.metricFactory.filterMetrics(props.logGroup); - this.metricFactory.metricsWithoutWidgets(props.logGroup); + if (props.createOptionalMetricFilters) { + this.metricFactory.metricsWithoutWidgets(props.logGroup); + } } widgets(): IWidget[] { diff --git a/test/monitoring/fluentbit/FluentBitMonitoring.test.ts b/test/monitoring/fluentbit/FluentBitMonitoring.test.ts index 2593f961..cee90395 100644 --- a/test/monitoring/fluentbit/FluentBitMonitoring.test.ts +++ b/test/monitoring/fluentbit/FluentBitMonitoring.test.ts @@ -5,7 +5,7 @@ import { FluentBitMonitoring } from "../../../lib"; import { addMonitoringDashboardsToStack } from "../../utils/SnapshotUtil"; import { TestMonitoringScope } from "../TestMonitoringScope"; -test("snapshot test", () => { +test("snapshot test without all filters", () => { const stack = new Stack(); const scope = new TestMonitoringScope(stack, "Scope"); const logGroup = new LogGroup(stack, "DummyLogGroup"); @@ -17,3 +17,17 @@ test("snapshot test", () => { addMonitoringDashboardsToStack(stack, monitoring); expect(Template.fromStack(stack)).toMatchSnapshot(); }); + +test("snapshot test with all filters", () => { + const stack = new Stack(); + const scope = new TestMonitoringScope(stack, "Scope"); + const logGroup = new LogGroup(stack, "DummyLogGroup"); + const monitoring = new FluentBitMonitoring(scope, { + logGroup, + namespace: "DummyNamespace", + createOptionalMetricFilters: true, + }); + + addMonitoringDashboardsToStack(stack, monitoring); + expect(Template.fromStack(stack)).toMatchSnapshot(); +}); diff --git a/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap b/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap index f004ff56..c6044226 100644 --- a/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap +++ b/test/monitoring/fluentbit/__snapshots__/FluentBitMonitoring.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`snapshot test 1`] = ` +exports[`snapshot test with all filters 1`] = ` Object { "Parameters": Object { "BootstrapVersion": Object { @@ -360,3 +360,316 @@ Object { }, } `; + +exports[`snapshot test without all filters 1`] = ` +Object { + "Parameters": Object { + "BootstrapVersion": Object { + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]", + "Type": "AWS::SSM::Parameter::Value", + }, + }, + "Resources": Object { + "Alarm7103F465": Object { + "Properties": Object { + "DashboardBody": "{\\"widgets\\":[]}", + }, + "Type": "AWS::CloudWatch::Dashboard", + }, + "DummyLogGroup09717585": Object { + "DeletionPolicy": "Retain", + "Properties": Object { + "RetentionInDays": 731, + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "Resource": Object { + "Properties": Object { + "DashboardBody": Object { + "Fn::Join": Array [ + "", + Array [ + "{\\"widgets\\":[{\\"type\\":\\"text\\",\\"width\\":24,\\"height\\":1,\\"x\\":0,\\"y\\":0,\\"properties\\":{\\"markdown\\":\\"### FluentBit\\"}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":0,\\"y\\":1,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Input Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"fluentbit_input_records_total\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":12,\\"y\\":1,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Output Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"fluentbit_output_retries_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_retries_failed_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_errors_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_dropped_records_total\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":0,\\"y\\":6,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Filter Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"fluentbit_filter_emit_records_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_filter_drop_records_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_filter_add_records_total\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":12,\\"y\\":6,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Storage Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"total_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"mem_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks_up\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks_down\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}}]}", + ], + ], + }, + }, + "Type": "AWS::CloudWatch::Dashboard", + }, + "ScopeFluentBitfluentbitfilteraddrecordstotalDefaultDummyLogGroupMetricFilter63BA010D": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_filter_add_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_filter_add_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitfilterdroprecordstotalDefaultDummyLogGroupMetricFilter278C811C": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_filter_drop_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_filter_drop_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitfilteremitrecordstotalDefaultDummyLogGroupMetricFilter2054D228": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_filter_emit_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_filter_emit_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitinputrecordstotalDefaultDummyLogGroupMetricFilter7383F883": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_input_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_input_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputdroppedrecordstotalDefaultDummyLogGroupMetricFilter10A7E033": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_dropped_records_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_dropped_records_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputerrorstotalDefaultDummyLogGroupMetricFilter26B2575E": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_errors_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_errors_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputretriesfailedtotalDefaultDummyLogGroupMetricFilter774C1B5E": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_retries_failed_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_retries_failed_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfluentbitoutputretriestotalDefaultDummyLogGroupMetricFilter1ABA9ECA": Object { + "Properties": Object { + "FilterPattern": "{ $.metric = \\"fluentbit_output_retries_total\\" }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fluentbit_output_retries_total", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.value", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfschunksDefaultDummyLogGroupMetricFilterC8FA94CD": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.fs_chunks = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fs_chunks", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.fs_chunks", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfschunksdownDefaultDummyLogGroupMetricFilter631D8E97": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.fs_chunks_down = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fs_chunks_down", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.fs_chunks_down", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitfschunksupDefaultDummyLogGroupMetricFilter4D35B33A": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.fs_chunks_up = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "fs_chunks_up", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.fs_chunks_up", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBitmemchunksDefaultDummyLogGroupMetricFilter8595E036": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.mem_chunks = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "mem_chunks", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.mem_chunks", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "ScopeFluentBittotalchunksDefaultDummyLogGroupMetricFilter972137A0": Object { + "Properties": Object { + "FilterPattern": "{ $.storage_layer.chunks.total_chunks = * }", + "LogGroupName": Object { + "Ref": "DummyLogGroup09717585", + }, + "MetricTransformations": Array [ + Object { + "MetricName": "total_chunks", + "MetricNamespace": "DummyNamespace", + "MetricValue": "$.storage_layer.chunks.total_chunks", + }, + ], + }, + "Type": "AWS::Logs::MetricFilter", + }, + "Summary68521F81": Object { + "Properties": Object { + "DashboardBody": Object { + "Fn::Join": Array [ + "", + Array [ + "{\\"widgets\\":[{\\"type\\":\\"text\\",\\"width\\":24,\\"height\\":1,\\"x\\":0,\\"y\\":0,\\"properties\\":{\\"markdown\\":\\"### FluentBit\\"}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":0,\\"y\\":1,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Output Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"fluentbit_output_retries_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_retries_failed_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_errors_total\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fluentbit_output_dropped_records_total\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}},{\\"type\\":\\"metric\\",\\"width\\":12,\\"height\\":5,\\"x\\":12,\\"y\\":1,\\"properties\\":{\\"view\\":\\"timeSeries\\",\\"title\\":\\"Storage Metrics\\",\\"region\\":\\"", + Object { + "Ref": "AWS::Region", + }, + "\\",\\"metrics\\":[[\\"DummyNamespace\\",\\"total_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"mem_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks_up\\",{\\"stat\\":\\"Maximum\\"}],[\\"DummyNamespace\\",\\"fs_chunks_down\\",{\\"stat\\":\\"Maximum\\"}]],\\"yAxis\\":{\\"left\\":{\\"min\\":0,\\"label\\":\\"Count\\",\\"showUnits\\":false}}}}]}", + ], + ], + }, + }, + "Type": "AWS::CloudWatch::Dashboard", + }, + }, + "Rules": Object { + "CheckBootstrapVersion": Object { + "Assertions": Array [ + Object { + "Assert": Object { + "Fn::Not": Array [ + Object { + "Fn::Contains": Array [ + Array [ + "1", + "2", + "3", + "4", + "5", + ], + Object { + "Ref": "BootstrapVersion", + }, + ], + }, + ], + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI.", + }, + ], + }, + }, +} +`;