opensearch-project · seanneumann · Jul 20, 2023 · Jul 20, 2023
@@ -0,0 +1,33 @@
+# Observability Category: S3 Log Fields
+
+S3 log fields set describes a standardized structured representation of Amazon S3 bucket logs, enabling efficient monitoring, analysis, and management of operations performed on S3 buckets.
+
+## Field Names and Types
+
+| Field Name                | Type    |
+|---------------------------|---------|
+| aws.s3.bucket             | keyword |
+| aws.s3.key                | keyword |
+| aws.s3.copy_source        | keyword |
+| aws.s3.upload_id          | keyword |
+| aws.s3.delete             | keyword |
+| aws.s3.part_number        | keyword |
+
+## Field Explanations
+
+- **aws.s3.bucket**: The name of the S3 bucket.
+- **aws.s3.key**: The object key in the S3 bucket.
+- **aws.s3.copy_source**: The source from where the object was copied, if applicable.
+- **aws.s3.upload_id**: The upload identifier, if the operation is multipart upload.
+- **aws.s3.delete**: The deletion marker, if the operation was a deletion.
+- **aws.s3.part_number**: The part number, if the operation is a part of a multipart upload.
+
+## Fields for KPI Monitoring and Alerts
+
+The following fields are suitable for creating KPIs to monitor and alert when exhibiting abnormal behavior:
+
+- **aws.s3.bucket**: Monitoring operations on various buckets can help identify unauthorized access attempts or abnormal activity.
+- **aws.s3.key**: Tracking object keys can help identify frequently accessed or modified objects.
+- **aws.s3.delete**: Observing the deletion field can help identify accidental or malicious data deletions.
+
+By using these fields, users can efficiently monitor, analyze, and manage data in S3 buckets, aiding in performance optimization and security management.
@@ -0,0 +1,66 @@
+# Observability Category: VPC Flow Log Fields
+
+VPC flow logs field set describes a standardized structured representation of VPC network traffic flow information, enabling efficient monitoring, analysis, and management of network interactions across different cloud instances and services.
+
+## Field Names and Types
+
+| Field Name                     | Type    |
+|--------------------------------|---------|
+| aws.vpc.version                | keyword |
+| aws.vpc.account-id             | keyword |
+| aws.vpc.interface-id           | keyword |
+| cloud.region                   | keyword |
+| aws.vpc.vpc-id                 | keyword |
+| aws.vpc.subnet-id              | keyword |
+| aws.vpc.az-id                  | keyword |
+| aws.vpc.instance-id            | keyword |
+| aws.vpc.srcaddr                | ip      |
+| aws.vpc.dstaddr                | ip      |
+| aws.vpc.srcport                | integer |
+| aws.vpc.dstport                | integer |
+| aws.vpc.protocol               | keyword |
+| aws.vpc.packets                | integer |
+| aws.vpc.bytes                  | integer |
+| aws.vpc.pkt-src-aws-service    | keyword |
+| aws.vpc.pkt-dst-aws-service    | keyword |
+| aws.vpc.flow-direction         | keyword |
+| aws.vpc.start                  | keyword |
+| aws.vpc.end                    | keyword |
+| aws.vpc.action                 | keyword |
+| aws.vpc.log-status             | keyword |
+
+## Field Explanations
+
+- **aws.vpc.version**: The version of the VPC flow logs.
+- **aws.vpc.account-id**: The unique identifier of the AWS account.
+- **aws.vpc.interface-id**: The ID of the network interface for which the flow log was recorded.
+- **cloud.region**: The region of the AWS resource within the provider's infrastructure.
+- **aws.vpc.vpc-id**: The ID of the VPC for the flow log.
+- **aws.vpc.subnet-id**: The ID of the subnet for the flow log.
+- **aws.vpc.az-id**: The ID of the availability zone for the flow log.
+- **aws.vpc.instance-id**: The ID of the instance for the flow log.
+- **aws.vpc.srcaddr**: The source IP address of the traffic.
+- **aws.vpc.dstaddr**: The destination IP address of the traffic.
+- **aws.vpc.srcport**: The source port of the traffic.
+- **aws.vpc.dstport**: The destination port of the traffic.
+- **aws.vpc.protocol**: The protocol of the network traffic.
+- **aws.vpc.packets**: The number of packets transferred in the flow.
+- **aws.vpc.bytes**: The number of bytes transferred in the flow.
+- **aws.vpc.pkt-src-aws-service**: The AWS service that the flow originates from.
+- **aws.vpc.pkt-dst-aws-service**: The AWS service that the flow is sent to.
+- **aws.vpc.flow-direction**: The direction of the network flow, either ingress or egress.
+- **aws.vpc.start**: The start time of the flow.
+- **aws.vpc.end**: The end time of the flow.
+- **aws.vpc.action**: The action taken for the network flow, either ACCEPT or REJECT.
+- **aws.vpc.log-status**: The status of the logging, usually "OK".
+
+## Fields for KPI Monitoring and Alerts
+
+The following fields are suitable for creating KPIs to monitor and alert when exhibiting abnormal behavior:
+
+- **aws.vpc.packets**: Monitoring the number of packets can help identify unusual data transfer patterns, which could indicate a security concern.
+- **aws.vpc.bytes**: Monitoring the number of bytes can identify potential bandwidth issues.
+- **aws.vpc.action**: Keeping track of rejected actions can help identify potential security concerns or misconfigured security rules.
+- **aws.vpc.flow-direction**: Observing the flow direction can help identify potential security concerns or network configuration issues.
+
+By using these fields, users can efficiently monitor, analyze, and manage network traffic within their VPC, aiding in performance optimization and security management.
@@ -0,0 +1,136 @@
+[
+  {
+    "@timestamp": "2023-07-17T08:14:05.000Z",
+    "body": "2 121111111111 eni-0e250409d410e1290 162.142.125.177 10.0.0.200 38471 12313 6 1 44 1674898496 1674898507 ACCEPT OK",
+    "event": {
+      "result": "ACCEPT",
+      "name": "flow_log",
+      "domain": "vpc.flow_log"
+    },
+    "attributes": {
+      "data_stream": {
+        "dataset": "vpc.flow_log",
+        "namespace": "production",
+        "type": "logs_vpc"
+      }
+    },
+    "cloud": {
+      "provider": "aws",
+      "account": {
+        "id": "111111111111"
+      },
+      "region": "ap-southeast-2",
+      "resource_id": "vpc-0d4d4e82b7d743527",
+      "platform": "aws_vpc"
+    },
+    "aws": {
+      "s3": {
+        "bucket": "centralizedlogging-loghubloggingbucket0fa53b76-t57zyhgb8c2",
+        "key": "AWSLogs/111111111111/vpcflowlogs/us-east-2/2023/01/28/111111111111_vpcflowlogs_us-east-2_fl-023c6afa025ee5a04_20230128T0930Z_3a9dfd9d.log.gz"
+      },
+      "vpc": {
+        "version" : "2",
+        "account-id" : "111111111111",
+        "interface-id" : "eni-0e250409d410e1290",
+        "region": "ap-southeast-2",
+        "vpc-id": "vpc-0d4d4e82b7d743527",
+        "subnet-id": "subnet-aaaaaaaa012345678",
+        "az-id": "apse2-az3",
+        "instance-id": "i-0c50d5961bcb2d47b",
+        "srcaddr" : "162.142.125.177",
+        "dstaddr" : "10.0.0.200",
+        "srcport" : 38471,
+        "dstport" : 12313,
+        "protocol" : "6",
+        "packets" : 1,
+        "bytes" : 44,
+        "pkt-src-aws-service": "S3",
+        "pkt-dst-aws-service": "-",
+        "flow-direction": "ingress",
+        "start" : "1674898496",
+        "end" : "1674898507",
+        "action" : "ACCEPT",
+        "log-status" : "OK"
+      }
+    },
+    "communication": {
+      "source": {
+        "address": "162.142.125.177",
+        "port": 38471,
+        "packets": 1,
+        "bytes" : 44
+      },
+      "destination": {
+        "address": "10.0.0.200",
+        "port": 12313
+      }
+    }
+  },
+  {
+    "@timestamp": "2023-07-17T08:14:05.000Z",
+    "body": "2 111111111111 eni-0e250409d410e1290 162.142.125.177 10.0.0.200 38471 12313 6 1 44 1674898496 1674898507 ACCEPT OK",
+    "event": {
+      "result": "ACCEPT",
+      "name": "flow_log",
+      "domain": "vpc.flow_log"
+    },
+    "attributes": {
+      "data_stream": {
+        "dataset": "vpc.flow_log",
+        "namespace": "production",
+        "type": "logs_vpc"
+      }
+    },
+    "cloud": {
+      "provider": "aws",
+      "account": {
+        "id": "111111111111"
+      },
+      "region": "ap-southeast-2",
+      "resource_id": "vpc-0d4d4e82b7d743527",
+      "platform": "aws_vpc"
+    },
+    "aws": {
+      "s3": {
+        "bucket": "centralizedlogging-loghubloggingbucket0fa53b76-t57zyhgb8c2",
+        "key": "AWSLogs/111111111111/vpcflowlogs/us-east-2/2023/01/28/111111111111_vpcflowlogs_us-east-2_fl-023c6afa025ee5a04_20230128T0930Z_3a9dfd9d.log.gz"
+      },
+      "vpc": {
+        "version" : "2",
+        "account-id" : "111111111111",
+        "interface-id" : "eni-0e250409d410e1290",
+        "region": "ap-southeast-2",
+        "vpc-id": "vpc-0d4d4e82b7d743527",
+        "subnet-id": "subnet-aaaaaaaa012345678",
+        "az-id": "apse2-az3",
+        "instance-id": "i-0c50d5961bcb2d47b",
+        "srcaddr" : "10.0.0.200",
+        "dstaddr" : "162.142.125.177",
+        "srcport" : 12313,
+        "dstport" : 38471,
+        "protocol" : "6",
+        "packets" : 1,
+        "bytes" : 440,
+        "pkt-src-aws-service": "-",
+        "pkt-dst-aws-service": "S3",
+        "flow-direction": "egress",
+        "start" : "1674898496",
+        "end" : "1674898507",
+        "action" : "REJECT",
+        "log-status" : "OK"
+      }
+    },
+    "communication": {
+      "source": {
+        "address": "10.0.0.200",
+        "port": 12313,
+        "packets": 1,
+        "bytes" : 440
+      },
+      "destination": {
+        "address": "162.142.125.177",
+        "port": 38471
+      }
+    }
+  }
+]
@@ -0,0 +1,62 @@
+{
+  "catalog": "observability",
+  "version": "1.0",
+  "attributes": [
+    {
+      "aws.s3.bucket": {
+        "category": "aws_s3",
+        "component": "s3",
+        "caption": "Bucket",
+        "description": "Bucket name in S3",
+        "examples": ["my_bucket", "test_bucket"],
+        "object_name": "bucket",
+        "object_type": "keyword"
+      },
+      "aws.s3.key": {
+        "category": "aws_s3",
+        "component": "s3",
+        "caption": "Key",
+        "description": "Object key in S3",
+        "examples": ["my_object_key", "test_object_key"],
+        "object_name": "key",
+        "object_type": "keyword"
+      },
+      "aws.s3.copy_source": {
+        "category": "aws_s3",
+        "component": "s3",
+        "caption": "Copy Source",
+        "description": "Source of the copy operation in S3",
+        "examples": ["/source_bucket/my_object_key"],
+        "object_name": "copy_source",
+        "object_type": "keyword"
+      },
+      "aws.s3.upload_id": {
+        "category": "aws_s3",
+        "component": "s3",
+        "caption": "Upload ID",
+        "description": "ID of the upload operation in S3",
+        "examples": ["3D6B2C9A4B0944C8BA"],
+        "object_name": "upload_id",
+        "object_type": "keyword"
+      },
+      "aws.s3.delete": {
+        "category": "aws_s3",
+        "component": "s3",
+        "caption": "Delete",
+        "description": "Information about the delete operation in S3",
+        "examples": ["Success", "Failure"],
+        "object_name": "delete",
+        "object_type": "keyword"
+      },
+      "aws.s3.part_number": {
+        "category": "aws_s3",
+        "component": "s3",
+        "caption": "Part Number",
+        "description": "Part number in the multipart upload in S3",
+        "examples": ["1", "2", "3"],
+        "object_name": "part_number",
+        "object_type": "keyword"
+      }
+    }
+  ]
+}
@@ -0,0 +1,41 @@
+{
+  "template": {
+    "mappings": {
+      "_meta": {
+        "version": "1.0.0",
+        "catalog": "observability",
+        "type": "logs",
+        "component": "s3"
+      },
+      "properties": {
+        "aws": {
+          "type": "object",
+          "properties": {
+            "s3": {
+              "properties": {
+                "bucket": {
+                  "type": "keyword"
+                },
+                "key": {
+                  "type": "keyword"
+                },
+                "copy_source": {
+                  "type": "keyword"
+                },
+                "upload_id": {
+                  "type": "keyword"
+                },
+                "delete": {
+                  "type": "keyword"
+                },
+                "part_number": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}