antmedia-kafka-grafana.yaml

AWSTemplateFormatVersion: 2010-09-09
Description: >-
  Single EC2  CloudFormation Templates.
Parameters:
  VpcId:
    Type: 'AWS::EC2::VPC::Id'
    Description: VpcId of your existing Virtual Private Cloud (VPC)
    ConstraintDescription: must be the VPC Id of an existing Virtual Private Cloud.
  Subnets:
    Type: 'AWS::EC2::Subnet::Id'
    Description: The list of SubnetIds in your Virtual Private Cloud (VPC)
    ConstraintDescription: >-
      must be a list of at least two existing subnets associated with at least
      two different availability zones. They should be residing in the selected
      Virtual Private Cloud.
  InstanceType:
    Description: Ant Media Server EC2 instance type
    Type: String
    Default: t2.large
    AllowedValues:
      - t2.large
      - t2.xlarge
      - t2.2xlarge
      - m5.large
      - m5.xlarge
      - m5.2xlarge
      - m5.4xlarge
      - m5.12xlarge
      - m5.24xlarge
      - m4.large
      - m4.xlarge
      - m4.2xlarge
      - m4.4xlarge
      - m4.10xlarge
      - m4.16xlarge
      - c5d.large
      - c5d.xlarge
      - c5d.2xlarge
      - c5d.4xlarge
      - c5d.9xlarge
      - c5d.18xlarge
      - c5.large
      - c5.xlarge
      - c5.2xlarge
      - c5.4xlarge
      - c5.9xlarge
      - c5.12xlarge
      - c5.18xlarge
      - c5.24xlarge
      - c4.large
      - c4.xlarge
      - c4.2xlarge
      - c4.4xlarge
      - c4.8xlarge
    ConstraintDescription: must be a valid EC2 instance type.
  KeyName:
    Description: Name of an existing EC2 KeyPair to enable SSH access to the instances
    Type: 'AWS::EC2::KeyPair::KeyName'
    MinLength: '1'
    MaxLength: '255'
    AllowedPattern: '[\x20-\x7E]*'
    ConstraintDescription: can contain only ASCII characters.
  SSHLocation:
    Description: The IP address range that can be used to SSH to the EC2 instances
    Type: String
    MinLength: '9'
    MaxLength: '18'
    Default: 0.0.0.0/0
    AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})'
    ConstraintDescription: must be a valid IP CIDR range of the form x.x.x.x/x.
  ElasticIPAddress: 
    Description: If you want to assign an Elastic IP Address, please select True
    Default: false
    Type: String
    AllowedValues: 
      - true
      - false

Conditions:
  CreateIP: !Equals 
    - !Ref ElasticIPAddress
    - true

Resources:
  DescribeImagesRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Action: sts:AssumeRole
            Effect: Allow
            Principal:
              Service: lambda.amazonaws.com
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
      Policies:
        - PolicyName: DescribeImages
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Action: ec2:DescribeImages
                Effect: Allow
                Resource: "*"
  GetLatestAMI:
    Type: AWS::Lambda::Function
    Properties:
      Runtime: python3.6
      Handler: index.handler
      Role: !Sub ${DescribeImagesRole.Arn}
      Timeout: 60
      Code:
        ZipFile: |
          import boto3
          import cfnresponse
          import json
          import traceback

          def handler(event, context):
            try:
              response = boto3.client('ec2').describe_images(
                  Owners=[event['ResourceProperties']['Owner']],
                  Filters=[
                    {'Name': 'name', 'Values': [event['ResourceProperties']['Name']]},
                    {'Name': 'architecture', 'Values': [event['ResourceProperties']['Architecture']]},
                    {'Name': 'root-device-type', 'Values': ['ebs']},
                  ],
              )

              amis = sorted(response['Images'],
                            key=lambda x: x['CreationDate'],
                            reverse=True)
              id = amis[0]['ImageId']

              cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, id)
            except:
              traceback.print_last()
              cfnresponse.send(event, context, cfnresponse.FAIL, {}, "ok")

  UbuntuAmi:
    Type: Custom::FindAMI
    Properties:
      ServiceToken: !Sub ${GetLatestAMI.Arn}
      Owner: "099720109477"
      Name: "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"
      Architecture: "x86_64"

  MonitorSecurityGroup:
      Type: 'AWS::EC2::SecurityGroup'
      Properties:
        VpcId: !Ref VpcId
        GroupDescription: Monitor SecurityGroup
        SecurityGroupIngress:
          - IpProtocol: tcp
            FromPort: '22'
            ToPort: '22'
            CidrIp: 0.0.0.0/0
          - IpProtocol: tcp
            FromPort: '3000'
            ToPort: '3000'
            CidrIp: 0.0.0.0/0
          - IpProtocol: tcp
            FromPort: '9092'
            ToPort: '9092'
            CidrIp: 0.0.0.0/0
          - IpProtocol: tcp
            FromPort: '9093'
            ToPort: '9093'
            CidrIp: 0.0.0.0/0

  MonitorInstance:
      Type: 'AWS::EC2::Instance'
      Properties:
        KeyName: !Ref KeyName
        ImageId: !Ref UbuntuAmi
        InstanceType: !Ref InstanceType
        SubnetId: !Ref Subnets
        SecurityGroupIds:
          - !GetAtt MonitorSecurityGroup.GroupId
        Tags:
        - Key: Name
          Value: Monitor
        UserData:
          Fn::Base64: !Sub |
            #!/bin/bash
            apt-get update 
            apt-get install -y python-pip
            apt-get install -y python-setuptools
            mkdir -p /opt/aws/bin
            python /usr/lib/python2.7/dist-packages/easy_install.py --script-dir /opt/aws/bin https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz
            /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource TestInstance --region ${AWS::Region}
            sudo apt-get install apt-transport-https software-properties-common wget -y
            wget -qO- https://downloads.apache.org/kafka/2.7.0/kafka_2.13-2.7.0.tgz | tar -zxvf -  -C /opt/ && mv /opt/kafka* /opt/kafka
            wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
            echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-7.x.list
            wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -
            sudo add-apt-repository "deb https://packages.grafana.com/oss/deb stable main"
            apt-get update && apt-get install elasticsearch logstash grafana openjdk-8-jdk -y

            CPU=$(grep -c 'processor' /proc/cpuinfo)
            MEMORY=$(awk '/MemTotal/ {print int($2/1024/1024)}' /proc/meminfo)
            LOCAL_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)
            PUBLIC_IP=$(curl http://169.254.169.254/latest/meta-data/public-ipv4)

            DASHBOARD_URL="https://raw.githubusercontent.com/ant-media/Scripts/master/monitor/antmediaserver.json"
            DATASOURCE_URL="https://raw.githubusercontent.com/ant-media/Scripts/master/monitor/datasource.json"

            if [ "$MEMORY" -ge "7" ]; then
                    sudo sed -i -e 's/-Xms1g/-Xms4g/g' -e 's/-Xmx1g/-Xmx4g/g' /etc/logstash/jvm.options
            fi


            sudo sed -i "s/#.*pipeline.workers: 2/pipeline.workers: $CPU/g" /etc/logstash/logstash.yml
            sudo sed -i 's/num.partitions=1/num.partitions=4/g' /opt/kafka/config/server.properties

            sudo cat <<EOF >> /lib/systemd/system/kafka.service

            [Unit]
            Description=Apache Kafka Server
            Requires=network.target remote-fs.target
            After=network.target remote-fs.target kafka-zookeeper.service

            [Service]
            Type=simple
            Environment=JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
            ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties
            ExecStop=/opt/kafka/bin/kafka-server-stop.sh

            [Install]
            WantedBy=multi-user.target

            EOF

            sudo cat << EOF >> /lib/systemd/system/kafka-zookeeper.service

            [Unit]
            Description=Apache Zookeeper Server
            Requires=network.target remote-fs.target
            After=network.target remote-fs.target

            [Service]
            Type=simple
            Environment=JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
            ExecStart=/opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties
            ExecStop=/opt/kafka/bin/zookeeper-server-stop.sh

            [Install]
            WantedBy=multi-user.target

            EOF

            sudo cat <<EOF >> /etc/logstash/conf.d/logstash.conf
            input {
              kafka {
                bootstrap_servers => "127.0.0.1:9092"
                client_id => "logstash"
                group_id => "logstash"
                consumer_threads => 4
                topics => ["ams-instance-stats","ams-webrtc-stats","kafka-webrtc-tester-stats"]
                codec => "json"
                tags => ["log", "kafka_source"]
                type => "log"
              }
            }

            output {
              elasticsearch {
                 hosts => ["127.0.0.1:9200"] 
                 index => "logstash-%{[type]}-%{+YYYY.MM.dd}"
              }
            }
            EOF

            sudo cat <<EOF >> /opt/kafka/config/server.properties
            advertised.listeners=INTERNAL_PLAINTEXT://$LOCAL_IP:9092,EXTERNAL_PLAINTEXT://$PUBLIC_IP:9093
            listeners=INTERNAL_PLAINTEXT://0.0.0.0:9092,EXTERNAL_PLAINTEXT://0.0.0.0:9093
            inter.broker.listener.name=INTERNAL_PLAINTEXT
            listener.security.protocol.map=INTERNAL_PLAINTEXT:PLAINTEXT,EXTERNAL_PLAINTEXT:PLAINTEXT
            EOF

            sudo systemctl enable grafana-server && sudo systemctl restart grafana-server

            wget -q $DASHBOARD_URL -O /tmp/antmediaserver.json
            wget -q $DATASOURCE_URL -O /tmp/antmedia-datasource.json

            sleep 5

            sudo curl "http://127.0.0.1:3000/api/dashboards/db" \
                -u "admin:admin" \
                -H "Content-Type: application/json" \
                --data-binary "@/tmp/antmediaserver.json" > /tmp/curl.log


            sudo curl -X "POST" "http://127.0.0.1:3000/api/datasources" \
                -H "Content-Type: application/json" \
                -u "admin:admin" \
                --data-binary "@/tmp/antmedia-datasource.json" >> /tmp/curl.log 




            sudo systemctl daemon-reload
            sudo systemctl enable logstash.service && sudo systemctl enable elasticsearch && sudo systemctl enable kafka && sudo systemctl enable kafka-zookeeper
            sudo systemctl restart kafka-zookeeper && sudo systemctl restart kafka && sudo systemctl restart logstash && sudo systemctl restart elasticsearch

  ElasticIP:
    Type: AWS::EC2::EIP
    Condition: CreateIP
    Properties:
      Domain: "vpc"
  ElasticIPAssignment:
    Type: AWS::EC2::EIPAssociation
    Condition: CreateIP
    Properties:
      EIP: !Ref ElasticIP
      InstanceId: !Ref MonitorInstance
Outputs:
  InstanceElasticIP:
    Description: Elastic IP Address
    Condition: CreateIP
    Value: !Ref ElasticIP 

  PublicIPV4:
    Description: Public IP Address of Monitoring Tool
    Value: !GetAtt MonitorInstance.PublicIp


  MonitorURL:
    Description: "Grafana Default Username and Password: admin/admin"
    Value: !Join 
      - ''
      - - 'http://'
        - !GetAtt 
          - MonitorInstance
          - PublicIp
        - ':3000'