AWSTemplateFormatVersion: '2010-09-09' Parameters: KeyName: Description: Name of an existing EC2 KeyPair to enable SSH access to the instance Type: AWS::EC2::KeyPair::KeyName ConstraintDescription: Can contain only ASCII characters. LatestAmiId: Type: 'AWS::SSM::Parameter::Value' Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' MSKStack: Description: The name of the MSK stack Type: String VPCStack: Description: The name of the VPC stack Type: String Resources: Cloud9EC2Bastion: Type: AWS::Cloud9::EnvironmentEC2 Properties: ImageId: 'amazonlinux-2023-x86_64' AutomaticStopTimeMinutes: 600 Description: "Cloud9 EC2 environment" InstanceType: m5.large Name: !Sub "${AWS::StackName}-Cloud9EC2Bastion" SubnetId: Fn::ImportValue: !Sub "${VPCStack}-PublicSubnetOne" Tags: - Key: 'Purpose' Value: 'Cloud9EC2BastionHostInstance' KafkaClientEC2Instance1: Type: AWS::EC2::Instance CreationPolicy: ResourceSignal: Timeout: PT30M Properties: InstanceType: m5.large KeyName: !Ref 'KeyName' IamInstanceProfile: !Ref EC2InstanceProfile AvailabilityZone: Fn::Select: - 0 - Fn::GetAZs: {Ref: 'AWS::Region'} SubnetId: Fn::ImportValue: !Sub "${VPCStack}-PrivateSubnetMSKOne" SecurityGroupIds: - Fn::ImportValue: !Sub "${MSKStack}-KafkaClientEC2InstanceSecurityGroupId" ImageId: !Ref LatestAmiId Tags: - Key: 'Name' Value: !Sub ${AWS::StackName}-KafkaClientInstance1 UserData: Fn::Base64: !Sub | #!/bin/bash yum update -y yum install python3.7 -y yum install java-1.8.0-openjdk-devel -y yum install nmap-ncat -y yum install git -y yum erase awscli -y yum install jq -y yum install maven -y amazon-linux-extras install docker -y service docker start usermod -a -G docker ec2-user cd /home/ec2-user wget https://bootstrap.pypa.io/get-pip.py su -c "python3.7 get-pip.py --user" -s /bin/sh ec2-user su -c "/home/ec2-user/.local/bin/pip3 install boto3 --user" -s /bin/sh ec2-user su -c "/home/ec2-user/.local/bin/pip3 install kafka-python --user" -s /bin/sh ec2-user # install AWS CLI 2 - access with aws2 curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip ./aws/install -b /usr/local/bin/aws2 ln -s /usr/local/bin/aws2/aws /usr/local/bin/aws # Create dirs, get Apache Kafka 2.7.0 su -c "mkdir -p kafka270" -s /bin/sh ec2-user ln -s /home/ec2-user/kafka270 /home/ec2-user/kafka cd kafka su -c "wget https://archive.apache.org/dist/kafka/2.7.0/kafka_2.13-2.7.0.tgz" -s /bin/sh ec2-user su -c "tar -xzf kafka_2.13-2.7.0.tgz --strip 1" -s /bin/sh ec2-user # Initialize the Kafka cert trust store su -c 'find /usr/lib/jvm/ -name "cacerts" -exec cp {} /tmp/kafka.client.truststore.jks \;' -s /bin/sh ec2-user # Setup prometheus agent cd /home/ec2-user su -c "mkdir prometheus" -s /bin/sh ec2-user cd prometheus su -c "wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.13.0/jmx_prometheus_javaagent-0.13.0.jar" -s /bin/sh ec2-user # Copy files from S3 cd /tmp su -c "mkdir -p kafka" -s /bin/sh ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/producer.properties_msk /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/consumer.properties /tmp/kafka" -l ec2-user su -c "echo -n ' git -C /tmp/kafka clone https://github.com/aws-samples/sasl-scram-secrets-manager-client-for-msk.git cd /tmp/kafka/sasl-scram-secrets-manager-client-for-msk/ && mvn clean install -f pom.xml && cp target/SaslScramSecretsManagerClient-1.0-SNAPSHOT.jar /tmp/kafka cd /tmp/kafka && rm -rf sasl-scram-secrets-manager-client-for-msk git -C /tmp/kafka clone https://github.com/aws-samples/clickstream-producer-for-apache-kafka.git cd /tmp/kafka/clickstream-producer-for-apache-kafka/ && mvn clean package -f pom.xml && cp target/KafkaClickstreamClient-1.0-SNAPSHOT.jar /tmp/kafka cd /tmp/kafka && rm -rf clickstream-producer-for-apache-kafka ' > /tmp/kafka/java-producer-installs.sh" -l ec2-user su -c "chmod +x /tmp/kafka/java-producer-installs.sh" -l ec2-user su -c "/tmp/kafka/java-producer-installs.sh" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/sasl-scram-secrets-manager-client-for-msk.git" -l ec2-user # su -c "cd /tmp/kafka/sasl-scram-secrets-manager-client-for-msk/ && mvn clean install -f pom.xml && cp target/SaslScramSecretsManagerClient-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user # su -c "cd /tmp/kafka && rm -rf sasl-scram-secrets-manager-client-for-msk" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/clickstream-producer-for-apache-kafka.git" -l ec2-user # su -c "cd /tmp/kafka/clickstream-producer-for-apache-kafka/ && mvn clean package -f pom.xml && cp target/KafkaClickstreamClient-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user # su -c "cd /tmp/kafka && rm -rf clickstream-producer-for-apache-kafka" -l ec2-user su -c "echo -n ' git -C /tmp/kafka clone https://github.com/aws-samples/mirrormaker2-msk-migration.git cd /tmp/kafka/mirrormaker2-msk-migration/ && mvn clean install -f pom.xml && cp CustomMM2ReplicationPolicy/target/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /tmp/kafka && cp CustomMM2ReplicationPolicy/target/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /home/ec2-user/kafka/libs cd /tmp/kafka && rm -rf mirrormaker2-msk-migration git -C /tmp/kafka clone https://github.com/aws-samples/clickstream-consumer-for-apache-kafka.git cd /tmp/kafka/clickstream-consumer-for-apache-kafka/ && mvn clean package -f pom.xml && cp target/KafkaClickstreamConsumer-1.0-SNAPSHOT.jar /tmp/kafka cd /tmp/kafka && rm -rf clickstream-consumer-for-apache-kafka ' > /tmp/kafka/java-consumer-installs.sh" -l ec2-user su -c "chmod +x /tmp/kafka/java-consumer-installs.sh" -l ec2-user su -c "/tmp/kafka/java-consumer-installs.sh" -l ec2-user su -c "echo -n ' git -C /tmp/kafka clone https://github.com/aws-samples/amazon-msk-client-authentication.git cd /tmp/kafka/amazon-msk-client-authentication/ && mvn clean package -f pom.xml && cp target/AuthMSK-1.0-SNAPSHOT.jar /tmp/kafka cd /tmp/kafka && rm -rf amazon-msk-client-authentication ' > /tmp/kafka/java-client-authentication-installs.sh" -l ec2-user su -c "chmod +x /tmp/kafka/java-client-authentication-installs.sh" -l ec2-user su -c "/tmp/kafka/java-client-authentication-installs.sh" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/mirrormaker2-msk-migration.git" -l ec2-user # su -c "cd /tmp/kafka/mirrormaker2-msk-migration/ && mvn clean install -f pom.xml && cp CustomMM2ReplicationPolicy/target/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /tmp/kafka && cp CustomMM2ReplicationPolicy/target/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /home/ec2-user/kafka/libs" -l ec2-user # su -c "cd /tmp/kafka && rm -rf mirrormaker2-msk-migration" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/clickstream-consumer-for-apache-kafka.git" -l ec2-user # su -c "cd /tmp/kafka/clickstream-consumer-for-apache-kafka/ && mvn clean package -f pom.xml && cp target/KafkaClickstreamConsumer-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user # su -c "cd /tmp/kafka && rm -rf clickstream-consumer-for-apache-kafka" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/amazon-msk-client-authentication.git" -l ec2-user # su -c "cd /tmp/kafka/amazon-msk-client-authentication/ && mvn clean package -f pom.xml && cp target/AuthMSK-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user # su -c "cd /tmp/kafka && rm -rf amazon-msk-client-authentication" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/generatePropertiesFiles.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/generateStartupFile.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/connect-distributed.properties /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/connect-distributed-no-auth.properties /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/kafka-consumer-python.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/setup-env.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/GlobalSeqNo.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-msc.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-msc-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-hbc.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-hbc-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-cpc.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-cpc-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-cpc-cust-repl-policy.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-cpc-cust-repl-policy-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-cpc-cust-repl-policy-no-auth-sync.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-msc-cust-repl-policy.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-msc-cust-repl-policy-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/kafka-connect.yml /home/ec2-user/prometheus" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/kafka-producer-consumer.yml /home/ec2-user/prometheus" -l ec2-user # Setup unit in systemd for Kafka Connect echo -n " [Unit] Description=Kafka Connect After=network.target [Service] Type=simple User=ec2-user Environment='KAFKA_OPTS=-javaagent:/home/ec2-user/prometheus/jmx_prometheus_javaagent-0.13.0.jar=3600:/home/ec2-user/prometheus/kafka-connect.yml' ExecStart=/bin/sh -c '/home/ec2-user/kafka/bin/connect-distributed.sh /tmp/kafka/connect-distributed.properties > /tmp/kafka/kafka-connect.log 2>&1' Restart=on-abnormal [Install] WantedBy=multi-user.target" > /etc/systemd/system/kafka-connect.service #setup bash env su -c "echo 'export PS1=\"KafkaClientEC2Instance1 [\u@\h \W\\]$ \"' >> /home/ec2-user/.bash_profile" -s /bin/sh ec2-user su -c "echo '[ -f /tmp/kafka/setup_env ] && . /tmp/kafka/setup_env' >> /home/ec2-user/.bash_profile" -s /bin/sh ec2-user /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource KafkaClientEC2Instance1 --region ${AWS::Region} KafkaClientEC2Instance2: Type: AWS::EC2::Instance CreationPolicy: ResourceSignal: Timeout: PT30M Properties: InstanceType: m5.large KeyName: !Ref 'KeyName' IamInstanceProfile: !Ref EC2InstanceProfile AvailabilityZone: Fn::Select: - 1 - Fn::GetAZs: {Ref: 'AWS::Region'} SubnetId: Fn::ImportValue: !Sub "${VPCStack}-PrivateSubnetMSKTwo" SecurityGroupIds: - Fn::ImportValue: !Sub "${MSKStack}-KafkaClientEC2InstanceSecurityGroupId" ImageId: !Ref LatestAmiId Tags: - Key: 'Name' Value: !Sub ${AWS::StackName}-KafkaClientInstance2 UserData: Fn::Base64: !Sub | #!/bin/bash yum update -y yum install python3.7 -y yum install java-1.8.0-openjdk-devel -y yum install nmap-ncat -y yum install git -y yum erase awscli -y yum install jq -y yum install maven -y amazon-linux-extras install docker -y service docker start usermod -a -G docker ec2-user cd /home/ec2-user wget https://bootstrap.pypa.io/get-pip.py su -c "python3.7 get-pip.py --user" -s /bin/sh ec2-user su -c "/home/ec2-user/.local/bin/pip3 install boto3 --user" -s /bin/sh ec2-user su -c "/home/ec2-user/.local/bin/pip3 install kafka-python --user" -s /bin/sh ec2-user # install AWS CLI 2 - access with aws2 curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" unzip awscliv2.zip ./aws/install -b /usr/local/bin/aws2 ln -s /usr/local/bin/aws2/aws /usr/local/bin/aws # Create dirs, get Apache Kafka 2.7.0 su -c "mkdir -p kafka270" -s /bin/sh ec2-user ln -s /home/ec2-user/kafka270 /home/ec2-user/kafka cd kafka su -c "wget https://ftp.wayne.edu/apache/kafka/2.7.0/kafka_2.13-2.7.0.tgz" -s /bin/sh ec2-user su -c "tar -xzf kafka_2.13-2.7.0.tgz --strip 1" -s /bin/sh ec2-user # Initialize the Kafka cert trust store su -c 'find /usr/lib/jvm/ -name "cacerts" -exec cp {} /tmp/kafka.client.truststore.jks \;' -s /bin/sh ec2-user # Setup prometheus agent cd /home/ec2-user su -c "mkdir prometheus" -s /bin/sh ec2-user cd prometheus su -c "wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.13.0/jmx_prometheus_javaagent-0.13.0.jar" -s /bin/sh ec2-user # Copy files from S3 cd /tmp su -c "mkdir -p kafka" -s /bin/sh ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/producer.properties_msk /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/consumer.properties /tmp/kafka" -l ec2-user su -c "echo -n ' git -C /tmp/kafka clone https://github.com/aws-samples/sasl-scram-secrets-manager-client-for-msk.git cd /tmp/kafka/sasl-scram-secrets-manager-client-for-msk/ && mvn clean install -f pom.xml && cp target/SaslScramSecretsManagerClient-1.0-SNAPSHOT.jar /tmp/kafka cd /tmp/kafka && rm -rf sasl-scram-secrets-manager-client-for-msk git -C /tmp/kafka clone https://github.com/aws-samples/clickstream-producer-for-apache-kafka.git cd /tmp/kafka/clickstream-producer-for-apache-kafka/ && mvn clean package -f pom.xml && cp target/KafkaClickstreamClient-1.0-SNAPSHOT.jar /tmp/kafka cd /tmp/kafka && rm -rf clickstream-producer-for-apache-kafka ' > /tmp/kafka/java-producer-installs.sh" -l ec2-user su -c "chmod +x /tmp/kafka/java-producer-installs.sh" -l ec2-user su -c "/tmp/kafka/java-producer-installs.sh" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/sasl-scram-secrets-manager-client-for-msk.git" -l ec2-user # su -c "cd /tmp/kafka/sasl-scram-secrets-manager-client-for-msk/ && mvn clean install -f pom.xml && cp target/SaslScramSecretsManagerClient-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user # su -c "cd /tmp/kafka && rm -rf sasl-scram-secrets-manager-client-for-msk" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/clickstream-producer-for-apache-kafka.git" -l ec2-user # su -c "cd /tmp/kafka/clickstream-producer-for-apache-kafka/ && mvn clean package -f pom.xml && cp target/KafkaClickstreamClient-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user # su -c "cd /tmp/kafka && rm -rf clickstream-producer-for-apache-kafka" -l ec2-user su -c "echo -n ' git -C /tmp/kafka clone https://github.com/aws-samples/mirrormaker2-msk-migration.git cd /tmp/kafka/mirrormaker2-msk-migration/ && mvn clean install -f pom.xml && cp CustomMM2ReplicationPolicy/target/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /tmp/kafka && cp CustomMM2ReplicationPolicy/target/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /home/ec2-user/kafka/libs cd /tmp/kafka && rm -rf mirrormaker2-msk-migration git -C /tmp/kafka clone https://github.com/aws-samples/clickstream-consumer-for-apache-kafka.git cd /tmp/kafka/clickstream-consumer-for-apache-kafka/ && mvn clean package -f pom.xml && cp target/KafkaClickstreamConsumer-1.0-SNAPSHOT.jar /tmp/kafka cd /tmp/kafka && rm -rf clickstream-consumer-for-apache-kafka ' > /tmp/kafka/java-consumer-installs.sh" -l ec2-user su -c "chmod +x /tmp/kafka/java-consumer-installs.sh" -l ec2-user su -c "/tmp/kafka/java-consumer-installs.sh" -l ec2-user su -c "echo -n ' git -C /tmp/kafka clone https://github.com/aws-samples/amazon-msk-client-authentication.git cd /tmp/kafka/amazon-msk-client-authentication/ && mvn clean package -f pom.xml && cp target/AuthMSK-1.0-SNAPSHOT.jar /tmp/kafka cd /tmp/kafka && rm -rf amazon-msk-client-authentication ' > /tmp/kafka/java-client-authentication-installs.sh" -l ec2-user su -c "chmod +x /tmp/kafka/java-client-authentication-installs.sh" -l ec2-user su -c "/tmp/kafka/java-client-authentication-installs.sh" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/mirrormaker2-msk-migration.git" -l ec2-user # su -c "cd /tmp/kafka/mirrormaker2-msk-migration/ && mvn clean install -f pom.xml && cp CustomMM2ReplicationPolicy/target/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /tmp/kafka && cp CustomMM2ReplicationPolicy/target/CustomMM2ReplicationPolicy-1.0-SNAPSHOT.jar /home/ec2-user/kafka/libs" -l ec2-user # su -c "cd /tmp/kafka && rm -rf mirrormaker2-msk-migration" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/clickstream-consumer-for-apache-kafka.git" -l ec2-user # su -c "cd /tmp/kafka/clickstream-consumer-for-apache-kafka/ && mvn clean package -f pom.xml && cp target/KafkaClickstreamConsumer-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user # su -c "cd /tmp/kafka && rm -rf clickstream-consumer-for-apache-kafka" -l ec2-user # su -c "git -C /tmp/kafka clone https://github.com/aws-samples/amazon-msk-client-authentication.git" -l ec2-user # su -c "cd /tmp/kafka/amazon-msk-client-authentication/ && mvn clean package -f pom.xml && cp target/AuthMSK-1.0-SNAPSHOT.jar /tmp/kafka" -l ec2-user # su -c "cd /tmp/kafka && rm -rf amazon-msk-client-authentication" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/generatePropertiesFiles.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/generateStartupFile.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/connect-distributed.properties /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/connect-distributed-no-auth.properties /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/kafka-consumer-python.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/setup-env.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/GlobalSeqNo.py /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-msc.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-msc-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-hbc.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-hbc-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-cpc.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-cpc-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-cpc-cust-repl-policy.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-cpc-cust-repl-policy-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-msc-cust-repl-policy.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/mm2-msc-cust-repl-policy-no-auth.json /tmp/kafka" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/kafka-connect.yml /home/ec2-user/prometheus" -l ec2-user su -c "aws s3 cp s3://aws-streaming-artifacts/msk-lab-resources/kafka-producer-consumer.yml /home/ec2-user/prometheus" -l ec2-user # Setup unit in systemd for Kafka Connect echo -n " [Unit] Description=Kafka Connect After=network.target [Service] Type=simple User=ec2-user Environment='KAFKA_OPTS=-javaagent:/home/ec2-user/prometheus/jmx_prometheus_javaagent-0.13.0.jar=3600:/home/ec2-user/prometheus/kafka-connect.yml' ExecStart=/bin/sh -c '/home/ec2-user/kafka/bin/connect-distributed.sh /tmp/kafka/connect-distributed.properties > /tmp/kafka/kafka-connect.log 2>&1' Restart=on-abnormal [Install] WantedBy=multi-user.target" > /etc/systemd/system/kafka-connect.service #setup bash env su -c "echo 'export PS1=\"KafkaClientEC2Instance1 [\u@\h \W\\]$ \"' >> /home/ec2-user/.bash_profile" -s /bin/sh ec2-user su -c "echo '[ -f /tmp/kafka/setup_env ] && . /tmp/kafka/setup_env' >> /home/ec2-user/.bash_profile" -s /bin/sh ec2-user /opt/aws/bin/cfn-signal -e $? --stack ${AWS::StackName} --resource KafkaClientEC2Instance2 --region ${AWS::Region} EC2InstanceProfile: Type: AWS::IAM::InstanceProfile Properties: InstanceProfileName: !Join - '-' - - 'EC2MMMSKCFProfile' - !Ref 'AWS::StackName' Roles: - !Ref EC2Role EC2Role: Type: AWS::IAM::Role Properties: AssumeRolePolicyDocument: Version: 2012-10-17 Statement: - Sid: '' Effect: Allow Principal: Service: ec2.amazonaws.com Action: 'sts:AssumeRole' Path: "/" KafkaClientEC2InstancePolicy: Type: 'AWS::IAM::Policy' Properties: PolicyName: KafkaClientEC2InstancePolicy PolicyDocument: Version: 2012-10-17 Statement: - Effect: Allow Action: - s3:CreateBucket - cloudformation:DescribeStackResource - s3:ListBucket - kafka:GetBootstrapBrokers - cloudformation:DescribeStackEvents - kafka:DescribeCluster - cloudformation:GetTemplateSummary - cloudformation:ListStackResources - cloudformation:DescribeStacks - s3:GetObject - glue:GetSchemaByDefinition - glue:CreateSchema - glue:RegisterSchemaVersion - glue:PutSchemaVersionMetadata - glue:GetSchemaVersion - acm-pca:IssueCertificate - acm-pca:GetCertificate Resource: '*' Roles: - !Ref EC2Role Outputs: KafkaClientEC2Instance1PrivateDNS: Description: The Public DNS for the EC2 instance1 Value: !GetAtt KafkaClientEC2Instance1.PrivateDnsName KafkaClientEC2Instance2PrivateDNS: Description: The Public DNS for the EC2 instance2 Value: !GetAtt KafkaClientEC2Instance2.PrivateDnsName SSHKafkaClientEC2Instance1: Description: SSH command for Kafka the EC2 instance1 Value: !Sub ssh -A ec2-user@${KafkaClientEC2Instance1.PrivateDnsName} SSHKafkaClientEC2Instance2: Description: SSH command for Kafka the EC2 instance2 Value: !Sub ssh -A ec2-user@${KafkaClientEC2Instance2.PrivateDnsName}