An Introduction to Chaos Engineering: Trying to Break Stuff

Let me explain

What is Chaos Engineering?

$ pip3 install -U chaostoolkit chaostoolkit-aws
 $ chaos --version
 chaos, version 1.3.0

Now, our resilient application

• The Echo server will be a service running on an AWS EC2 instance with an Elastic IP attached to it.
• If the EC2 instance fails, another one will take its place and will reclaim the Elastic IP.
• KeepaliveD will be used to make this happen.

import java.io.*;
 import java.net.*;
 import java.lang.Thread;
 
 public class EchoServer {
  public static void main (String[] args) {
 
  try {
  ServerSocket server = new ServerSocket(5566);
 
  System.out.println("Listing on port 5566...");
  while (true) {
  Socket client = server.accept();
  EchoHandler handler = new EchoHandler(client);
  handler.start();
  }
  }
  catch (Exception e) {
  System.err.println("Exception caught:" + e);
  }
  }
 }
 
 class EchoHandler extends Thread {
  Socket client;
  InetAddress inetAddress;
  String configServer;
 
  EchoHandler (Socket client) {
  this.client = client;
 
  try {
  inetAddress = InetAddress.getLocalHost();
  } catch(Exception e) {}
  }
 
  public void run () {
  try {
  BufferedReader reader = new BufferedReader(new InputStreamReader(client.getInputStream()));
  PrintWriter writer = new PrintWriter(client.getOutputStream(), true);
 
  System.out.println("[connected] " + client.getInetAddress());
 
  while (true) {
  String line = reader.readLine();
  System.out.println(client.getInetAddress() + " says " + line);
 
  writer.println(line);
  if (line.trim().equals("bye")) {
  break;
  }
  }
  }
  catch (Exception e) {
  System.err.println("Exception caught: client disconnected.");
  }
  finally {
  try { 
  System.out.println("[disconnected] " + client.getInetAddress());
  client.close(); 
  }
  catch (Exception e ){ ; }
  }
  }
 }
 

Our experiment

{
  "version": "1.0.0",
  "title": "Validating High-Availability of the Echo server.",
  "description": "Ensure that it will be always an EC2 instance with the ElasticIP attached.",
  "tags": [
  "echoserver",
  "keepalived",
  "elasticip",
  "ha",
  "master",
  "backup"
  ],
  "configuration": {
  "aws_region": "us-east-1"
  },
  "steady-state-hypothesis": {
  "title": "The backup node will take the role of master when the original one is terminated.",
  "probes": [
  {
  "type": "probe",
  "name": "port-5566-is-listening-and-working-properly",
  "tolerance": true,
  "provider": {
  "type": "python",
  "module": "chaospythian.echo.probes",
  "func": "echoserver",
  "arguments": {
  "tcp_ip": "54.71.185.10",
  "tcp_port": 5566,
  "message": "testing text\n"
  }
  }
  },
  {
  "type": "probe",
  "name": "master-and-backup-instances-up-and-running",
  "tolerance": [1,2],
  "provider": {
  "type": "python",
  "module": "chaosaws.ec2.probes",
  "func": "count_instances",
  "arguments": {
  "filters": [
  {
  "Name": "instance-state-name",
  "Values": ["running"]
  },
  { 
  "Name": "tag:Name",
  "Values": ["production-echoserver"]
  }
  ]
 
  }
  }
  },
  {
  "type": "probe",
  "name": "one-ec2-instance-tagged-as-master",
  "tolerance": 1,
  "provider": {
  "type": "python",
  "module": "chaosaws.ec2.probes",
  "func": "count_instances",
  "arguments": {
  "filters": [
  {
  "Name": "instance-state-name",
  "Values": ["running"]
  },
  {
  "Name": "tag:State",
  "Values": ["MASTER"]
  },
  {
  "Name": "tag:Name",
  "Values": ["production-echoserver"]
  }
  ]
  }
  }
  },
  {
  "type": "probe",
  "name": "one-ec2-instance-tagged-as-backup",
  "tolerance": [0,1],
  "provider": {
  "type": "python",
  "module": "chaosaws.ec2.probes",
  "func": "count_instances",
  "arguments": {
  "filters": [
  {
  "Name": "instance-state-name",
  "Values": ["running"]
  },
  {
  "Name": "tag:State",
  "Values": ["BACKUP"]
  },
  {
  "Name": "tag:Name",
  "Values": ["production-echoserver"]
  }
  ]
  }
  }
 
  }
  ]
  },
  "method": [
  {
  "type": "action",
  "name": "terminate-master-node",
  "provider": {
  "type": "python",
  "module": "chaosaws.ec2.actions",
  "func": "terminate_instance",
  "arguments": {
  "filters": [
  {
  "Name": "instance-state-name",
  "Values": ["running"]
  },
  {
  "Name": "tag:State",
  "Values": ["MASTER"]
  },
  { 
  "Name": "tag:Name",
  "Values": ["production-echoserver"]
  }
  ]
 
  }
  },
  "pauses": {
  "after": 10
  }
  }
  ]
 }
 

Client-side:

$ nc 54.71.185.10 5566
 hello
 hello
 test
 test
 one two three
 one two three
 bye
 bye

Server-side:

Listing on port 5566...
 [connected] /190.104.119.195
 /190.104.119.195 says hello
 /190.104.119.195 says test
 /190.104.119.195 says one two three
 /190.104.119.195 says bye
 [disconnected] /190.104.119.195

Now the fun part: let's terminate the master node!

# -*- coding: utf-8 -*-
 import socket
 
 __all__ = ["echoserver"]
 
 
 def echoserver(tcp_ip, tcp_port, message):
  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  s.connect((tcp_ip, tcp_port))
  s.send(message.encode('ascii'))
  data = s.recv(1024)
  s.send("bye".encode('ascii'))
  s.close()
 
  return (data.decode('ascii').strip() == message.strip())