Simplifying Amazon CloudFormation with Troposphere

After working with CloudFormation templates for a while, you start to notice several shortcomings that make templates long, clunky, and nigh unreadable. Enter Troposphere.

The Troposphere python library allows for easier creation of the Aamzon CloudFormation JSON by writing Python code to describe the AWS resources. This effectively allows you to programmatically define your infrastructure without becoming as limited as we are with plain CloudFormation.

Getting started is as easy as pip install troposphere.

The Troposphere team has some great examples in their GitHub repository. I suggest taking a look there and working from their examples.

We’re going to step through the process of taking our previous VPC template and converting it into python template. This template creates a single VPC with a single subnet.

cf-snippet-vpc.json

{
  "AWSTemplateFormatVersion" : "2010-09-09",
  "Description" : "Provisions a very small VPC with 1 public subnets.",
  "Parameters" : {
        "VPCCIDR" : {
            "Description" : "The IP Address space used by this VPC.",
            "Type" : "String",
            "Default" : "10.0.0.0/16",
            "AllowedPattern" : "(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})/(\\\\d{1,2})"
        },
        "VPCName" : {
            "Description" : "The name of this VPC. Tags the VPC for identification.",
            "Type" : "String",
            "Default" : "Basic VPC"
        },
        "PublicSubnet1CIDR" : {
            "Description" : "The IP Address space used by this subnet.",
            "Type" : "String",
            "Default" : "10.0.1.0/24",
            "AllowedPattern" : "(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})/(\\\\d{1,2})"
        },
        "PublicSubnet1AZ" : {
            "Description" : "The availaibility zone to place this subnet in.",
            "Type" : "String",
            "Default" : "us-west-2a",
            "AllowedValues" : \["us-west-2a","us-west-2b","us-west-2c"\]
        }
    },
    "Resources" : {
        "VPC" : {
            "Type" : "AWS::EC2::VPC",
            "Properties" : {
                "CidrBlock" : {
                    "Ref" : "VPCCIDR"
                },
                "EnableDnsSupport" : true,
                "EnableDnsHostnames" : true,
                "Tags" : \[{"Key" : "Name", "Value" : {
                    "Ref" : "VPCName"
                }}\]
            }
        },
        "InternetGateway" : {
            "Type" : "AWS::EC2::InternetGateway",
            "Properties" : { 
                "Tags" : \[{"Key" : "Name", "Value" : {
                    "Ref" : "VPCName"
                }}\]
            }
        },
        "AttachGateway" : {
            "Type" : "AWS::EC2::VPCGatewayAttachment",
            "Properties" : {
                "VpcId" : { "Ref" : "VPC" },
                "InternetGatewayId" : { "Ref" : "InternetGateway" }
            }
        },
        "PublicSubnet1" : {
            "Type" : "AWS::EC2::Subnet",
            "Properties" : {
                "VpcId" : { "Ref" : "VPC" },
                "CidrBlock" : { "Ref" : "PublicSubnet1CIDR" },
                "AvailabilityZone" : { "Ref" : "PublicSubnet1AZ" },
                "Tags" : \[{"Key" : "Name", "Value" :
                    {"Fn::Join" : \["", \["", {"Ref" : "VPCName"}, "\_", { "Ref" : "PublicSubnet1AZ" }\]\]}
                }\]
            }
        },
        "PublicRouteTable" : {
            "Type" : "AWS::EC2::RouteTable",
            "Properties" : {
                "VpcId" : {"Ref" : "VPC"}
            }
        },
        "PublicRoute" : {
            "Type" : "AWS::EC2::Route",
            "DependsOn" : "AttachGateway",
            "Properties" : {
                "RouteTableId" : { "Ref" : "PublicRouteTable" },
                "DestinationCidrBlock" : "0.0.0.0/0",
                "GatewayId" : { "Ref" : "InternetGateway" }
            }
        },
        "PublicSubnetAcl" : {
            "Type" : "AWS::EC2::NetworkAcl",
            "Properties" : {
                "VpcId" : {"Ref" : "VPC"}
            }
        },
        "PublicInSubnetAclEntry" : {
            "Type" : "AWS::EC2::NetworkAclEntry",
            "Properties" : {
                "NetworkAclId" : {"Ref" : "PublicSubnetAcl"},
                "RuleNumber" : "32000",
                "Protocol" : "-1",
                "RuleAction" : "allow",
                "Egress" : "false",
                "CidrBlock" : "0.0.0.0/0",
                "Icmp" : { "Code" : "-1", "Type" : "-1"},
                "PortRange" : {"From" : "1", "To" : "65535"}
            }
        },
        "PublicOutSubnetAclEntry" : {
            "Type" : "AWS::EC2::NetworkAclEntry",
            "Properties" : {
                "NetworkAclId" : {"Ref" : "PublicSubnetAcl"},
                "RuleNumber" : "32000",
                "Protocol" : "-1",
                "RuleAction" : "allow",
                "Egress" : "true",
                "CidrBlock" : "0.0.0.0/0",
                "Icmp" : { "Code" : "-1", "Type" : "-1"},
                "PortRange" : {"From" : "1", "To" : "65535"}
            }
        },
        "PublicSubnetAclAssociation1" : {
            "Type" : "AWS::EC2::SubnetNetworkAclAssociation",
            "Properties" : {
                "SubnetId" : { "Ref" : "PublicSubnet1" },
                "NetworkAclId" : {"Ref" : "PublicSubnetAcl"}
            }
        },
        "PublicSubnetRTAssociation1" : {
            "Type" : "AWS::EC2::SubnetRouteTableAssociation",
            "Properties" : {
                "SubnetId" : { "Ref" : "PublicSubnet1" },
                "RouteTableId" : {"Ref" : "PublicRouteTable"}
            }
        }
    },
    "Outputs" : {
        "VPC" : {
            "Value" : {"Ref" : "VPC"},
            "Description" : "Information about the value"
        },
        "PublicRouteTable" : {
            "Value" : {"Ref" : "PublicRouteTable"},
            "Description" : "Information about the value"
        },
        "Subnet1ID" : {
            "Value" : {"Ref" : "PublicSubnet1"},
            "Description" : "Information about the value"
        }
        "Subnet1AZ" : {
            "Value" : { "Ref" : "PublicSubnet1AZ" },
            "Description" : "Information about the value"
        }
    }
}

To start with our troposphere template needs:

cf-snippet-vpc.py

#!/usr/bin/python
# -\*- coding: utf-8 -\*-

# Import libraries
from troposphere import Template

# Create the template
t = Template()

# Add the version of the template
t.add\_version('2010-09-09')

# Print the template to JSON
print(t.to\_json())

The above snippet results in:

{
    "AWSTemplateFormatVersion": "2010-09-09",
    "Resources": {}
}

This template will technically fail since we haven’t defined any resources - at least one resource definition is required for a valid template.

We can use a few different methods to add CloudFormation data objects to the template:

  • t.add_parameter(p):
    This method adds a CloudFormation parameter object to the JSON template. Here’s an example converted from our JSON formated VPC template:
    Parameter(
        "VPCCIDR",
        Description    = "The IP Address space used by this VPC. Changing the CIDR issues a replacement of the stack.",
        Type           = "String",
        Default        = "10.0.0.0/16",
        AllowedPattern = "(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})/(\\\\d{1,2})"
    )
    

    You’ll notice that it’s very similar to our JSON formatted parameters. The important thing to note is that all the fields are tacked on as key = "value".

  • t.add_condition(k, conditions[k]):
    This method adds a CloudFormation conditional to the JSON template. We don’t have one in our template, but here’s a quick example:
    {
        "IsUsWest2": Equals(Ref('AWS::Region'),"us-west-2")
    }
    
  • t.add_mapping(m):
    This method adds a CloudFormation map object to the JSON template. We don’t have one in our template, but here’s a quick example:
    ValidAZs = {}
    ValidAZs\["prod"\] = {"UsEast1" : \["us-east-1a", "us-east-1b", "us-east-1c"\], "UsWest1" : \[\], "UsWest2" : \["us-west-2a", "us-west-2b", "us-west-2c"\]}
    ValidAZs\["stag"\] = {"UsEast1" : \["us-east-1c", "us-east-1d", "us-east-1e"\], "UsWest1" : \[\], "UsWest2" : \["us-west-2a", "us-west-2b", "us-west-2c"\]}
    maps.append(\['ValidAZs', ValidAZs\])
    
  • t.add_resource(r):
    This method adds a CloudFormation resource object to the JSON template. Here’s an example converted from our JSON formated VPC template:
    resources = \[
        VPC(
            'VPC',
            CidrBlock            = Ref('VPCCIDR'),
            Tags                 = Tags(Name=Ref('VPCName')),
            EnableDnsSupport     = 'true',
            EnableDnsHostnames   = 'true'
        )
    \]
    
  • t.add_output(o):
    This method adds a CloudFormation output object to the JSON template. Here’s an example converted from our JSON formated VPC template:
    outputs = \[
        Output(
            "VPC",
            Value       = Ref('VPC'),
            Description = ""
        )
    \]
    

My favorite way to work with Troposphere is to hook resources and other elements in at the end. So, similar to:

#!/usr/bin/python
# -\*- coding: utf-8 -\*-

# Import libraries
from troposphere import \*

# An array to contain any parameters for the template.
parameters = \[\]

# An array to contain any conditions for the template
conditions = {}

# An array to contain any resource objects for the template.
resources = \[\]

# Build the template
t = Template()
t.add\_version('2010-09-09')
for p in parameters:
    t.add\_parameter(p)
for k in conditions:
    t.add\_condition(k, conditions\[k\])
for r in resources:
    t.add\_resource(r)

# Print the template to JSON
print(t.to\_json())

I’m not going to walk line by line through this next one, but suffice it to say it generates our template from before.

#!/usr/bin/python
# -\*- coding: utf-8 -\*-

# Import libraries
from troposphere import \*
from troposphere.ec2 import \*

# An array to contain any parameters for the template.
parameters = \[
    Parameter(
        "VPCCIDR",
        Description    = "The IP Address space used by this VPC. Changing the CIDR issues a replacement of the stack.",
        Type           = "String",
        Default        = "10.0.0.0/16",
        AllowedPattern = "(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})/(\\\\d{1,2})"
    ),
    Parameter(
        "VPCName",
        Description    = "The name of this VPC",
        Type           = "String",
        Default        = "Basic VPC"
    )
\]

# Append the required number of subnet parameters
i = 0
while i < 3:
    parameters.append(
        Parameter(
            "PublicSubnet"+ str(i + 1) +"CIDR",
            Description    = "The IP Address space used by this subnet. Changing the CIDR issues a replacement of the stack.",
            Type           = "String",
            Default        = "10.0.2.0/24",
            AllowedPattern = "(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})\\\\.(\\\\d{1,3})/(\\\\d{1,2})"
        )
    )
    parameters.append(
        Parameter(
            "PublicSubnet"+ str(i + 1) +"AZ",
            Description   = "The availaibility zone to place this subnet in. Changing the AZ issues a replacement of the stack.",
            Type          = "String",
            Default       = "us-east-1b",
            AllowedValues = \["us-east-1a","us-east-1b","us-east-1c","us-east-1e","us-west-2a","us-west-2b","us-west-2c"\]
        )
    )
    i += 1

# An array to contain any conditions for the template
conditions     = {}

ref\_stack\_id   = Ref('AWS::StackId')
ref\_region     = Ref('AWS::Region')
ref\_stack\_name = Ref('AWS::StackName')

# An array to contain any resource objects for the template.
resources = \[
    VPC(
        'VPC',
        CidrBlock            = Ref('VPCCIDR'),
        Tags                 = Tags(Name=Ref('VPCName')),
        EnableDnsSupport     = 'true',
        EnableDnsHostnames   = 'true'
    ),
    InternetGateway(
        'InternetGateway',
        Tags                 = Tags(Name=Ref('VPCName'))
    ),
    VPCGatewayAttachment(
        'AttachGateway',
        VpcId                = Ref('VPC'),
        InternetGatewayId    = Ref('InternetGateway')
    ),
    RouteTable(
        'PublicRouteTable',
        VpcId                = Ref('VPC')
    ),
    Route(
        'PublicRoute',
        DependsOn            = 'AttachGateway',
        GatewayId            = Ref('InternetGateway'),
        DestinationCidrBlock = '0.0.0.0/0',
        RouteTableId         = Ref('PublicRouteTable')
    ),
    NetworkAcl(
        'PublicSubnetAcl',
        VpcId                = Ref('VPC')
    ),
    NetworkAclEntry(
        'PublicInSubnetAclEntry',
        NetworkAclId         = Ref('PublicSubnetAcl'),
        RuleNumber           = '32000',
        Protocol             = '-1',
        PortRange            = PortRange(To='80', From='80'),
        Egress               = 'false',
        RuleAction           = 'allow',
        CidrBlock            = '0.0.0.0/0'
    ),
    NetworkAclEntry(
        'PublicOutSubnetAclEntry',
        NetworkAclId         = Ref('PublicSubnetAcl'),
        RuleNumber           = '32000',
        Protocol             = '-1',
        PortRange            = PortRange(To='1', From='65535'),
        Egress               = 'true',
        RuleAction           = 'allow',
        CidrBlock            = '0.0.0.0/0'
    )
\]

# Append the required number of subnet resources
i = 0
while i < 3:
    resources.append(
        Subnet(
            'PublicSubnet' + str(i + 1),
            CidrBlock      = '10.0.0.0/24',
            VpcId          = Ref('VPC'),
            Tags           = Tags(Application=ref\_stack\_id)
        )
    )
    resources.append(
        SubnetRouteTableAssociation(
            'Subnet' + str(i + 1) + 'RouteTableAssociation',
            SubnetId      = Ref('PublicSubnet' + str(i + 1)),
            RouteTableId  = Ref('PublicRouteTable')
        )
    )
    resources.append(
        SubnetNetworkAclAssociation(
            'Subnet' + str(i + 1) + 'NetworkAclAssociation',
            SubnetId      = Ref('PublicSubnet' + str(i + 1)),
            NetworkAclId  = Ref('PublicSubnetAcl')
        )
    )
    i += 1


# An array to contain any output objects for the template.
outputs = \[
    Output(
        "VPC",
        Value       = Ref('VPC'),
        Description = ""
    ),
    Output(
        "PublicRouteTable",
        Value       = Ref('PublicRouteTable'),
        Description = ""
    )
\]

# Append the required number of subnet parameters
i = 0
while i < 3:
    outputs.append(
        Output(
            "Subnet"+ str(i + 1) + "ID",
            Value       = Ref("PublicSubnet"+ str(i + 1)),
            Description = ""
        )
    )
    outputs.append(
        Output(
            "Subnet"+ str(i + 1) + "AZ",
            Value       = Ref("PublicSubnet"+ str(i + 1) +"AZ"),
            Description = ""
        )
    )
    i += 1

# Build the template
t = Template()
t.add\_version('2010-09-09')
t.add\_description("This CloudFormation template provisions a standard VPC with 3 subnets across 3 Availability Zones. All 3 are public.")
for p in parameters:
    t.add\_parameter(p)
for k in conditions:
    t.add\_condition(k, conditions\[k\])
for r in resources:
    t.add\_resource(r)
for o in outputs:
    t.add\_output(o)

# Print the template to JSON
print(t.to\_json())