# provider "azurerm" { # subscription_id = "${var.subscription_id}" # client_id = "${var.client_id}" # client_secret = "${var.client_secret}" # tenant_id = "${var.tenant_id}" # } resource "azurerm_resource_group" "rg" { name = "${var.resource_group}" location = "${var.location}" } # ********************** NETWORK SECURITY GROUPS ********************** # resource "azurerm_network_security_group" "master" { name = "${var.nsg_spark_master_name}" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" security_rule { name = "ssh" description = "Allow SSH" priority = 100 direction = "Inbound" access = "Allow" protocol = "Tcp" source_port_range = "*" destination_port_range = "22" source_address_prefix = "Internet" destination_address_prefix = "*" } security_rule { name = "http_webui_spark" description = "Allow Web UI Access to Spark" priority = 101 direction = "Inbound" access = "Allow" protocol = "Tcp" source_port_range = "*" destination_port_range = "8080" source_address_prefix = "Internet" destination_address_prefix = "*" } security_rule { name = "http_rest_spark" description = "Allow REST API Access to Spark" priority = 102 direction = "Inbound" access = "Allow" protocol = "Tcp" source_port_range = "*" destination_port_range = "6066" source_address_prefix = "Internet" destination_address_prefix = "*" } } resource "azurerm_network_security_group" "slave" { name = "${var.nsg_spark_slave_name}" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" security_rule { name = "ssh" description = "Allow SSH" priority = 100 direction = "Inbound" access = "Allow" protocol = "Tcp" source_port_range = "*" destination_port_range = "22" source_address_prefix = "Internet" destination_address_prefix = "*" } } resource "azurerm_network_security_group" "cassandra" { name = "${var.nsg_cassandra_name}" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" security_rule { name = "ssh" description = "Allow SSH" priority = 100 direction = "Inbound" access = "Allow" protocol = "Tcp" source_port_range = "*" destination_port_range = "22" source_address_prefix = "Internet" destination_address_prefix = "*" } } # ********************** VNET / SUBNETS ********************** # resource "azurerm_virtual_network" "spark" { name = "vnet-spark" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" address_space = ["${var.vnet_spark_prefix}"] } resource "azurerm_subnet" "subnet1" { name = "${var.vnet_spark_subnet1_name}" virtual_network_name = "${azurerm_virtual_network.spark.name}" resource_group_name = "${azurerm_resource_group.rg.name}" address_prefix = "${var.vnet_spark_subnet1_prefix}" network_security_group_id = "${azurerm_network_security_group.master.id}" depends_on = ["azurerm_virtual_network.spark"] } resource "azurerm_subnet" "subnet2" { name = "${var.vnet_spark_subnet2_name}" virtual_network_name = "${azurerm_virtual_network.spark.name}" resource_group_name = "${azurerm_resource_group.rg.name}" address_prefix = "${var.vnet_spark_subnet2_prefix}" } resource "azurerm_subnet" "subnet3" { name = "${var.vnet_spark_subnet3_name}" virtual_network_name = "${azurerm_virtual_network.spark.name}" resource_group_name = "${azurerm_resource_group.rg.name}" address_prefix = "${var.vnet_spark_subnet3_prefix}" } # ********************** PUBLIC IP ADDRESSES ********************** # resource "azurerm_public_ip" "master" { name = "${var.public_ip_master_name}" location = "${azurerm_resource_group.rg.location}" resource_group_name = "${azurerm_resource_group.rg.name}" public_ip_address_allocation = "Static" } resource "azurerm_public_ip" "slave" { name = "${var.public_ip_slave_name_prefix}${count.index}" location = "${azurerm_resource_group.rg.location}" resource_group_name = "${azurerm_resource_group.rg.name}" public_ip_address_allocation = "Static" count = "${var.vm_number_of_slaves}" } resource "azurerm_public_ip" "cassandra" { name = "${var.public_ip_cassandra_name}" location = "${azurerm_resource_group.rg.location}" resource_group_name = "${azurerm_resource_group.rg.name}" public_ip_address_allocation = "Static" } # ********************** NETWORK INTERFACE ********************** # resource "azurerm_network_interface" "master" { name = "${var.nic_master_name}" location = "${azurerm_resource_group.rg.location}" resource_group_name = "${azurerm_resource_group.rg.name}" network_security_group_id = "${azurerm_network_security_group.master.id}" depends_on = ["azurerm_virtual_network.spark", "azurerm_public_ip.master", "azurerm_network_security_group.master"] ip_configuration { name = "ipconfig1" subnet_id = "${azurerm_subnet.subnet1.id}" private_ip_address_allocation = "Static" private_ip_address = "${var.nic_master_node_ip}" public_ip_address_id = "${azurerm_public_ip.master.id}" } } resource "azurerm_network_interface" "slave" { name = "${var.nic_slave_name_prefix}${count.index}" location = "${azurerm_resource_group.rg.location}" resource_group_name = "${azurerm_resource_group.rg.name}" network_security_group_id = "${azurerm_network_security_group.slave.id}" count = "${var.vm_number_of_slaves}" depends_on = ["azurerm_virtual_network.spark", "azurerm_public_ip.slave", "azurerm_network_security_group.slave"] ip_configuration { name = "ipconfig1" subnet_id = "${azurerm_subnet.subnet2.id}" private_ip_address_allocation = "Static" private_ip_address = "${var.nic_slave_node_ip_prefix}${5 + count.index}" public_ip_address_id = "${element(azurerm_public_ip.slave.*.id, count.index)}" } } resource "azurerm_network_interface" "cassandra" { name = "${var.nic_cassandra_name}" location = "${azurerm_resource_group.rg.location}" resource_group_name = "${azurerm_resource_group.rg.name}" network_security_group_id = "${azurerm_network_security_group.cassandra.id}" depends_on = ["azurerm_virtual_network.spark", "azurerm_public_ip.cassandra", "azurerm_network_security_group.cassandra"] ip_configuration { name = "ipconfig1" subnet_id = "${azurerm_subnet.subnet3.id}" private_ip_address_allocation = "Static" private_ip_address = "${var.nic_cassandra_node_ip}" public_ip_address_id = "${azurerm_public_ip.cassandra.id}" } } # ********************** AVAILABILITY SET ********************** # resource "azurerm_availability_set" "slave" { name = "${var.availability_slave_name}" location = "${azurerm_resource_group.rg.location}" resource_group_name = "${azurerm_resource_group.rg.name}" platform_update_domain_count = 5 platform_fault_domain_count = 2 } # ********************** STORAGE ACCOUNTS ********************** # resource "azurerm_storage_account" "master" { name = "master${var.unique_prefix}" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" account_type = "${var.storage_master_type}" } resource "azurerm_storage_container" "master" { name = "${var.vm_master_storage_account_container_name}" resource_group_name = "${azurerm_resource_group.rg.name}" storage_account_name = "${azurerm_storage_account.master.name}" container_access_type = "private" depends_on = ["azurerm_storage_account.master"] } resource "azurerm_storage_account" "slave" { name = "slave${var.unique_prefix}${count.index}" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" count = "${var.vm_number_of_slaves}" account_type = "${var.storage_slave_type}" } resource "azurerm_storage_container" "slave" { name = "${var.vm_slave_storage_account_container_name}${count.index}" resource_group_name = "${azurerm_resource_group.rg.name}" storage_account_name = "${element(azurerm_storage_account.slave.*.name, count.index)}" container_access_type = "private" depends_on = ["azurerm_storage_account.slave"] } resource "azurerm_storage_account" "cassandra" { name = "cassandra${var.unique_prefix}" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" account_type = "${var.storage_cassandra_type}" } resource "azurerm_storage_container" "cassandra" { name = "${var.vm_cassandra_storage_account_container_name}" resource_group_name = "${azurerm_resource_group.rg.name}" storage_account_name = "${azurerm_storage_account.cassandra.name}" container_access_type = "private" depends_on = ["azurerm_storage_account.cassandra"] } # ********************** MASTER VIRTUAL MACHINE ********************** # resource "azurerm_virtual_machine" "master" { name = "${var.vm_master_name}" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" vm_size = "${var.vm_master_vm_size}" network_interface_ids = ["${azurerm_network_interface.master.id}"] depends_on = ["azurerm_storage_account.master", "azurerm_network_interface.master", "azurerm_storage_container.master"] storage_image_reference { publisher = "${var.os_image_publisher}" offer = "${var.os_image_offer}" sku = "${var.os_version}" version = "latest" } storage_os_disk { name = "${var.vm_master_os_disk_name}" vhd_uri = "http://${azurerm_storage_account.master.name}.blob.core.windows.net/${azurerm_storage_container.master.name}/${var.vm_master_os_disk_name}.vhd" create_option = "FromImage" caching = "ReadWrite" } os_profile { computer_name = "${var.vm_master_name}" admin_username = "${var.vm_admin_username}" admin_password = "${var.vm_admin_password}" } os_profile_linux_config { disable_password_authentication = false } connection { type = "ssh" host = "${azurerm_public_ip.master.ip_address}" user = "${var.vm_admin_username}" password = "${var.vm_admin_password}" } provisioner "remote-exec" { inline = [ "wget ${var.artifacts_location}${var.script_spark_provisioner_script_file_name}", "echo ${var.vm_admin_password} | sudo -S sh ./${var.script_spark_provisioner_script_file_name} -runas=master -master=${var.nic_master_node_ip}", ] } } # ********************** SLAVE VIRTUAL MACHINES ********************** # resource "azurerm_virtual_machine" "slave" { name = "${var.vm_slave_name_prefix}${count.index}" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" vm_size = "${var.vm_slave_vm_size}" network_interface_ids = ["${element(azurerm_network_interface.slave.*.id, count.index)}"] count = "${var.vm_number_of_slaves}" availability_set_id = "${azurerm_availability_set.slave.id}" depends_on = ["azurerm_storage_account.slave", "azurerm_network_interface.slave", "azurerm_storage_container.slave"] storage_image_reference { publisher = "${var.os_image_publisher}" offer = "${var.os_image_offer}" sku = "${var.os_version}" version = "latest" } storage_os_disk { name = "${var.vm_slave_os_disk_name_prefix}${count.index}" vhd_uri = "http://${element(azurerm_storage_account.slave.*.name, count.index)}.blob.core.windows.net/${element(azurerm_storage_container.slave.*.name, count.index)}/${var.vm_slave_os_disk_name_prefix}.vhd" create_option = "FromImage" caching = "ReadWrite" } os_profile { computer_name = "${var.vm_slave_name_prefix}${count.index}" admin_username = "${var.vm_admin_username}" admin_password = "${var.vm_admin_password}" } os_profile_linux_config { disable_password_authentication = false } connection { type = "ssh" host = "${element(azurerm_public_ip.slave.*.ip_address, count.index)}" user = "${var.vm_admin_username}" password = "${var.vm_admin_password}" } provisioner "remote-exec" { inline = [ "wget ${var.artifacts_location}${var.script_spark_provisioner_script_file_name}", "echo ${var.vm_admin_password} | sudo -S sh ./${var.script_spark_provisioner_script_file_name} -runas=slave -master=${var.nic_master_node_ip}", ] } } # ********************** CASSANDRA VIRTUAL MACHINE ********************** # resource "azurerm_virtual_machine" "cassandra" { name = "${var.vm_cassandra_name}" resource_group_name = "${azurerm_resource_group.rg.name}" location = "${azurerm_resource_group.rg.location}" vm_size = "${var.vm_cassandra_vm_size}" network_interface_ids = ["${azurerm_network_interface.cassandra.id}"] depends_on = ["azurerm_storage_account.cassandra", "azurerm_network_interface.cassandra", "azurerm_storage_container.cassandra"] storage_image_reference { publisher = "${var.os_image_publisher}" offer = "${var.os_image_offer}" sku = "${var.os_version}" version = "latest" } storage_os_disk { name = "${var.vm_cassandra_os_disk_name}" vhd_uri = "http://${azurerm_storage_account.cassandra.name}.blob.core.windows.net/${azurerm_storage_container.cassandra.name}/${var.vm_cassandra_os_disk_name}.vhd" create_option = "FromImage" caching = "ReadWrite" } os_profile { computer_name = "${var.vm_cassandra_name}" admin_username = "${var.vm_admin_username}" admin_password = "${var.vm_admin_password}" } os_profile_linux_config { disable_password_authentication = false } connection { type = "ssh" host = "${azurerm_public_ip.cassandra.ip_address}" user = "${var.vm_admin_username}" password = "${var.vm_admin_password}" } provisioner "remote-exec" { inline = [ "wget ${var.artifacts_location}${var.script_cassandra_provisioner_script_file_name}", "echo ${var.vm_admin_password} | sudo -S sh ./${var.script_cassandra_provisioner_script_file_name}", ] } }