Installation steps

Prepare local environment


Install following tools and packages:

Windows and macOS

Instruction for macOS and Windows is available in OpenStack installation steps.

Download packages


Unpack packages

tar -zxvf cb-installer-latest.tgz
mkdir cloudboostr
./ . cloudboostr latest

Navigate to cb-opscontrol/terraform/vsphere.

Prepare SSH keys

First step is to prepare SSH keys that will be used for jumpbox and bosh connection. Also GIT key is required for terraform and scripts to download the required repositories.

If SSH keys were prepared before hand or are being created by external tool like Vault skip this and go to the next step.

# Create SSH keys

mkdir keys
ssh-keygen -t rsa -b 4096 -C "$EMAIL" -f ./keys/jumpbox_devops -N '' -m pem
ssh-keygen -t rsa -b 4096 -C "$EMAIL" -f ./keys/bosh_devops -N '' -m pem
ssh-keygen -t rsa -b 4096 -C "$EMAIL" -f ./keys/git_private_key -N '' -m pem

Prepare vSphere and NSX-T resources


  1. Create resource pool if required

  2. Imported the following template into the resource pool: Or when there is an error with incorrect Storage: thin provisioned


  1. Import certificates (System => Trust => Import Certificate)

Upload keys

Then keys should be uploaded to the sensitive-data container. Filenames and container name can be changed in configuration if required.

aws s3 cp ./keys/bosh_devops         s3://sensitive-data/bosh_devops
aws s3 cp ./keys/     s3://sensitive-data/
aws s3 cp ./keys/jumpbox_devops      s3://sensitive-data/jumpbox_devops
aws s3 cp ./keys/  s3://sensitive-data/
aws s3 cp ./keys/git_private_key     s3://sensitive-data/git_private_key
aws s3 cp ./keys/ s3://sensitive-data/

Configure terraform

Create file terraform.tfvars based on terraform.tfvars.example.

Fill the OpsControl configuration

At minimum update the required fields using configuration required and optional parameters page.


nsxt_host = "<REDACTED>"
nsxt_username = "<REDACTED>"
nsxt_password = "<REDACTED>"
nsxt_allow_unverified_ssl = true

vsphere_host = "<REDACTED>"
vsphere_username = "<REDACTED>"
vsphere_password = "<REDACTED>"
vsphere_allow_unverified_ssl = true
vsphere_cluster = "<REDACTED>"
vsphere_datacenter = "<REDACTED>"
vsphere_resource_pool = "<REDACTED>"
vsphere_datastore = "<REDACTED>"
vsphere_bosh_datastore = "<REDACTED>"
vsphere_bosh_disks_folder = "<REDACTED>"
vsphere_bosh_template_folder = "<REDACTED>"
vsphere_bosh_vms_folder = "<REDACTED>"
edge_cluster_name = "<REDACTED>"

overlay_transport_zone_name = "<REDACTED>"
tier0_router_name = "<REDACTED>"
translated_snat_ip = "<REDACTED>"

sensitive_data_storage_container_name = "<REDACTED>"

# Keys filenames, public key for git_private_key should be added 
# to the Azure DevOps allowed SSH keys
sensitive_data_git_private_key_filename = "git_private_key"
sensitive_data_bosh_public_key_filename = "bosh_public_key"
sensitive_data_bosh_private_key_filename = "bosh_private_key"
sensitive_data_jumpbox_public_key_filename = "jumpbox_public_key"
sensitive_data_jumpbox_private_key_filename = "jumpbox_private_key"

bosh_state_container_name = "infrastructure-state"

### NETWORK #############################################################
# Network names sufix
env_name = "cbr-opscontrol"

jumpbox_whitelist = [""]
### DNS #################################################################
# DNS domain for externaly accessible services   [REQUIRED]
opscontrol_base_domain = "<REDACTED>"

# Floating IP created manually for the DNS       [REQUIRED]
dns_instance_public_ip = "<REDACTED>"

### DEPLOYMENTS CONFIG ##################################################
# Git repository address for the config files    [REQUIRED]
config_repository_url = "<REDACTED>"

# Git repository branch for the config files     [REQUIRED]
config_repository_branch = "master"

# Python3 alias used by the scripts
python_alias = "python3"

jumpbox_public_ip = "<REDACTED>"
jumpbox_private_ip = "<REDACTED>"
concourse_ui_lb_public_ip = "<REDACTED>"
grafana_lb_public_ip = "<REDACTED>"
control_plane_lb_public_ip = "<REDACTED>"
uaa_lb_public_ip = "<REDACTED>"

jumpbox_template_name = "jumpbox-template-ubuntu-xenial"

concourse_certificate_name = "opscontrol-concourse"
grafana_certificate_name = "opscontrol-grafana"
control_plane_certificate_name = "opscontrol-kibana"

public_dns_ip = ""

dns_sg_name = "dns"
concourse_sg_name = "concourse"
grafana_sg_name = "grafana"
control_plane_sg_name = "elk"
bosh_sg_name = "bosh"

dns_sg_id = "<REDACTED>"
concourse_sg_id = "<REDACTED>"
grafana_sg_id = "<REDACTED>"
control_plane_sg_id = "<REDACTED>"
bosh_sg_id = "<REDACTED>"

grafana_pool_name = "grafana"
grafana_pool_id = "<REDACTED>"
control_plane_pool_name = "kibana"
control_plane_pool_id = "<REDACTED>"
concourse_pool_name = "concourse"
concourse_pool_id = "<REDACTED>"

cb_deployments_package_bucket = "packages"
cb_deployments_package_target_cloud = "vsphere"
cb_deployments_package_version = "latest"

ssh_allowed_hosts = "<REDACTED>"

Additionally there is are variables for the network configuration:

consul_ip = "<REDACTED>"
consul_private_ip = "<REDACTED>"
control_plane_backup_bucket_name = "backups"
prometheus_backup_bucket_name = "backups"
network_cidr = "<REDACTED>"
dmz_subnet_cidr = "<REDACTED>"
dmz_gateway_ip = "<REDACTED>"
dmz_reserved_ips = ["<REDACTED>"]
dmz_static_ips = ["<REDACTED>"]
dmz_dhcp_server_ip = "<REDACTED>"
dmz_dhcp_server_range_start = "<REDACTED>"
dmz_dhcp_server_range_end = "<REDACTED>"
dmz_router_ip = "<REDACTED>"
dns_instance_private_ip = "<REDACTED>"
mgmt_subnet_cidr = "<REDACTED>"
mgmt_reserved_ips = ["<REDACTED>"]
mgmt_gateway_ip = "<REDACTED>"
mgmt_dhcp_server_ip = "<REDACTED>"
mgmt_dhcp_server_range_start = "<REDACTED>"
mgmt_dhcp_server_range_end = "<REDACTED>"
mgmt_router_ip = "<REDACTED>"
bosh_ip = "<REDACTED>"
telemetry_gateway_ip = "<REDACTED>"
telemetry_reserved_ips = ["<REDACTED>"]
telemetry_static_ips = ["<REDACTED>"]
telemetry_dhcp_server_ip = "<REDACTED>"
telemetry_dhcp_server_range_start = "<REDACTED>"
telemetry_dhcp_server_range_end = "<REDACTED>"
telemetry_router_ip = "<REDACTED>"
telemetry_subnet_cidr = "<REDACTED>"
http_proxy_url = "http://<REDACTED>:8080/"
https_proxy_url = "http://<REDACTED>:8080/"
no_proxy = "<REDACTED>"

Configure environments

Configure environments for your purposes. To use default configuration clone the cb-config repository and change mandatory fields for infrastructure, Kubernetes, Cloud Foundry and common for all components.

Address and branch of that repository should be correctly filled in the "Fill the OpsControl configuration" step. Also git private key in sensitive-data container must work for that repository.

Run terraform

Start the deployment operation in directory "terraform" in the opscontrol repository:

terraform init
terraform apply

Terraform backend

Terraform can be configured to use external backend instead of local files. To use that feature follow the guide in the official documentation.

The process of deployment may take from 30 minutes to two hours depending on the underlying infrastructure. If the error occurs there is an error message like:

module.custom_scripts.null_resource.deploy_dns (remote-exec): Task 31904 | 12:51:59 | Preparing deployment: Preparing deployment
module.custom_scripts.null_resource.deploy_dns (remote-exec):  (00:00:00)
module.custom_scripts.null_resource.deploy_dns (remote-exec):                      L Error: Instance group 'opscontrol-dns-master' references an unknown network 'dmz'
module.custom_scripts.null_resource.deploy_dns (remote-exec): Task 31904 | 12:51:59 | Error: Instance group 'opscontrol-dns-master' references an unknown network 'dmz'

Error: Error applying plan:

1 error(s) occurred:

* module.custom_scripts.null_resource.deploy_dns: error executing "/tmp/": Process exited with status 1

The description of the error is in most cases enough to find out which configuration is wrong. After fixing the problem you can re-run the terraform apply and it will continue from where it finished.

After successfull deployment there will be an information message with the password to the concourse. Next step is the environment deployment which is described on this page.

vSphere specific configuration

  • nsx_password - NSX-T password (opscontrol_var: "nsx_password")
  • nsx_user - NSX-T username (opscontrol_var: "nsx_user")
  • nsx_host - NSX-T host (opscontrol_var: "nsx_address")
  • nsx_allow_unverified_ssl - Allow insecure SSL to NSX-T APIs (opscontrol_var: "nsx_allow_unverified_ssl")
  • nsx_ca - NSX-T CA (opscontrol_var: "nsx_ca")

  • vcenter_cluster - vCenter cluster (opscontrol_var: "vcenter_cluster")

  • vcenter_disks - vCenter disks (opscontrol_var: "vcenter_disks")
  • vcenter_vms - vCenter VMs (opscontrol_var: "vcenter_vms")
  • vcenter_templates - vCenter templates (opscontrol_var: "vcenter_templates")
  • vcenter_password - vCenter password (opscontrol_var: "vcenter_password")
  • vcenter_user - vCenter username (opscontrol_var: "vcenter_user")
  • vcenter_host - vCenter host (opscontrol_var: "vcenter_ip")
  • vcenter_datastore - vCenter datastore (opscontrol_var: "vcenter_ds")
  • vcenter_datacenter - vCenter datacenter (opscontrol_var: "vcenter_dc")
  • vcenter_resource_pool - vCenter resource pool
  • edge_cluster_name - vCenter edge cluster name

  • tier0_router_name - Tier0 router name (opscontrol_var: "tier0_router_name")

  • translated_snat_ip - Translated SNAT IP (opscontrol_var: "translated_snat_ip")
  • overlay_tz_name - Overlay transport zone name (opscontrol_var: "overlay_tz_name")
  • vcenter_allow_unverified_ssl - Allow insecure SSL to vCenter APIs (opscontrol_var: "vcenter_allow_unverified_ssl")

  • vcenter_bosh_datastore - vCenter BOSH datastore (opscontrol_var: "vcenter_bosh_datastore")

  • vcenter_bosh_disks_folder - vCenter BOSH disks folder (opscontrol_var: "vcenter_bosh_disks_folder")
  • vcenter_bosh_template_folder - vCenter BOSH template folder (opscontrol_var: "vcenter_bosh_template_folder")
  • vcenter_bosh_vms_folder - vCenter BOSH VMs folder (opscontrol_var: "vcenter_bosh_vms_folder")

  • bosh_vm_cpu - BOSH director VM vCPU (ex. "2")

  • bosh_vm_ram - BOSH director VM RAM (ex. "2048")
  • bosh_vm_disk - BOSH director VM disk size in MB (ex. "20000")

All network configuration for vSphere is different, don't use common configuration for network

  • mgmt_subnet_cidr - Management subnet CIDR (ex. ""},
  • mgmt_dhcp_server_ip - Management subnet DHCP server IP (ex. "")
  • mgmt_dhcp_server_range_start - Management subnet DHCP server range start (ex. "")
  • mgmt_dhcp_server_range_end - Management subnet DHCP range end (ex. "")
  • mgmt_router_ip - Management subnet router IP (ex. "")
  • mgmt_reserved_ips - Management subnet reserver IPs range (ex. "")
  • mgmt_gateway_ip - Management subnet gateway IP (ex. "")

  • dmz_subnet_cidr - DMZ subnet CIDR (ex. ""},

  • dmz_dhcp_server_ip - DMZ subnet DHCP server IP (ex. "")
  • dmz_dhcp_server_range_start - DMZ subnet DHCP range start (ex. ""},
  • dmz_dhcp_server_range_end - DMZ subnet DHCP range end (ex. "")
  • dmz_router_ip - DMZ subnet router IP (ex. "")
  • dmz_reserved_ips - DMZ subnet reserver ips range (ex. ""},
  • dmz_static_ips - DMZ subnet static IPs range (ex. "")
  • dmz_gateway_ip - DMZ subnet gateway IP (ex. "")

  • services_subnet_cidr - Services subnet CIDR (ex. "")

  • services_dhcp_server_ip - Services subnet DHCP server IP (ex. ""},
  • services_dhcp_server_range_start - Services subnet DHCP range start (ex. "")
  • services_dhcp_server_range_end - Services subnet DHCP range end (ex. "")
  • services_router_ip - Services subnet router IP (ex. "")
  • services_reserved_ips - Services subnet reserved IPs (ex. "")
  • services_gateway_ip - Services subnet gateway IP (ex. "")

  • cf_subnet_cidr - CF subnet CIDR (ex. "")

  • cf_dhcp_server_ip - CF subnet DHCP server IP (ex. "")
  • cf_dhcp_server_range_start - CF subnet DHCP range start (ex. "")
  • cf_dhcp_server_range_end - CF subnet DHCP range end (ex. "")
  • cf_router_ip - CF subnet router IP (ex. "")
  • cf_reserved_ips - CF subnet reserved IPs range (ex. "")
  • cf_gateway_ip - CF subnet gateway IP (ex. "")

  • k8s_subnet_cidr - K8s subnet CIDR (ex. "")

  • k8s_dhcp_server_ip - K8s subnet DHCP server IP (ex. "")
  • k8s_dhcp_server_range_start - K8s subnet DHCP range start (ex. "")
  • k8s_dhcp_server_range_end - K8s subnet DHCP range end (ex. "")
  • k8s_router_ip - K8s subnet router IP (ex. "")
  • k8s_reserved_ips - K8s subnet reserved IPs range (ex. "")
  • k8s_gateway_ip - K8s subnet gateway IP (ex. "")

  • cf_router_certificate_name - CF router certificate name (ex. "cf-router")

  • cf_router_lb_public_ip - CF router Load Balancer public IP (ex. "")
  • cf_router_pool_id - CF router Load Balancer pool id
  • cf_ssh_pool_id - CF ssh Load Balancer pool id
  • k8s_lb_public_ip - K8s API Load Balancer public IP (ex. "")
  • k8s_api_pool_id - K8s API Load Balancer pool id
  • k8s_ingress_http_pool_id - K8s ingress Load Balancer pool id
  • k8s_ingress_https_pool_id - K8s ingress Load Balancer pool id
  • cf_ssh_lb_public_ip - (ex. "")
  • jumpbox_public_ip - (ex. "")
  • bosh_sg_id - BOSH subnet security group id
  • cf_sg_id - CF subnet security group id
  • dns_sg_id - DNS subnet security group id
  • k8s_sg_id - K8s subnet security group id

  • bosh_sg_name - BOSH security group name (ex. "env1-bosh")

  • cf_sg_name - CF security group name (ex. "env1-cf")
  • dns_sg_name - DNS security group name (ex. "env1-dns")
  • k8s_sg_name - K8s security group name (ex. "env1-k8s")

  • cf_router_pool_name - CF router Load Balancer pool name (ex. "cf-router")

  • cf_ssh_pool_name - CF SSH Load Balancer pool name (ex. "cf-ssh")
  • k8s_api_pool_name - K8s API Load Balancer pool name (ex. "k8s-api")
  • k8s_ingress_http_pool_name - K8s HTTP ingress Load Balancer pool name (ex. "k8s-ingress-http")
  • k8s_ingress_https_pool_name - K8s HTTPS ingress Load Balancer pool name (ex. "k8s-ingress-https")


    "source_type": "package",
    "package_bucket": "cloudboostr-build-packages",
    "package_target_cloud": "vsphere",
    "package_version": "latest",

    "pipelines": [
            "name": "deploy_bosh",
            "file": "ci/pipelines/vsphere/deploy-pipeline.yml",
            "vars": []
            "name": "destroy_env",
            "file": "ci/pipelines/vsphere/destroy-pipeline.yml",
            "vars": []
            "name": "backup_bosh",
            "file": "ci/pipelines/backup-pipeline.yml",
            "vars": [
                {"name": "timer_interval", "value": "24h"}
            "name": "restore_latest_bosh",
            "file": "ci/pipelines/restore-latest-pipeline.yml",
            "vars": []
            "name": "restore_custom_bosh",
            "file": "ci/pipelines/restore-custom-pipeline.yml",
            "vars": []

    "vars": [
        {"name": "nsx_password", "opscontrol_var": "nsx_password"},
        {"name": "nsx_user", "opscontrol_var": "nsx_user"},
        {"name": "nsx_host", "opscontrol_var": "nsx_address"},
        {"name": "nsx_allow_unverified_ssl", "opscontrol_var": "nsx_allow_unverified_ssl"},
        {"name": "nsx_ca", "opscontrol_var": "nsx_ca"},
        {"name": "vcenter_cluster", "opscontrol_var": "vcenter_cluster"},
        {"name": "vcenter_disks", "opscontrol_var": "vcenter_disks"},
        {"name": "vcenter_vms", "opscontrol_var": "vcenter_vms"},
        {"name": "vcenter_templates", "opscontrol_var": "vcenter_templates"},
        {"name": "vcenter_password", "opscontrol_var": "vcenter_password"},
        {"name": "vcenter_user", "opscontrol_var": "vcenter_user"},
        {"name": "vcenter_host", "opscontrol_var": "vcenter_ip"},
        {"name": "vcenter_datastore", "opscontrol_var": "vcenter_ds"},
        {"name": "vcenter_datacenter", "opscontrol_var": "vcenter_dc"},
        {"name": "tier0_router_name", "opscontrol_var": "tier0_router_name"},
        {"name": "translated_snat_ip", "opscontrol_var": "translated_snat_ip"},
        {"name": "overlay_tz_name", "opscontrol_var": "overlay_tz_name"},
        {"name": "vcenter_allow_unverified_ssl", "opscontrol_var": "vcenter_allow_unverified_ssl"},

        {"name": "vcenter_bosh_datastore", "opscontrol_var": "vcenter_bosh_datastore"},
        {"name": "vcenter_bosh_disks_folder", "opscontrol_var": "vcenter_bosh_disks_folder"},
        {"name": "vcenter_bosh_template_folder", "opscontrol_var": "vcenter_bosh_template_folder"},
        {"name": "vcenter_bosh_vms_folder", "opscontrol_var": "vcenter_bosh_vms_folder"},

        {"name": "bosh_vm_cpu", "value": "2"},
        {"name": "bosh_vm_ram", "value": "2048"},
        {"name": "bosh_vm_disk", "value": "20000"},

        {"name": "opscontrol_cidr", "opscontrol_var": "opscontrol_cidr"},

        {"name": "jumpbox_template_name", "value": "ubuntu-xenial-16.04-cloudimg"},
        {"name": "dns_instance_private_ip", "value": ""},
        {"name": "public_dns_ip", "value": ""},

        {"name": "network_cidr", "value": ""},

        {"name": "mgmt_subnet_cidr", "value": ""},
        {"name": "mgmt_dhcp_server_ip", "value": ""},
        {"name": "mgmt_dhcp_server_range_start", "value": ""},
        {"name": "mgmt_dhcp_server_range_end", "value": ""},
        {"name": "mgmt_router_ip", "value": ""},
        {"name": "mgmt_reserved_ips", "value": ""},

        {"name": "mgmt_gateway_ip", "value": ""},

        {"name": "dmz_subnet_cidr", "value": ""},
        {"name": "dmz_dhcp_server_ip", "value": ""},
        {"name": "dmz_dhcp_server_range_start", "value": ""},
        {"name": "dmz_dhcp_server_range_end", "value": ""},
        {"name": "dmz_router_ip", "value": ""},
        {"name": "dmz_reserved_ips", "value": ""},
        {"name": "dmz_static_ips", "value": ""},

        {"name": "dmz_gateway_ip", "value": ""},

        {"name": "services_subnet_cidr", "value": ""},
        {"name": "services_dhcp_server_ip", "value": ""},
        {"name": "services_dhcp_server_range_start", "value": ""},
        {"name": "services_dhcp_server_range_end", "value": ""},
        {"name": "services_router_ip", "value": ""},
        {"name": "services_reserved_ips", "value": ""},

        {"name": "services_gateway_ip", "value": ""},

        {"name": "cf_subnet_cidr", "value": ""},
        {"name": "cf_dhcp_server_ip", "value": ""},
        {"name": "cf_dhcp_server_range_start", "value": ""},
        {"name": "cf_dhcp_server_range_end", "value": ""},
        {"name": "cf_router_ip", "value": ""},
        {"name": "cf_reserved_ips", "value": ""},

        {"name": "cf_gateway_ip", "value": ""},

        {"name": "k8s_subnet_cidr", "value": ""},
        {"name": "k8s_dhcp_server_ip", "value": ""},
        {"name": "k8s_dhcp_server_range_start", "value": ""},
        {"name": "k8s_dhcp_server_range_end", "value": ""},
        {"name": "k8s_router_ip", "value": ""},
        {"name": "k8s_reserved_ips", "value": ""},

        {"name": "k8s_gateway_ip", "value": ""},

        {"name": "cf_router_certificate_name", "value": "cf-router"},
        {"name": "cf_router_lb_public_ip", "value": ""},
        {"name": "cf_router_pool_id", "value": "880365fc-e51b-43fc-8a21-8c81c7b45c86"},
        {"name": "cf_ssh_pool_id", "value": "8b517426-7abe-4be8-9f89-a480d8e6d176"},
        {"name": "k8s_lb_public_ip", "value": ""},
        {"name": "k8s_api_pool_id", "value": "ea24e0e9-c767-4826-95d3-f2898104d5cf"},
        {"name": "k8s_ingress_http_pool_id", "value": "d2d49418-5bad-4ac1-944c-71ab6af916fe"},
        {"name": "k8s_ingress_https_pool_id", "value": "ca503518-3f92-44d3-8ca5-22914f37c75a"},
        {"name": "cf_ssh_lb_public_ip", "value": ""},
        {"name": "jumpbox_public_ip", "value": ""},
        {"name": "bosh_sg_id", "value": "30a2c141-9871-4fdc-87fd-403bcc76e595"},
        {"name": "cf_sg_id", "value": "393e6c31-e14c-4ec8-9cd6-b1c6c6d2ea30"},
        {"name": "dns_sg_id", "value": "a6ebf127-8492-40e4-a1be-0c16437c1f17"},
        {"name": "k8s_sg_id", "value": "e478e527-5cdb-4a7e-b4d3-42fb7c03d200"},

        {"name": "bosh_sg_name", "value": "env1-bosh"},
        {"name": "cf_sg_name", "value": "env1-cf"},
        {"name": "dns_sg_name", "value": "env1-dns"},
        {"name": "k8s_sg_name", "value": "env1-k8s"},

        {"name": "cf_router_pool_name", "value": "cf-router"},
        {"name": "cf_ssh_pool_name", "value": "cf-ssh"},
        {"name": "k8s_api_pool_name", "value": "k8s-api"},
        {"name": "k8s_ingress_http_pool_name", "value": "k8s-ingress-http"},
        {"name": "k8s_ingress_https_pool_name", "value": "k8s-ingress-https"},

        {"name": "jumpbox_whitelist", "value": "[]"},

        {"name": "bosh_private_ip", "value": ""},
        {"name": "bosh_instance_type", "value": "t2.small"},
        {"name": "bosh_director_name", "value": "bosh"},

        {"name": "ssh_allowed_hosts", "opscontrol_var": "ssh_allowed_hosts"},

        {"name": "git_private_key", "opscontrol_var": "git_private_key"},
        {"name": "jumpbox_public_key", "opscontrol_var": "jumpbox_public_key"},
        {"name": "jumpbox_private_key", "opscontrol_var": "jumpbox_private_key"},
        {"name": "bosh_private_key", "opscontrol_var": "bosh_private_key"},
        {"name": "bosh_public_key", "opscontrol_var": "bosh_public_key"}