Skip to content

Installation steps

Prepare local environment

Linux

Install following tools and packages:

Windows and macOS

Instruction for macOS and Windows is available in OpenStack installation steps.

Download packages

/vsphere/latest/cb-bosh-deployment-latest.tgz
/vsphere/latest/cb-cf-deployment-latest.tgz
/vsphere/latest/cb-concourse-deployment-latest.tgz
/vsphere/latest/cb-dns-deployment-latest.tgz
/vsphere/latest/cb-docker-images-latest.tgz
/vsphere/latest/cb-elk-deployment-latest.tgz
/vsphere/latest/cb-env-latest.tgz
/vsphere/latest/cb-installer-latest.tgz
/vsphere/latest/cb-k8s-deployment-latest.tgz
/vsphere/latest/cb-opscontrol-latest.tgz
/vsphere/latest/cb-utils-latest.tgz
/vshpere/latest/cb-common-latest.tgz
/vsphere/latest/cb-vault-deployment-latest.tgz
/vsphere/latest/cb-keycloak-deployment-latest.tgz

Unpack packages

tar -zxvf cb-installer-latest.tgz
mkdir cloudboostr
./unpackage.sh . cloudboostr latest

Navigate to cb-opscontrol/terraform/vsphere.

Prepare SSH keys

First step is to prepare SSH keys that will be used for jumpbox, k8s cluster and bosh connection. Also GIT key is required for terraform and scripts to download the required repositories.

If SSH keys were prepared before hand or are being created by external tool like Vault skip this and go to the next step.

# Create SSH keys

export EMAIL=[YOUR_EMAIL_HERE]
mkdir keys
ssh-keygen -t rsa -b 4096 -C "$EMAIL" -f ./keys/jumpbox_devops -N '' -m pem
ssh-keygen -t rsa -b 4096 -C "$EMAIL" -f ./keys/k8s_devops -N '' -m pem
ssh-keygen -t rsa -b 4096 -C "$EMAIL" -f ./keys/bosh_devops -N '' -m pem
ssh-keygen -t rsa -b 4096 -C "$EMAIL" -f ./keys/git_private_key -N '' -m pem

Prepare vSphere and NSX-T resources

vSphere

  1. Create resource pool if required

  2. Imported the following template into the resource pool: https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.ova Storage: thin provisioned

NSX-T

  1. Import certificates (System => Trust => Import Certificate)
opscontrol-kibana
env-cf

Upload keys

Then keys should be uploaded to the sensitive-data container. Filenames and container name can be changed in configuration if required.

aws s3 cp ./keys/jumpbox_devops      s3://sensitive-data/jumpbox_devops
aws s3 cp ./keys/jumpbox_devops.pub  s3://sensitive-data/jumpbox_devops.pub
aws s3 cp ./keys/k8s_devops          s3://sensitive-data/k8s_devops
aws s3 cp ./keys/k8s_devops.pub      s3://sensitive-data/k8s_devops.pub
aws s3 cp ./keys/git_private_key     s3://sensitive-data/git_private_key
aws s3 cp ./keys/git_private_key.pub s3://sensitive-data/git_private_key.pub

Configure terraform

Create file terraform.tfvars based on terraform.tfvars.example.

Fill the OpsControl configuration

At minimum update the required fields using configuration required and optional parameters page.

Example

nsxt_host = "<REDACTED>"
nsxt_username = "<REDACTED>"
nsxt_password = "<REDACTED>"
nsxt_allow_unverified_ssl = true

vsphere_host = "<REDACTED>"
vsphere_username = "<REDACTED>"
vsphere_password = "<REDACTED>"
vsphere_allow_unverified_ssl = true
vsphere_cluster = "<REDACTED>"
vsphere_datacenter = "<REDACTED>"
vsphere_resource_pool = "<REDACTED>"
vsphere_datastore = "<REDACTED>"
vsphere_bosh_datastore = "<REDACTED>"
vsphere_bosh_disks_folder = "<REDACTED>"
vsphere_bosh_template_folder = "<REDACTED>"
vsphere_bosh_vms_folder = "<REDACTED>"
edge_cluster_name = "<REDACTED>"

overlay_transport_zone_name = "<REDACTED>"
tier0_router_name = "<REDACTED>"
translated_snat_ip = "<REDACTED>"

sensitive_data_storage_container_name = "<REDACTED>"

# Keys filenames, public key for git_private_key should be added 
# to the Azure DevOps allowed SSH keys
sensitive_data_git_private_key_filename = "git_private_key"
sensitive_data_jumpbox_public_key_filename = "jumpbox_public_key"
sensitive_data_jumpbox_private_key_filename = "jumpbox_private_key"
sensitive_data_k8s_public_key_filename = "k8s_public_key"
sensitive_data_k8s_private_key_filename = "k8s_private_key"

bosh_state_container_name = "infrastructure-state"

### NETWORK #############################################################
# Network names sufix
env_name = "cbr-opscontrol"

jumpbox_whitelist = ["0.0.0.0/0"]
### DNS #################################################################
# DNS domain for externaly accessible services   [REQUIRED]
opscontrol_base_domain = "<REDACTED>"

# Floating IP created manually for the DNS       [REQUIRED]
dns_instance_public_ip = "<REDACTED>"

### DEPLOYMENTS CONFIG ##################################################
# Git repository address for the config files    [REQUIRED]
config_repository_url = "<REDACTED>"

# Git repository branch for the config files     [REQUIRED]
config_repository_branch = "master"

# Python3 alias used by the scripts
python_alias = "python3"

jumpbox_public_ip = "<REDACTED>"
jumpbox_private_ip = "<REDACTED>"
concourse_ui_lb_public_ip = "<REDACTED>"
control_plane_lb_public_ip = "<REDACTED>"

jumpbox_template_name = "jumpbox-template-ubuntu-xenial"

concourse_certificate_name = "opscontrol-concourse"
control_plane_certificate_name = "opscontrol-kibana"

public_dns_ip = "10.113.50.135"

cb_deployments_package_bucket = "packages"
cb_deployments_package_target_cloud = "vsphere"
cb_deployments_package_version = "latest"

ssh_allowed_hosts = "<REDACTED>"

Additionally there is are variables for the network configuration:

control_plane_backup_bucket_name = "backups"
prometheus_backup_bucket_name = "backups"
network_cidr = "<REDACTED>"
dmz_subnet_cidr = "<REDACTED>"
dmz_gateway_ip = "<REDACTED>"
dmz_reserved_ips = ["<REDACTED>"]
dmz_static_ips = ["<REDACTED>"]
dmz_dhcp_server_ip = "<REDACTED>"
dmz_dhcp_server_range_start = "<REDACTED>"
dmz_dhcp_server_range_end = "<REDACTED>"
dmz_router_ip = "<REDACTED>"
dns_instance_private_ip = "<REDACTED>"
mgmt_subnet_cidr = "<REDACTED>"
mgmt_reserved_ips = ["<REDACTED>"]
mgmt_gateway_ip = "<REDACTED>"
mgmt_dhcp_server_ip = "<REDACTED>"
mgmt_dhcp_server_range_start = "<REDACTED>"
mgmt_dhcp_server_range_end = "<REDACTED>"
mgmt_router_ip = "<REDACTED>"
telemetry_gateway_ip = "<REDACTED>"
telemetry_reserved_ips = ["<REDACTED>"]
telemetry_static_ips = ["<REDACTED>"]
telemetry_dhcp_server_ip = "<REDACTED>"
telemetry_dhcp_server_range_start = "<REDACTED>"
telemetry_dhcp_server_range_end = "<REDACTED>"
telemetry_router_ip = "<REDACTED>"
telemetry_subnet_cidr = "<REDACTED>"
http_proxy_url = "http://<REDACTED>:8080/"
https_proxy_url = "http://<REDACTED>:8080/"
no_proxy = "<REDACTED>"

Configure environments

Configure environments for your purposes. To use default configuration clone the cb-config repository and change mandatory fields for infrastructure, Kubernetes, Cloud Foundry and common for all components.

Address and branch of that repository should be correctly filled in the "Fill the OpsControl configuration" step. Also git private key in sensitive-data container must work for that repository.

Run terraform

Start the deployment operation in directory "terraform" in the opscontrol repository:

./terraform_init.sh vsphere
cd vshpere
terraform apply

Terraform backend

Terraform can be configured to use external backend instead of local files. To use that feature follow the guide in the official documentation.

The process of deployment may take from 30 minutes to two hours depending on the underlying infrastructure. If the error occurs there is an error message like:

module.custom_scripts.null_resource.deploy_dns (remote-exec): Task 31904 | 12:51:59 | Preparing deployment: Preparing deployment
module.custom_scripts.null_resource.deploy_dns (remote-exec):  (00:00:00)
module.custom_scripts.null_resource.deploy_dns (remote-exec):                      L Error: Instance group 'opscontrol-dns-master' references an unknown network 'dmz'
module.custom_scripts.null_resource.deploy_dns (remote-exec): Task 31904 | 12:51:59 | Error: Instance group 'opscontrol-dns-master' references an unknown network 'dmz'


Error: Error applying plan:

1 error(s) occurred:

* module.custom_scripts.null_resource.deploy_dns: error executing "/tmp/terraform_1281010007.sh": Process exited with status 1

The description of the error is in most cases enough to find out which configuration is wrong. After fixing the problem you can re-run the terraform apply and it will continue from where it finished.

After successfull deployment there will be an information message with the password to the concourse. Next step is the environment deployment which is described on this page.

vSphere specific configuration

  • nsx_password - NSX-T password (opscontrol_var: "nsx_password")
  • nsx_user - NSX-T username (opscontrol_var: "nsx_user")
  • nsx_host - NSX-T host (opscontrol_var: "nsx_address")
  • nsx_allow_unverified_ssl - Allow insecure SSL to NSX-T APIs (opscontrol_var: "nsx_allow_unverified_ssl")

  • nsx_ca - NSX-T CA (opscontrol_var: "nsx_ca")

  • vcenter_cluster - vCenter cluster (opscontrol_var: "vcenter_cluster")

  • vcenter_password - vCenter password (opscontrol_var: "vcenter_password")
  • vcenter_user - vCenter username (opscontrol_var: "vcenter_user")
  • vcenter_host - vCenter host (opscontrol_var: "vcenter_ip")
  • vcenter_datastore - vCenter datastore (opscontrol_var: "vcenter_ds")
  • vcenter_datacenter - vCenter datacenter (opscontrol_var: "vcenter_dc")
  • vcenter_resource_pool - vCenter resource pool

  • edge_cluster_name - vCenter edge cluster name

  • tier0_router_name - Tier0 router name (opscontrol_var: "tier0_router_name")

  • translated_snat_ip - Translated SNAT IP (opscontrol_var: "translated_snat_ip")
  • overlay_tz_name - Overlay transport zone name (opscontrol_var: "overlay_tz_name")
  • vcenter_allow_unverified_ssl - Allow insecure SSL to vCenter APIs (opscontrol_var: "vcenter_allow_unverified_ssl")

All network configuration for vSphere is different, don't use common configuration for network

  • mgmt_subnet_cidr - Management subnet CIDR (ex. "10.90.1.0/26"},
  • mgmt_dhcp_server_ip - Management subnet DHCP server IP (ex. "10.90.1.2/26")
  • mgmt_dhcp_server_range_start - Management subnet DHCP server range start (ex. "10.90.1.3")
  • mgmt_dhcp_server_range_end - Management subnet DHCP range end (ex. "10.90.1.10")
  • mgmt_router_ip - Management subnet router IP (ex. "10.90.1.1/26")
  • mgmt_reserved_ips - Management subnet reserver IPs range (ex. "10.90.1.2-10.90.1.10")

  • mgmt_gateway_ip - Management subnet gateway IP (ex. "10.90.1.1")

  • dmz_subnet_cidr - DMZ subnet CIDR (ex. "10.90.2.0/26"},

  • dmz_dhcp_server_ip - DMZ subnet DHCP server IP (ex. "10.90.2.2/26")
  • dmz_dhcp_server_range_start - DMZ subnet DHCP range start (ex. "10.90.2.3"},
  • dmz_dhcp_server_range_end - DMZ subnet DHCP range end (ex. "10.90.2.10")
  • dmz_router_ip - DMZ subnet router IP (ex. "10.90.2.1/26")
  • dmz_reserved_ips - DMZ subnet reserver ips range (ex. "10.90.2.2-10.90.2.10"},
  • dmz_static_ips - DMZ subnet static IPs range (ex. "10.90.2.11-10.90.2.20")

  • dmz_gateway_ip - DMZ subnet gateway IP (ex. "10.90.2.1")

  • services_subnet_cidr - Services subnet CIDR (ex. "10.90.4.0/22")

  • services_dhcp_server_ip - Services subnet DHCP server IP (ex. "10.90.4.2/22"},
  • services_dhcp_server_range_start - Services subnet DHCP range start (ex. "10.90.4.3")
  • services_dhcp_server_range_end - Services subnet DHCP range end (ex. "10.90.4.10")
  • services_router_ip - Services subnet router IP (ex. "10.90.4.1/22")
  • services_reserved_ips - Services subnet reserved IPs (ex. "10.90.4.2-10.90.4.10")

  • services_gateway_ip - Services subnet gateway IP (ex. "10.90.4.1")

  • cf_subnet_cidr - CF subnet CIDR (ex. "10.90.16.0/22")

  • cf_dhcp_server_ip - CF subnet DHCP server IP (ex. "10.90.16.2/22")
  • cf_dhcp_server_range_start - CF subnet DHCP range start (ex. "10.90.16.3")
  • cf_dhcp_server_range_end - CF subnet DHCP range end (ex. "10.90.16.10")
  • cf_router_ip - CF subnet router IP (ex. "10.90.16.1/22")
  • cf_reserved_ips - CF subnet reserved IPs range (ex. "10.90.16.2-10.90.16.10")

  • cf_gateway_ip - CF subnet gateway IP (ex. "10.90.16.1")

  • k8s_subnet_cidr - K8s subnet CIDR (ex. "10.90.32.0/22")

  • k8s_dhcp_server_ip - K8s subnet DHCP server IP (ex. "10.90.32.2/22")
  • k8s_dhcp_server_range_start - K8s subnet DHCP range start (ex. "10.90.32.3")
  • k8s_dhcp_server_range_end - K8s subnet DHCP range end (ex. "10.90.32.10")
  • k8s_router_ip - K8s subnet router IP (ex. "10.90.32.1/22")
  • k8s_reserved_ips - K8s subnet reserved IPs range (ex. "10.90.32.2-10.90.32.10")

  • k8s_gateway_ip - K8s subnet gateway IP (ex. "10.90.32.1")

  • cf_router_certificate_name - CF router certificate name (ex. "cf-router")

  • cf_router_lb_public_ip - CF router Load Balancer public IP (ex. "10.10.5.203")
  • k8s_lb_public_ip - K8s API Load Balancer public IP (ex. "10.10.5.204")
  • cf_ssh_lb_public_ip - (ex. "10.10.5.202")

  • jumpbox_public_ip - (ex. "10.10.5.201")

  • bosh_sg_name - BOSH security group name (ex. "env1-bosh")

  • cf_sg_name - CF security group name (ex. "env1-cf")
  • dns_sg_name - DNS security group name (ex. "env1-dns")

  • k8s_sg_name - K8s security group name (ex. "env1-k8s")

  • cf_router_pool_name - CF router Load Balancer pool name (ex. "cf-router")

  • cf_ssh_pool_name - CF SSH Load Balancer pool name (ex. "cf-ssh")
  • k8s_api_pool_name - K8s API Load Balancer pool name (ex. "k8s-api")
  • k8s_ingress_http_pool_name - K8s HTTP ingress Load Balancer pool name (ex. "k8s-ingress-http")
  • k8s_ingress_https_pool_name - K8s HTTPS ingress Load Balancer pool name (ex. "k8s-ingress-https")

Example

{
    "source_type": "package",
    "package_bucket": "cloudboostr-build-packages",
    "package_target_cloud": "vsphere",
    "package_version": "latest",

    "pipelines": [
        {
            "name": "deploy_infra",
            "file": "ci/pipelines/vsphere/deploy-pipeline.yml",
            "vars": []
        },
        {
            "name": "destroy_env",
            "file": "ci/pipelines/vsphere/destroy-pipeline.yml",
            "vars": []
        },
        {
            "name": "update_users",
            "file": "ci/pipelines/update-users.yml",
            "vars": [
                {"name": "timer_interval", "value": "24h"}
            ]
        }
    ],

    "vars": [
        {"name": "nsx_password", "opscontrol_var": "nsx_password"},
        {"name": "nsx_user", "opscontrol_var": "nsx_user"},
        {"name": "nsx_host", "opscontrol_var": "nsx_address"},
        {"name": "nsx_allow_unverified_ssl", "opscontrol_var": "nsx_allow_unverified_ssl"},
        {"name": "nsx_ca", "opscontrol_var": "nsx_ca"},
        {"name": "vcenter_cluster", "opscontrol_var": "vcenter_cluster"},
        {"name": "vcenter_password", "opscontrol_var": "vcenter_password"},
        {"name": "vcenter_user", "opscontrol_var": "vcenter_user"},
        {"name": "vcenter_host", "opscontrol_var": "vcenter_ip"},
        {"name": "vcenter_datastore", "opscontrol_var": "vcenter_ds"},
        {"name": "vcenter_datacenter", "opscontrol_var": "vcenter_dc"},
        {"name": "tier0_router_name", "opscontrol_var": "tier0_router_name"},
        {"name": "translated_snat_ip", "opscontrol_var": "translated_snat_ip"},
        {"name": "overlay_tz_name", "opscontrol_var": "overlay_tz_name"},
        {"name": "vcenter_allow_unverified_ssl", "opscontrol_var": "vcenter_allow_unverified_ssl"},

        {"name": "opscontrol_cidr", "opscontrol_var": "opscontrol_cidr"},

        {"name": "jumpbox_template_name", "value": "ubuntu-xenial-16.04-cloudimg"},
        {"name": "dns_instance_private_ip", "value": "10.90.2.11"},
        {"name": "public_dns_ip", "value": "10.10.4.2"},

        {"name": "network_cidr", "value": "10.90.0.0/16"},

        {"name": "mgmt_subnet_cidr", "value": "10.90.1.0/26"},
        {"name": "mgmt_dhcp_server_ip", "value": "10.90.1.2/26"},
        {"name": "mgmt_dhcp_server_range_start", "value": "10.90.1.3"},
        {"name": "mgmt_dhcp_server_range_end", "value": "10.90.1.10"},
        {"name": "mgmt_router_ip", "value": "10.90.1.1/26"},
        {"name": "mgmt_reserved_ips", "value": "10.90.1.2-10.90.1.10"},

        {"name": "mgmt_gateway_ip", "value": "10.90.1.1"},

        {"name": "dmz_subnet_cidr", "value": "10.90.2.0/26"},
        {"name": "dmz_dhcp_server_ip", "value": "10.90.2.2/26"},
        {"name": "dmz_dhcp_server_range_start", "value": "10.90.2.3"},
        {"name": "dmz_dhcp_server_range_end", "value": "10.90.2.10"},
        {"name": "dmz_router_ip", "value": "10.90.2.1/26"},
        {"name": "dmz_reserved_ips", "value": "10.90.2.2-10.90.2.10"},
        {"name": "dmz_static_ips", "value": "10.90.2.11-10.90.2.20"},

        {"name": "dmz_gateway_ip", "value": "10.90.2.1"},

        {"name": "services_subnet_cidr", "value": "10.90.4.0/22"},
        {"name": "services_dhcp_server_ip", "value": "10.90.4.2/22"},
        {"name": "services_dhcp_server_range_start", "value": "10.90.4.3"},
        {"name": "services_dhcp_server_range_end", "value": "10.90.4.10"},
        {"name": "services_router_ip", "value": "10.90.4.1/22"},
        {"name": "services_reserved_ips", "value": "10.90.4.2-10.90.4.10"},

        {"name": "services_gateway_ip", "value": "10.90.4.1"},

        {"name": "cf_subnet_cidr", "value": "10.90.16.0/22"},
        {"name": "cf_dhcp_server_ip", "value": "10.90.16.2/22"},
        {"name": "cf_dhcp_server_range_start", "value": "10.90.16.3"},
        {"name": "cf_dhcp_server_range_end", "value": "10.90.16.10"},
        {"name": "cf_router_ip", "value": "10.90.16.1/22"},
        {"name": "cf_reserved_ips", "value": "10.90.16.2-10.90.16.10"},

        {"name": "cf_gateway_ip", "value": "10.90.16.1"},

        {"name": "k8s_subnet_cidr", "value": "10.90.32.0/22"},
        {"name": "k8s_dhcp_server_ip", "value": "10.90.32.2/22"},
        {"name": "k8s_dhcp_server_range_start", "value": "10.90.32.3"},
        {"name": "k8s_dhcp_server_range_end", "value": "10.90.32.10"},
        {"name": "k8s_router_ip", "value": "10.90.32.1/22"},
        {"name": "k8s_reserved_ips", "value": "10.90.32.2-10.90.32.10"},

        {"name": "k8s_gateway_ip", "value": "10.90.32.1"},

        {"name": "cf_router_certificate_name", "value": "cf-router"},
        {"name": "cf_router_lb_public_ip", "value": "10.10.5.203"},
        {"name": "k8s_lb_public_ip", "value": "10.10.5.204"},
        {"name": "cf_ssh_lb_public_ip", "value": "10.10.5.202"},
        {"name": "jumpbox_public_ip", "value": "10.10.5.201"},

        {"name": "cf_sg_name", "value": "env1-cf"},
        {"name": "dns_sg_name", "value": "env1-dns"},
        {"name": "k8s_sg_name", "value": "env1-k8s"},

        {"name": "cf_router_pool_name", "value": "cf-router"},
        {"name": "cf_ssh_pool_name", "value": "cf-ssh"},
        {"name": "k8s_api_pool_name", "value": "k8s-api"},
        {"name": "k8s_ingress_http_pool_name", "value": "k8s-ingress-http"},
        {"name": "k8s_ingress_https_pool_name", "value": "k8s-ingress-https"},

        {"name": "jumpbox_whitelist", "value": "[0.0.0.0/0]"},


        {"name": "ssh_allowed_hosts", "opscontrol_var": "ssh_allowed_hosts"},

        {"name": "git_private_key", "opscontrol_var": "git_private_key"},
        {"name": "jumpbox_public_key", "opscontrol_var": "jumpbox_public_key"},
        {"name": "jumpbox_private_key", "opscontrol_var": "jumpbox_private_key"}
    ]
}